less mem dont remove null

lib/handlers/upload_handler.dart

@@ -49,10 +49,10 @@ Future<Response> handleFileUpload(Request request) async {
           await part.pipe(tempFile.openWrite());
 
           // Upload to Supabase
-          final bytes = await tempFile.readAsBytes();
+
           await supabase.storage.from('csvhich').uploadBinary(
                 filename,
-                bytes,
+                await tempFile.readAsBytes(),
                 fileOptions: FileOptions(
                   upsert: true,
                   contentType: part.headers['content-type'],
@@ -75,13 +75,19 @@ Future<Response> handleFileUpload(Request request) async {
 
           //upload file to storage archive bucket
           final archiveFilename = 'upload/${timestamp}_$filename';
-          await supabase.storage.from('csvhich_archive').uploadBinary(
+
+          // await supabase.storage.from('csvhich_archive').uploadBinary(
+          //       archiveFilename,
+          //       await tempFile.readAsBytes(),
+          //       fileOptions: FileOptions(
+          //         upsert: true,
+          //         contentType: part.headers['content-type'],
+          //       ),
+          //     );
+          await supabase.storage.from('csvhich').copy(
+                filename,
                 archiveFilename,
-                bytes,
-                fileOptions: FileOptions(
-                  upsert: true,
-                  contentType: part.headers['content-type'],
-                ),
+                destinationBucket: 'csvhich_archive',
               );
 
           await processCsvData(tempFile, supabase);

lib/models/data.dart

@@ -116,6 +116,39 @@ final Map<String, String> scopes = {
   "ExportPGRGPNmois.txt": "date",
   "exportlicence.txt": "tlc",
 };
+final Map<String, List<String>> idToRemove = {
+  "secondprgtype.txt": ["day_of_origin"],
+  "exportPGRGPN.txt": ["date", "tlc"],
+  "ExportPGRGPNmois.txt": ["date", "tlc"],
+  "exportlicence.txt": ["tlc"],
+};
+final Map<String, Map<String, dynamic>> ids = {
+  "secondprgtype.txt": {
+    "table": "aclegs_log",
+    "headers": [
+      "day_of_origin",
+      "dep_sched_dt",
+      "fn_carrier",
+      "fn_number",
+      "dep_ap_sched",
+      "arr_ap_sched",
+      // "dep_ap_actual",
+      // "arr_ap_actual"
+    ]
+  },
+  "exportPGRGPN.txt": {
+    "table": "pnlegs_log",
+    "headers": ["tlc", "date", "dep", "des", "al", "fnum,", "label"]
+  },
+  "ExportPGRGPNmois.txt": {
+    "table": "pnlegs_log",
+    "headers": ["tlc", "date", "dep", "des", "al", "fnum,", "label"]
+  },
+  "exportlicence.txt": {
+    "table": "qualifs_log",
+    "headers": ["tlc", "college", "ac", "base"]
+  },
+};
 List<Map<String, String?>> createListOfMaps(
     List<String> headers, List<List<dynamic>> data) {
   // Initialize an empty list to hold the maps
@@ -1241,29 +1274,33 @@ const maxItemsToRemove = 500;
 processCsvData(File tempfile, SupabaseClient supabase) async {
   List<FilesAsData> csvData = [];
   final filename = tempfile.path.split('/').last;
-  final bytes = await tempfile.readAsBytes();
   // Check if file is a zip archive
-  String? csvcontent;
+
   if (filename.toLowerCase().endsWith('.zip')) {
-    final archive = ZipDecoder().decodeBytes(bytes);
+    Archive archive = ZipDecoder().decodeBytes(await tempfile.readAsBytes());
     for (final file in archive) {
       if (!file.isFile) continue;
 
-      csvcontent = utf8.decode(file.content as List<int>);
-      csvData.add(FilesAsData(filename: file.name, data: csvcontent));
+      csvData.add(FilesAsData(
+          filename: file.name, data: utf8.decode(file.content as List<int>)));
     }
+    archive.clearSync();
   } else {
     // For non-zip files, store in extracted folder
-    csvcontent = utf8.decode(bytes);
-    csvData.add(FilesAsData(filename: filename, data: csvcontent));
+
+    csvData.add(FilesAsData(
+        filename: filename, data: utf8.decode(await tempfile.readAsBytes())));
   }
 
-  for (final FilesAsData data in csvData) {
+  while (csvData.isNotEmpty) {
+    final data = csvData.first;
+    csvData.removeAt(0);
     print("    processing ${data.filename}");
     //inserting data
     if (tables.keys.contains(data.filename)) {
-      final mapsToInsert =
+      List<Map<String, String?>> mapsToInsert =
           createListOfMaps(headers[data.filename] ?? [], csv2list(data.data));
+      final completeMapsToInsert = mapsToInsert;
       final scopeName = scopes[data.filename!]!;
       final scopeInNew = mapsToInsert
           .fold(<String>{}, (t, e) => t..add(e[scopeName] ?? "")).toList();
@@ -1287,59 +1324,46 @@ processCsvData(File tempfile, SupabaseClient supabase) async {
 
       List<int> indexToRemove = [];
       List<int> indexToMaintain = [];
+
       // print("Deleting old data scope: ${mapsToInsert.length}");
+      final keys2check = idToRemove[data.filename]!;
+      final mawjoudin = oldComparable.fold(<Map>{},
+          (t, e) => {...t, extractMapWithKeys(e, keys2check)}).toList();
       for (int i = 0; i < oldComparable.length; i++) {
         final item = oldComparable[i];
         final index = findIndex(mapsToInsert, item);
-        if (index != -1) {
+
+        final mawjood =
+            mawjoudin.contains(extractMapWithKeys(item, keys2check));
+        if (index > -1) {
           indexToMaintain.add(i);
           mapsToInsert.removeAt(index);
+        } else if (!mawjood) {
+          if (!mawjood) {
+            // print("not deleted: ${extractMapWithKeys(item, keys2check)}");
+          }
         } else {
           indexToRemove.add(i);
         }
       }
 
-      // print(
-      //     " indexToRemove: ${indexToRemove.length} , indexToMaintain: ${indexToMaintain.length}");
-      // print("delete from db");
-
-      // print("delete from db ${indexToRemove.length}");
-
+//removing index to remove with id
       for (var e in splitList(indexToRemove.map((e) => old[e]['id']).toList(),
           headerToNb(indexToRemove.map((e) => old[e]['id']).toList()))) {
         await supabase
             .from(tables[data.filename]!) // Replace with your actual table name
             .delete()
             .inFilter('id', e);
-        // print("Deleted old data scope: ${e.length}");
       }
+      // print("Deleted old data scope: ${e.length}");
 
-      // splitList(indexToRemove.map((e) => old[e]['id']).toList(), 200).forEach(
-      //     (e) async => await supabase
-      //         .from(
-      //             tables[data.filename]!) // Replace with your actual table name
-      //         .delete()
-      //         .inFilter('id', e));
-      // print("insert in db");
-
+//insering new data
       await supabase
           .from(tables[data.filename]!) // Replace with your actual table name
           .insert(mapsToInsert);
 
       print(
           "         insert:${mapsToInsert.length} remove:${indexToRemove.length} maintain:${indexToMaintain.length}");
-      // for (var e in splitList(mapsToInsert, headerToNb(mapsToInsert))) {
-      //   await supabase
-      //       .from(tables[data.filename]!) // Replace with your actual table name
-      //       .insert(e);
-      //   print("Inserted old data scope: ${e.length}");
-      // }
-      // splitList(mapsToInsert, 200).forEach((e) async => await supabase
-      //     .from(tables[data.filename]!) // Replace with your actual table name
-      //     .insert(e));
-      // print("end");
-
-      // print(createListOfMaps(headers[data.filename] ?? [], csv2list(data.data)));
     } else {
       print("        filename: ${data.filename} unknown, not inserted.");
     }
@@ -1406,3 +1430,54 @@ int headerToNb(List list) {
   //print("header2nb: $res");
   return res;
 }
+
+Map<String, dynamic> findDifferences(
+    Map<String, dynamic> map1, Map<String, dynamic> map2) {
+  Map<String, dynamic> differences = {};
+
+  // Iterate through keys in map2
+  for (var key in map2.keys) {
+    // Check if key exists in map1
+    if (!map1.containsKey(key) || map1[key] != map2[key]) {
+      // Add to differences if it's different or doesn't exist in map1
+      differences[key] = map2[key];
+    }
+  }
+
+  return differences;
+}
+
+Map<String, dynamic> findSimilarities(
+    Map<String, dynamic> map1, Map<String, dynamic> map2) {
+  Map<String, dynamic> similarities = {};
+
+  // Iterate through keys in map1
+  for (var key in map1.keys) {
+    // Check if key exists in map2 and has the same value as map1
+    if (map2.containsKey(key) && map1[key] == map2[key]) {
+      // Add to similarities if it's the same in both maps
+      similarities[key] = map1[key];
+    }
+  }
+
+  return similarities;
+}
+
+bool containsAll(List<String> sourceList, List<String> targetList) {
+  for (var item in sourceList) {
+    if (!targetList.contains(item)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+Map<K, V> extractMapWithKeys<K, V>(Map<K, V> originalMap, List<K> keys) {
+  Map<K, V> extractedMap = {};
+  for (K key in keys) {
+    if (originalMap.containsKey(key)) {
+      extractedMap[key] = originalMap[key]!;
+    }
+  }
+  return extractedMap;
+}

pubspec.lock

@@ -26,10 +26,10 @@ packages:
     dependency: "direct main"
     description:
       name: archive
-      sha256: cb6a278ef2dbb298455e1a713bda08524a175630ec643a242c399c932a0a1f7d
+      sha256: "08064924cbf0ab88280a0c3f60db9dd24fec693927e725ecb176f16c629d1cb8"
       url: "https://pub.dev"
     source: hosted
-    version: "3.6.1"
+    version: "4.0.1"
   args:
     dependency: transitive
     description:
@@ -82,10 +82,10 @@ packages:
     dependency: transitive
     description:
       name: coverage
-      sha256: "4b03e11f6d5b8f6e5bb5e9f7889a56fe6c5cbe942da5378ea4d4d7f73ef9dfe5"
+      sha256: e3493833ea012784c740e341952298f1cc77f1f01b1bbc3eb4eecf6984fb7f43
       url: "https://pub.dev"
     source: hosted
-    version: "1.11.0"
+    version: "1.11.1"
   crypto:
     dependency: transitive
     description:
@@ -94,6 +94,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "3.0.6"
+  ffi:
+    dependency: transitive
+    description:
+      name: ffi
+      sha256: "16ed7b077ef01ad6170a3d0c57caa4a112a38d7a2ed5602e0aca9ca6f3d98da6"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.1.3"
   file:
     dependency: transitive
     description:
@@ -170,26 +178,26 @@ packages:
     dependency: transitive
     description:
       name: intl
-      sha256: d6f56758b7d3014a48af9701c085700aac781a92a87a62b1333b46d8879661cf
+      sha256: "00f33b908655e606b86d2ade4710a231b802eec6f11e87e4ea3783fd72077a50"
       url: "https://pub.dev"
     source: hosted
-    version: "0.19.0"
+    version: "0.20.1"
   io:
     dependency: transitive
     description:
       name: io
-      sha256: "2ec25704aba361659e10e3e5f5d672068d332fc8ac516421d483a11e5cbd061e"
+      sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b
       url: "https://pub.dev"
     source: hosted
-    version: "1.0.4"
+    version: "1.0.5"
   jiffy:
     dependency: "direct main"
     description:
       name: jiffy
-      sha256: "3497caaa36d36a29033e66803c9739ce6bccbc7e241ca46070f76ee9e6f6eb0c"
+      sha256: "1c1b86459969ff9f32dc5b0ffe392f1e08181e66396cf9dd8fa7c90552a691af"
       url: "https://pub.dev"
     source: hosted
-    version: "6.3.1"
+    version: "6.3.2"
   js:
     dependency: transitive
     description:
@@ -266,10 +274,10 @@ packages:
     dependency: transitive
     description:
       name: package_config
-      sha256: "1c5b77ccc91e4823a5af61ee74e6b972db1ef98c2ff5a18d3161c982a55448bd"
+      sha256: "92d4488434b520a62570293fbd33bb556c7d49230791c1b4bbd973baf6d2dc67"
       url: "https://pub.dev"
     source: hosted
-    version: "2.1.0"
+    version: "2.1.1"
   path:
     dependency: transitive
     description:
@@ -286,6 +294,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.5.1"
+  posix:
+    dependency: transitive
+    description:
+      name: posix
+      sha256: a0117dc2167805aa9125b82eee515cc891819bac2f538c83646d355b16f58b9a
+      url: "https://pub.dev"
+    source: hosted
+    version: "6.0.1"
   postgrest:
     dependency: transitive
     description:
@@ -298,10 +314,10 @@ packages:
     dependency: transitive
     description:
       name: pub_semver
-      sha256: "40d3ab1bbd474c4c2328c91e3a7df8c6dd629b79ece4c4bd04bee496a224fb0c"
+      sha256: "7b3cfbf654f3edd0c6298ecd5be782ce997ddf0e00531b9464b55245185bbbbd"
       url: "https://pub.dev"
     source: hosted
-    version: "2.1.4"
+    version: "2.1.5"
   realtime_client:
     dependency: transitive
     description:
@@ -386,18 +402,18 @@ packages:
     dependency: transitive
     description:
       name: source_maps
-      sha256: "708b3f6b97248e5781f493b765c3337db11c5d2c81c3094f10904bfa8004c703"
+      sha256: "190222579a448b03896e0ca6eca5998fa810fda630c1d65e2f78b3f638f54812"
       url: "https://pub.dev"
     source: hosted
-    version: "0.10.12"
+    version: "0.10.13"
   source_span:
     dependency: transitive
     description:
       name: source_span
-      sha256: "53e943d4206a5e30df338fd4c6e7a077e02254531b138a15aec3bd143c1a8b3c"
+      sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c"
       url: "https://pub.dev"
     source: hosted
-    version: "1.10.0"
+    version: "1.10.1"
   stack_trace:
     dependency: transitive
     description:
@@ -450,10 +466,10 @@ packages:
     dependency: "direct dev"
     description:
       name: test
-      sha256: be7697dcfa9d47333e6ceb38400b4c8b8de15dc97023e8eb219189ba19c5d423
+      sha256: "22eb7769bee38c7e032d532e8daa2e1cc901b799f603550a4db8f3a5f5173ea2"
       url: "https://pub.dev"
     source: hosted
-    version: "1.25.11"
+    version: "1.25.12"
   test_api:
     dependency: transitive
     description:
@@ -466,10 +482,10 @@ packages:
     dependency: transitive
     description:
       name: test_core
-      sha256: "3e47cac78f28a4dd71ea232db15fda6345934f472c2f70f97d35476809e045ca"
+      sha256: "84d17c3486c8dfdbe5e12a50c8ae176d15e2a771b96909a9442b40173649ccaa"
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.7"
+    version: "0.6.8"
   typed_data:
     dependency: transitive
     description:
@@ -482,10 +498,10 @@ packages:
     dependency: transitive
     description:
       name: vm_service
-      sha256: "0968250880a6c5fe7edc067ed0a13d4bae1577fe2771dcf3010d52c4a9d3ca14"
+      sha256: ddfa8d30d89985b96407efce8acbdd124701f96741f2d981ca860662f1c0dc02
       url: "https://pub.dev"
     source: hosted
-    version: "14.3.1"
+    version: "15.0.0"
   watcher:
     dependency: transitive
     description: