compare delete chunk 100 insert OK

lib/handlers/file_upload_api.dart

@@ -1,4 +1,3 @@
-import 'dart:collection';
 import 'dart:io';
 import 'package:file_upload_processor/handlers/base_api.dart';
 import 'package:intl/intl.dart';
@@ -61,9 +60,9 @@ class FileUploadApi extends BaseApi {
             ),
           );
 
-      print('File uploaded to Supabase: $timestampedFilename');
+      print('+File uploaded to <$bucket>: $timestampedFilename');
     } catch (e) {
-      print('Error uploading to Supabase: $e');
+      print('!Error uploading to Supabase: $e');
       rethrow;
     }
   }
@@ -163,12 +162,20 @@ class FileUploadApi extends BaseApi {
           files.add(dataFilePath);
         }
         bytes.clear();
+        //upload to supabase storage
+        await uploadToSupabase(rawFilePath, filename, supabaseClient,
+            bucket: 'csvhich', timestamped: false, upsert: true);
+        //upload to supabase storage archive timestamped
         await uploadToSupabase(rawFilePath, filename, supabaseClient,
             bucket: 'csvhich_archive', timestamped: true, upsert: false);
+        //insert data to supabase csvhichupdates
+        await supabaseClient
+            .from('csvhichupdates')
+            .insert({'filename': filename});
 
         for (var file in files) {
           final fileProcess = FileProcess(file, supabaseClient);
-          await fileProcess.go();
+          await fileProcess.go(donttouchdb: false);
         }
       }
 
@@ -363,7 +370,7 @@ class FileProcess {
 
       // Assign each value to the corresponding header
       for (int j = 0; j < headers.length; j++) {
-        row[headers[j]] = values[j].removeQuotes.trim().nullIfEmpty;
+        row[headers[j]] = values[j].trim().removeQuotes.trim().nullIfEmpty;
       }
 
       // Add the row map to the data list
@@ -375,7 +382,7 @@ class FileProcess {
   }
 
   List<String> get filesTomonitor => _headers.keys.toList();
-  Future<void> go() async {
+  Future<void> go({bool donttouchdb = false}) async {
     if (!filesTomonitor.contains(filename)) return;
     final mapsToInsert = await parseCsv();
     final scopeName = scopes[filename] ?? "";
@@ -386,12 +393,12 @@ class FileProcess {
     List<Map<String, dynamic>> oldComparable = [];
 
     //load old data
-    for (var e in splitList(scopeInNew, headerToNb(scopeInNew))) {
+    for (var e in chunkList(scopeInNew, 30)) {
       final res = await supabase
           .from(tables[filename]!)
           .select()
           .inFilter(scopeName, e)
-          .limit(100000);
+          .limit(300000);
 
       oldIds.addAll(res.map((e) => {"id": e["id"]}));
       oldComparable.addAll(res.map((e) => e..remove("id")));
@@ -400,89 +407,70 @@ class FileProcess {
     final comparisonResult = compareLists(oldComparable, mapsToInsert);
     final indexToRemove = comparisonResult.removeIndices;
     final indexToMaintain = comparisonResult.maintainIndices;
-    final indexToInsert = comparisonResult.insertIndices;
+    final dataToInsert = comparisonResult.insertData;
 
     try {
-      // removing index to remove with id
-      for (var e in splitList(
-          indexToRemove.map((e) => oldIds[e]['id']).toList(),
-          headerToNb(indexToRemove.map((e) => oldIds[e]['id']).toList()))) {
-        await supabase
-            .from(tables[filename]!) // Replace with your actual table name
-            .delete()
-            .inFilter('id', e);
-      }
+      if (!donttouchdb)
+        for (var e in chunkList(
+            indexToRemove.map((f) => oldIds[f]['id']).toList(), 100)) {
+          await supabase
+              .from(tables[filename]!) // Replace with your actual table name
+              .delete()
+              .inFilter('id', e);
+        }
 
       // insering new data
-      await supabase
-          .from(tables[filename]!) // Replace with your actual table name
-          .insert(indexToInsert.map((e) => mapsToInsert.elementAt(e)).toList());
+      if (!donttouchdb)
+        await supabase
+            .from(tables[filename]!) // Replace with your actual table name
+            .insert(dataToInsert);
     } catch (e, stackTrace) {
       print('Error: $e\n$stackTrace');
     }
     print(
-        "         insert:${indexToInsert.length} remove:${indexToRemove.length} maintain:${indexToMaintain.length}");
+        "         insert:${dataToInsert.length} remove:${indexToRemove.length} maintain:${indexToMaintain.length}");
   }
 
   ({
     List<int> maintainIndices,
     List<int> removeIndices,
-    List<int> insertIndices
+    // List<int> insertIndices
+    List<Map> insertData
   }) compareLists(
       List<Map<String, dynamic>> map1, List<Map<String, dynamic>> map2) {
     List<int> maintainIndices = [];
     List<int> removeIndices = [];
-    List<int> insertIndices = [];
+    List<Map<String, dynamic>> insertData = map2;
 
     // Find indices to maintain and remove in map1
     for (int i = 0; i < map1.length; i++) {
-      if (map2.contains(map1[i])) {
+      final pos = insertData.findMap(map1[i]);
+      if (pos > -1) {
         maintainIndices.add(i); // Item exists in both lists
+        insertData.removeAt(pos);
       } else {
         removeIndices.add(i); // Item does not exist in map2
       }
     }
 
-    // Find indices to insert from map2
-    for (int i = 0; i < map2.length; i++) {
-      if (!map1.contains(map2[i])) {
-        insertIndices.add(i); // Item does not exist in map1
-      }
-    }
-
     return (
       maintainIndices: maintainIndices,
       removeIndices: removeIndices,
-      insertIndices: insertIndices
+      insertData: insertData
     );
   }
 
-  List<List<T>> splitList<T>(List<T> originalList, int maxSize) {
-    List<List<T>> sublists = [];
-
-    for (int i = 0; i < originalList.length; i += maxSize) {
-      // Create a sublist for the current chunk
-      List<T> sublist = originalList.sublist(
-        i,
-        (i + maxSize > originalList.length) ? originalList.length : i + maxSize,
-      );
-      sublists.add(sublist);
+  List<List<T>> chunkList<T>(List<T> list, int chunkSize) {
+    if (chunkSize <= 0) {
+      throw ArgumentError('chunkSize must be greater than 0');
     }
 
-    return sublists;
-  }
-
-  int headerToNb(List list) {
-    //header max of 16k
-    final maxheader = 4 * 1024 * 8;
-    if (list.isEmpty) {
-      return list.length;
+    List<List<T>> chunks = [];
+    for (var i = 0; i < list.length; i += chunkSize) {
+      chunks.add(list.sublist(
+          i, i + chunkSize > list.length ? list.length : i + chunkSize));
     }
-    final length1 = (list.toString().length / list.length).ceil() * 8;
-    final lengthurl = 200 * 8;
-    final res = ((maxheader - lengthurl) / length1).floor();
-    //print("header2nb: $res");
-    return res;
+    return chunks;
   }
 }
 
@@ -494,16 +482,39 @@ extension RemoveQuotes on String {
   String get removeQuotes {
     if (isEmpty) return this; // Return if the string is empty
 
-    // Check if the first character is a quote
-    bool startsWithQuote = startsWith('"') || startsWith("'");
-    // Check if the last character is a quote
-    bool endsWithQuote = endsWith('"') || endsWith("'");
-
     // Remove the first and last characters if they are quotes
     String result = this;
+    // Check if the first character is a quote
+    bool startsWithQuote = result.startsWith('"') || result.startsWith("'");
     if (startsWithQuote) result = result.substring(1);
+    // Check if the last character is a quote
+    bool endsWithQuote = result.endsWith('"') || result.endsWith("'");
     if (endsWithQuote) result = result.substring(0, result.length - 1);
 
     return result;
   }
 }
+
+bool mapsAreEqual(Map<String, dynamic> map1, Map<String, dynamic> map2) {
+  if (map1.length != map2.length) return false;
+  for (var key in map1.keys) {
+    if (map1[key] != map2[key]) return false;
+  }
+  return true;
+}
+
+extension ContainsMap on List<Map<String, dynamic>> {
+  bool containsMap(Map<String, dynamic> map) {
+    for (var item in this) {
+      if (mapsAreEqual(item, map)) return true;
+    }
+    return false;
+  }
+
+  int findMap(Map<String, dynamic> map) {
+    for (int i = 0; i < this.length; i++) {
+      if (mapsAreEqual(this.elementAt(i), map)) return i;
+    }
+    return -1;
+  }
+}

pubspec.lock

@@ -85,10 +85,10 @@ packages:
     dependency: "direct main"
     description:
       name: http_parser
-      sha256: "76d306a1c3afb33fe82e2bbacad62a61f409b5634c915fceb0d799de1a913360"
+      sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571"
       url: "https://pub.dev"
     source: hosted
-    version: "4.1.1"
+    version: "4.1.2"
   intl:
     dependency: "direct main"
     description:
@@ -157,10 +157,10 @@ packages:
     dependency: transitive
     description:
       name: postgrest
-      sha256: "9f759ac497a24839addbed69d9569ea6d51d2e4834c672b8c2a73752fb6945c8"
+      sha256: b74dc0f57b5dca5ce9f57a54b08110bf41d6fc8a0483c0fec10c79e9aa0fb2bb
       url: "https://pub.dev"
     source: hosted
-    version: "2.4.0"
+    version: "2.4.1"
   realtime_client:
     dependency: transitive
     description:
@@ -229,10 +229,10 @@ packages:
     dependency: transitive
     description:
       name: stream_channel
-      sha256: "4ac0537115a24d772c408a2520ecd0abb99bca2ea9c4e634ccbdbfae64fe17ec"
+      sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d"
       url: "https://pub.dev"
     source: hosted
-    version: "2.1.3"
+    version: "2.1.4"
   string_scanner:
     dependency: transitive
     description:
@@ -245,10 +245,10 @@ packages:
     dependency: "direct main"
     description:
       name: supabase
-      sha256: ea3daaf4fc76df9bf42ca00142f8d07b94400943a93d563e87f5575ea78f1c2c
+      sha256: "270f63cd87a16578fee87e40cbf61062e8cdbce68d5e723e665f4651d70ddd8c"
       url: "https://pub.dev"
     source: hosted
-    version: "2.6.1"
+    version: "2.6.2"
   term_glyph:
     dependency: transitive
     description:

pubspec.yaml

@@ -14,4 +14,4 @@ dependencies:
   mime: #^1.0.5
   supabase: #^2.0.8
   intl: #^0.19.0
-  jiffy: ^6.3.2
+  jiffy: #^6.3.2