all right

.vscode/sftp.json

@@ -0,0 +1,12 @@
+{
+    "name": "debian",
+    "host": "baas.fares.cyou",
+    "protocol": "sftp",
+    "port": 22,
+    "username": "debian",
+    "password":"Farrous08",
+    "remotePath": "/home/debian",
+    "uploadOnSave": false,
+    "useTempFile": false,
+    "openSsh": false
+}

bin/main.dart

@@ -19,7 +19,7 @@ void main() async {
   final handler = const Pipeline()
       .addMiddleware(logRequests())
       // .addMiddleware(checkAuthorization())
-      .addHandler(router.call);
+      .addHandler(router);
 
   // Start the server
   final server = await shelf_io.serve(handler, '0.0.0.0', 8080);

lib/handlers/upload_handler.dart

@@ -12,17 +12,17 @@ Future<Response> handleFileUpload(Request request) async {
   }
   final token = authHeader.substring(7); // Remove 'Bearer ' prefix
 
+  final contentType = request.headers['content-type'];
+  if (contentType == null || !contentType.contains('multipart/form-data')) {
+    return Response(400, body: 'Unsupported content-type');
+  }
+
   // Initialize Supabase client with the bearer token
   final supabase = SupabaseClient(
     'http://baas.fares.cyou:8000',
     token,
   );
 
-  final contentType = request.headers['content-type'];
-  if (contentType == null || !contentType.contains('multipart/form-data')) {
-    return Response(400, body: 'Unsupported content-type');
-  }
-
   if (request.multipart() case var multipart?) {
     List<Map<String, dynamic>> uploadedFiles = [];
     await for (final part in multipart.parts) {
@@ -54,6 +54,8 @@ Future<Response> handleFileUpload(Request request) async {
                 );
 
             //insertline in csvhichupdates
+            print("inserting incsvhichupdates");
+
             await supabase.from('csvhichupdates').insert({
               'filename': filename,
               'updated_at': DateTime.now().toUtc().toIso8601String(),
@@ -92,7 +94,7 @@ Future<Response> handleFileUpload(Request request) async {
                 "updated_at": now,
               },
             );
-            print("fileupload: realtime res: $res");
+            print("realtime response: $res");
 
             //add filename ta list
             uploadedFiles.add({
@@ -103,6 +105,8 @@ Future<Response> handleFileUpload(Request request) async {
             // Clean up temporary file
             await tempFile.delete();
           } catch (e) {
+            supabase.dispose();
+
             print("Error processing file:\n $e");
             return Response.internalServerError(body: e.toString());
           }
@@ -110,6 +114,7 @@ Future<Response> handleFileUpload(Request request) async {
       }
     }
 
+    supabase.dispose();
     if (uploadedFiles.isNotEmpty) {
       return Response.ok(
         '{"status": "success", "files": ${uploadedFiles.toString()}}',
@@ -119,6 +124,7 @@ Future<Response> handleFileUpload(Request request) async {
 
     return Response(400, body: 'No files were uploaded');
   } else {
+    supabase.dispose();
     return Response(401, body: 'Not a multipart request');
   }
 }

lib/models/data.dart

@@ -1234,6 +1234,10 @@ class FilesAsData {
   });
 }
 
+const maxItemsToSelect = 200;
+const maxItemsToInsert = 500;
+const maxItemsToRemove = 500;
+
 processCsvData(File tempfile, SupabaseClient supabase) async {
   List<FilesAsData> csvData = [];
   final filename = tempfile.path.split('/').last;
@@ -1263,11 +1267,18 @@ processCsvData(File tempfile, SupabaseClient supabase) async {
       final scopeInNew = mapsToInsert
           .fold(<String>{}, (t, e) => t..add(e[scopeName] ?? "")).toList();
 
-      final old = await supabase
-          .from(tables[data.filename]!)
-          .select()
-          .inFilter(scopeName, scopeInNew)
-          .limit(100000);
+      print("Loading old data scope: ${scopeInNew.length}");
+      List<Map<String, dynamic>> old = [];
+      for (var e in splitList(scopeInNew, headerToNb(scopeInNew))) {
+        final res = await supabase
+            .from(tables[data.filename]!)
+            .select()
+            .inFilter(scopeName, e)
+            .limit(100000);
+        old.addAll(res);
+        print("Loaded old data scope: ${e.length}");
+      }
+
       // Replace with your actual table name
       final oldComparable = old
           .map((e) => filterMapByKeys(e, headers[data.filename] ?? []))
@@ -1275,6 +1286,7 @@ processCsvData(File tempfile, SupabaseClient supabase) async {
 
       List<int> indexToRemove = [];
       List<int> indexToMaintain = [];
+      print("Deleting old data scope: ${mapsToInsert.length}");
       for (int i = 0; i < oldComparable.length; i++) {
         final item = oldComparable[i];
         final index = findIndex(mapsToInsert, item);
@@ -1290,17 +1302,38 @@ processCsvData(File tempfile, SupabaseClient supabase) async {
           " indexToRemove: ${indexToRemove.length} , indexToMaintain: ${indexToMaintain.length}");
       print("delete from db");
 
-      splitList(indexToRemove.map((e) => old[e]['id']).toList(), 50).forEach(
-          (e) async => await supabase
-              .from(
-                  tables[data.filename]!) // Replace with your actual table name
-              .delete()
-              .inFilter('id', e));
+      print("delete from db ${indexToRemove.length}");
+
+      for (var e in splitList(indexToRemove.map((e) => old[e]['id']).toList(),
+          headerToNb(indexToRemove.map((e) => old[e]['id']).toList()))) {
+        await supabase
+            .from(tables[data.filename]!) // Replace with your actual table name
+            .delete()
+            .inFilter('id', e);
+        print("Deleted old data scope: ${e.length}");
+      }
+
+      // splitList(indexToRemove.map((e) => old[e]['id']).toList(), 200).forEach(
+      //     (e) async => await supabase
+      //         .from(
+      //             tables[data.filename]!) // Replace with your actual table name
+      //         .delete()
+      //         .inFilter('id', e));
       print("insert in db");
 
-      splitList(mapsToInsert, 50).forEach((e) async => await supabase
+      await supabase
           .from(tables[data.filename]!) // Replace with your actual table name
-          .insert(e));
+          .insert(mapsToInsert);
+      // print("insert in db ${mapsToInsert.length}");
+      // for (var e in splitList(mapsToInsert, headerToNb(mapsToInsert))) {
+      //   await supabase
+      //       .from(tables[data.filename]!) // Replace with your actual table name
+      //       .insert(e);
+      //   print("Inserted old data scope: ${e.length}");
+      // }
+      // splitList(mapsToInsert, 200).forEach((e) async => await supabase
+      //     .from(tables[data.filename]!) // Replace with your actual table name
+      //     .insert(e));
       print("end");
 
       // print(createListOfMaps(headers[data.filename] ?? [], csv2list(data.data)));
@@ -1357,3 +1390,16 @@ List<List<T>> splitList<T>(List<T> originalList, int maxSize) {
 
   return sublists;
 }
+
+int headerToNb(List list) {
+  //header max of 16k
+  final maxheader = 4 * 1024 * 8;
+  if (list.isEmpty) {
+    return list.length;
+  }
+  final length1 = (list.toString().length / list.length).ceil() * 8;
+  final lengthurl = 200 * 8;
+  final res = ((maxheader - lengthurl) / length1).floor();
+  print("header2nb: $res");
+  return res;
+}

shelf/Dockerfile

@@ -0,0 +1,22 @@
+# Use latest stable channel SDK.
+FROM dart:stable AS build
+
+# Resolve app dependencies.
+WORKDIR /app
+COPY pubspec.* ./
+RUN dart pub get
+
+# Copy app source code (except anything in .dockerignore) and AOT compile app.
+COPY . .
+RUN dart pub get --offline
+RUN dart compile exe bin/main.dart -o bin/main
+
+# Build minimal serving image from AOT-compiled `/main`
+# and the pre-built AOT-runtime in the `/runtime/` directory of the base image.
+FROM scratch
+COPY --from=build /runtime/ /
+COPY --from=build /app/bin/main /app/bin/
+
+# Start server.
+EXPOSE 8080
+CMD ["/app/bin/main"]

shelf/docker-compose.yml

@@ -0,0 +1,31 @@
+version: '3.8'
+
+# services:
+#   dart_server:
+#     image: google/dart
+#     volumes:
+#       - ./app:/app
+#     working_dir: /app
+#     command: dart run bin/server.dart
+#     ports:
+#       - "8082:8080"
+
+services:
+  dart-server:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ./app:/app
+    ports:
+      - "8082:8080"
+    environment:
+      # Add any environment variables needed for your application here
+      # Example: MY_ENV_VAR: "value"
+    restart: unless-stopped
+
+
+networks:
+  default:
+    name: caddy-network
+    external: true