backup hp

lib/handlers/file_upload_api.dart

@@ -175,7 +175,7 @@ class FileUploadApi extends BaseApi {
 
         for (var file in files) {
           final fileProcess = FileProcess(file, supabaseClient);
-          await fileProcess.go(donttouchdb: false);
+          await fileProcess.go(donttouchdb: true);
         }
       }
 
@@ -315,8 +315,8 @@ class FileProcess {
     "ExportPGRGPNmois.txt": ["date", "tlc"],
     "exportlicence.txt": ["tlc"],
   };
-  final Map<String, Map<String, dynamic>> ids = {
-    "secondprgtype.txt": {
+  final Map<String, List<Map<String, dynamic>>> trackers = {
+/*    "secondprgtype.txt": {
       "table": "aclegs_log",
       "headers": [
         "day_of_origin",
@@ -329,18 +329,33 @@ class FileProcess {
         // "arr_ap_actual"
       ]
     },
-    "exportPGRGPN.txt": {
-      "table": "pnlegs_log",
-      "headers": ["tlc", "date", "dep", "des", "al", "fnum,", "label"]
-    },
-    "ExportPGRGPNmois.txt": {
+    */
+    "exportPGRGPN.txt": [
+      {
+        "table": "pnlegs_log_roster",
+        "groupby": ["date", "tlc"],
+        "track": ["dep", "des", "al", "fnum", "label"]
+      },
+      // {
+      //   "table": "pnlegs_log_duty",
+      //   "groupby": ["date", "dep", "des", "al", "fnum", "label"],
+      //   "track": ["tlc"]
+      // },
+      // {
+      //   "table": "pnlegs_log_sched",
+      //   "groupby": ["date", "dep", "des", "al", "fnum", "label"],
+      //   "changes": ["hdep", "hdes"]
+      // },
+    ],
+/*    "ExportPGRGPNmois.txt": {
       "table": "pnlegs_log",
-      "headers": ["tlc", "date", "dep", "des", "al", "fnum,", "label"]
+      "headers": ["tlc", "date", "dep", "des", "al", "fnum", "label"]
     },
     "exportlicence.txt": {
       "table": "qualifs_log",
       "headers": ["tlc", "college", "ac", "base"]
     },
+ */
   };
 
   Future<List<Map<String, dynamic>>> parseCsv() async {
@@ -430,6 +445,37 @@ class FileProcess {
       }
       print(
           "   Scope:$scopeInNew      insert:${dataToInsert.length} remove:${indexToRemove.length} maintain:${indexToMaintain.length}");
+
+      for (var tracker in trackers[filename] ?? []) {
+        final table = tracker["table"];
+        final groupby = tracker["groupby"] ?? [];
+        final track = tracker["track"] ?? [];
+        final stateOld = oldComparable.groupBy(
+            (e) => groupby.map((f) => e[f]).join("|"),
+            dataFunction: (e) => e
+                .filterKeys(track)
+                .values
+                .map((j) => j == null ? "" : j)
+                .join("_"));
+        final stateNew = dataToInsert.groupBy(
+            (e) => groupby.map((f) => e[f]).join("|"),
+            dataFunction: (e) => e
+                .filterKeys(track)
+                .values
+                .map((j) => j == null ? "" : j)
+                .join("_"));
+        List logs = [];
+        for (var key
+            in (stateOld.keys.toList()..addAll(stateNew.keys)).toSet()) {
+          final (add, remove) = (stateNew[key] ?? []).diff(stateOld[key] ?? []);
+          if (add.isNotEmpty || remove.isNotEmpty) {
+            logs.add("$key:\n     +$add}\n     -$remove\n");
+          }
+        }
+
+        print("   Tracker:$table");
+        print("     $logs");
+      }
     }
   }
 
@@ -476,6 +522,73 @@ class FileProcess {
   }
 }
 
+extension IterableDiff<T> on Iterable<T> {
+  (Iterable<T> add, Iterable<T> remove) diff(Iterable<T> listB) {
+    // Convert listA to a list for easier indexing
+    final listA = this;
+
+    // Items to add are those in listB but not in listA
+    final add = listB.where((item) => !listA.contains(item));
+
+    // Items to remove are those in listA but not in listB
+    final remove = listA.where((item) => !listB.contains(item));
+
+    return (add, remove);
+  }
+}
+
+extension CompareIterables<T> on Iterable<T> {
+  /// Compares this iterable with another iterable and returns a map containing:
+  /// - 'added': Items that are in the other iterable but not in this one.
+  /// - 'removed': Items that are in this iterable but not in the other one.
+  (Iterable<T> add, Iterable<T> remove) compareWith(Iterable<T> other) {
+    final Set<T> thisSet = this.toSet();
+    final Set<T> otherSet = other.toSet();
+
+    final Set<T> added = otherSet.difference(thisSet);
+    final Set<T> removed = thisSet.difference(otherSet);
+
+    return (added, removed);
+  }
+}
+
+extension FilterMapByKeys on Map {
+  /// Returns a new map containing only the keys (and their associated values)
+  /// that are present in the [keysToKeep] list.
+  Map<K, V> filterKeys<K, V>(List<K> keysToKeep) {
+    return Map<K, V>.fromEntries(
+      entries
+          .where((entry) => keysToKeep.contains(entry.key))
+          .cast<MapEntry<K, V>>(),
+    );
+  }
+}
+
+extension RemoveNull<T> on Iterable<T?> {
+  /// Returns a new iterable with all null values removed.
+  Iterable<T> removeNull() {
+    return where((element) => element != null).cast<T>();
+  }
+}
+
+extension GroupBy<T> on Iterable<T> {
+  Map<K, List> groupBy<K>(K Function(T) keyFunction,
+      {Function(T)? dataFunction, bool Function(T)? keyIsNullFunction}) {
+    final map = <K, List>{};
+    for (final element in this) {
+      final key = keyFunction(element);
+      final keyIsNull =
+          keyIsNullFunction == null ? false : keyIsNullFunction(element);
+      if (keyIsNull || key == null) continue;
+      if (dataFunction != null) {
+        map.putIfAbsent(key, () => []).add(dataFunction(element));
+      } else
+        map.putIfAbsent(key, () => []).add(element);
+    }
+    return map;
+  }
+}
+
 extension NullIfEmpty on String {
   String? get nullIfEmpty => isEmpty ? null : this;
 }