accepept upload, unzip, upload to supabase

.gitignore

@@ -0,0 +1,3 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/

CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.0
+
+- Initial version.

Dockerfile

@@ -0,0 +1,38 @@
+FROM dart:3.3.0 AS build
+
+# Set working directory
+WORKDIR /app
+
+# Copy pubspec files
+COPY pubspec.* ./
+RUN dart pub get
+
+# Copy source code
+COPY . .
+
+# Compile to native code using AOT
+RUN dart compile exe bin/server.dart -o server
+
+# Create runtime image
+FROM debian:bookworm-slim
+
+# Install SSL certificates for HTTPS requests
+RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /app
+
+# Create upload directories
+RUN mkdir -p /app/uploaded/raw /app/uploaded/data
+
+# Copy compiled binary from build stage
+COPY --from=build /app/server /app/server
+
+# Set execute permissions
+RUN chmod +x /app/server
+
+# Expose port
+EXPOSE 8080
+
+# Start server
+CMD ["/app/server"]

analysis_options.yaml

@@ -0,0 +1,30 @@
+# This file configures the static analysis results for your project (errors,
+# warnings, and lints).
+#
+# This enables the 'recommended' set of lints from `package:lints`.
+# This set helps identify many issues that may lead to problems when running
+# or consuming Dart code, and enforces writing Dart using a single, idiomatic
+# style and format.
+#
+# If you want a smaller set of lints you can change this to specify
+# 'package:lints/core.yaml'. These are just the most critical lints
+# (the recommended set includes the core lints).
+# The core lints are also what is used by pub.dev for scoring packages.
+
+#include: package:lints/recommended.yaml
+
+# Uncomment the following section to specify additional rules.
+
+# linter:
+#   rules:
+#     - camel_case_types
+
+# analyzer:
+#   exclude:
+#     - path/to/excluded/files/**
+
+# For more information about the core and recommended set of lints, see
+# https://dart.dev/go/core-lints
+
+# For additional information about configuring this file, see
+# https://dart.dev/guides/language/analysis-options

bin/server.dart

@@ -0,0 +1,15 @@
+import 'dart:io';
+
+import 'package:file_upload_processor/file_upload_api.dart';
+import 'package:shelf/shelf.dart' as shelf;
+import 'package:shelf/shelf_io.dart' as shelf_io;
+
+void main() async {
+  final api = FileUploadApi();
+  final handler = const shelf.Pipeline()
+      .addMiddleware(shelf.logRequests())
+      .addHandler(api.router);
+
+  final server = await shelf_io.serve(handler, InternetAddress.anyIPv4, 8080);
+  print('Server running on port ${server.port}');
+}

docker-compose.yml

@@ -0,0 +1,26 @@
+services:
+  tp5-shelf:
+    container_name: tp5-shelf
+    build:
+      context: .
+      dockerfile: Dockerfile
+    #ports:
+    #  - "8080:8080"
+    volumes:
+      - uploaded_data:/app/uploaded
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
+
+volumes:
+  uploaded_data:
+    name: file_upload_uploaded_data
+
+networks:
+  default:
+    name: caddy-network
+    external: true

lib/file_upload_api.dart

@@ -0,0 +1,172 @@
+import 'dart:io';
+import 'package:intl/intl.dart';
+import 'package:mime/mime.dart';
+import 'package:shelf/shelf.dart' as shelf;
+//import 'package:shelf/shelf_io.dart' as shelf_io;
+import 'package:shelf_router/shelf_router.dart';
+import 'package:path/path.dart' as path;
+import 'package:http_parser/http_parser.dart';
+import 'package:archive/archive.dart';
+import 'package:supabase/supabase.dart';
+
+class FileUploadApi {
+  SupabaseClient getSupabaseClient(shelf.Request request) {
+    final supabaseUrl = request.headers['supabase-url'];
+    if (supabaseUrl == null) {
+      throw Exception('Supabase URL not provided in headers');
+    }
+
+    final authHeader = request.headers['authorization'];
+    if (authHeader == null || !authHeader.startsWith('Bearer ')) {
+      throw Exception('Invalid or missing Authorization Bearer token');
+    }
+
+    final token = authHeader.substring(7); // Remove 'Bearer ' prefix
+
+    return SupabaseClient(
+      supabaseUrl,
+      token,
+    );
+  }
+
+  String getTimestampedFilename(String originalFilename) {
+    final timestamp = DateFormat('yyyyMMdd_HHmmss').format(DateTime.now());
+    return '${timestamp}_$originalFilename';
+  }
+
+  Future<void> uploadToSupabase(
+      String filePath, String filename, SupabaseClient supabaseClient,
+      {bool timestamped = false,
+      required String bucket,
+      bool upsert = true}) async {
+    try {
+      final file = File(filePath);
+      final timestampedFilename =
+          timestamped ? getTimestampedFilename(filename) : filename;
+
+      await supabaseClient.storage.from(bucket).upload(
+            timestampedFilename,
+            file,
+            fileOptions: FileOptions(
+              cacheControl: '3600',
+              upsert: upsert,
+            ),
+          );
+
+      print('File uploaded to Supabase: $timestampedFilename');
+    } catch (e) {
+      print('Error uploading to Supabase: $e');
+      rethrow;
+    }
+  }
+
+  Future<void> initializeDirectories() async {
+    final directories = [
+      Directory('./uploaded/raw'),
+      Directory('./uploaded/data'),
+    ];
+
+    for (var dir in directories) {
+      if (!await dir.exists()) {
+        await dir.create(recursive: true);
+      }
+    }
+  }
+
+  bool isZipFile(List<int> bytes) {
+    if (bytes.length < 4) return false;
+    return bytes[0] == 0x50 &&
+        bytes[1] == 0x4B &&
+        bytes[2] == 0x03 &&
+        bytes[3] == 0x04;
+  }
+
+  Future<void> processZipFile(String filePath) async {
+    final bytes = await File(filePath).readAsBytes();
+    final archive = ZipDecoder().decodeBytes(bytes);
+
+    for (final file in archive) {
+      final filename = file.name;
+      if (file.isFile) {
+        final data = file.content as List<int>;
+        final outFile = File(path.join('./uploaded/data', filename));
+        await outFile.parent.create(recursive: true);
+        await outFile.writeAsBytes(data);
+      }
+    }
+  }
+
+  Router get router {
+    final router = Router();
+
+    router.get('/', (shelf.Request request) {
+      return shelf.Response.ok('Hello World');
+    });
+
+    router.post('/upload', (shelf.Request request) async {
+      final supabaseClient = getSupabaseClient(request);
+      await initializeDirectories();
+
+      final contentType = request.headers['content-type'];
+      if (contentType == null ||
+          !contentType.toLowerCase().startsWith('multipart/form-data')) {
+        return shelf.Response.badRequest(
+            body: 'Content-Type must be multipart/form-data');
+      }
+
+      try {
+        final mediaType = MediaType.parse(contentType);
+        final boundary = mediaType.parameters['boundary'];
+        if (boundary == null) {
+          return shelf.Response.badRequest(body: 'Boundary not found');
+        }
+
+        final transformer = MimeMultipartTransformer(boundary);
+        final bodyBytes = await request.read().expand((e) => e).toList();
+        final stream = Stream.fromIterable([bodyBytes]);
+        final parts = await transformer.bind(stream).toList();
+
+        for (var part in parts) {
+          final contentDisposition = part.headers['content-disposition'];
+          if (contentDisposition == null) continue;
+
+          final filenameMatch =
+              RegExp(r'filename="([^"]*)"').firstMatch(contentDisposition);
+          if (filenameMatch == null) continue;
+
+          final filename = filenameMatch.group(1);
+          if (filename == null) continue;
+
+          final bytes = await part.fold<List<int>>(
+            [],
+            (prev, element) => [...prev, ...element],
+          );
+
+          final rawFilePath = path.join('./uploaded/raw', filename);
+          await File(rawFilePath).writeAsBytes(bytes);
+
+          if (isZipFile(bytes)) {
+            await processZipFile(rawFilePath);
+          } else {
+            final dataFilePath = path.join('./uploaded/data', filename);
+            await File(rawFilePath).copy(dataFilePath);
+          }
+          await uploadToSupabase(rawFilePath, filename, supabaseClient,
+              bucket: 'csvhich_archive', timestamped: true, upsert: false);
+        }
+
+        return shelf.Response.ok('File processed and uploaded successfully');
+      } catch (e, stackTrace) {
+        print('Error: $e\n$stackTrace');
+        return shelf.Response.internalServerError(
+            body: 'Error processing upload: $e');
+      } finally {
+        supabaseClient.dispose();
+        await File('./uploaded/raw').delete(recursive: true);
+        await File('./uploaded/data').delete(recursive: true);
+      }
+    });
+
+    return router;
+  }
+}

pubspec.lock

@@ -0,0 +1,293 @@
+# Generated by pub
+# See https://dart.dev/tools/pub/glossary#lockfile
+packages:
+  archive:
+    dependency: "direct main"
+    description:
+      name: archive
+      sha256: "6199c74e3db4fbfbd04f66d739e72fe11c8a8957d5f219f1f4482dbde6420b5a"
+      url: "https://pub.dev"
+    source: hosted
+    version: "4.0.2"
+  async:
+    dependency: transitive
+    description:
+      name: async
+      sha256: d2872f9c19731c2e5f10444b14686eb7cc85c76274bd6c16e1816bff9a3bab63
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.12.0"
+  clock:
+    dependency: transitive
+    description:
+      name: clock
+      sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.1.2"
+  collection:
+    dependency: transitive
+    description:
+      name: collection
+      sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.19.1"
+  crypto:
+    dependency: transitive
+    description:
+      name: crypto
+      sha256: "1e445881f28f22d6140f181e07737b22f1e099a5e1ff94b0af2f9e4a463f4855"
+      url: "https://pub.dev"
+    source: hosted
+    version: "3.0.6"
+  ffi:
+    dependency: transitive
+    description:
+      name: ffi
+      sha256: "16ed7b077ef01ad6170a3d0c57caa4a112a38d7a2ed5602e0aca9ca6f3d98da6"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.1.3"
+  functions_client:
+    dependency: transitive
+    description:
+      name: functions_client
+      sha256: "61597ed93be197b1be6387855e4b760e6aac2355fcfc4df6d20d2b4579982158"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.4.0"
+  gotrue:
+    dependency: transitive
+    description:
+      name: gotrue
+      sha256: d6362dff9a54f8c1c372bb137c858b4024c16407324d34e6473e59623c9b9f50
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.11.1"
+  http:
+    dependency: transitive
+    description:
+      name: http
+      sha256: b9c29a161230ee03d3ccf545097fccd9b87a5264228c5d348202e0f0c28f9010
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.2.2"
+  http_methods:
+    dependency: transitive
+    description:
+      name: http_methods
+      sha256: "6bccce8f1ec7b5d701e7921dca35e202d425b57e317ba1a37f2638590e29e566"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.1.1"
+  http_parser:
+    dependency: "direct main"
+    description:
+      name: http_parser
+      sha256: "76d306a1c3afb33fe82e2bbacad62a61f409b5634c915fceb0d799de1a913360"
+      url: "https://pub.dev"
+    source: hosted
+    version: "4.1.1"
+  intl:
+    dependency: "direct main"
+    description:
+      name: intl
+      sha256: "00f33b908655e606b86d2ade4710a231b802eec6f11e87e4ea3783fd72077a50"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.20.1"
+  jwt_decode:
+    dependency: transitive
+    description:
+      name: jwt_decode
+      sha256: d2e9f68c052b2225130977429d30f187aa1981d789c76ad104a32243cfdebfbb
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.3.1"
+  logging:
+    dependency: transitive
+    description:
+      name: logging
+      sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.3.0"
+  meta:
+    dependency: transitive
+    description:
+      name: meta
+      sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.16.0"
+  mime:
+    dependency: "direct main"
+    description:
+      name: mime
+      sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.0.0"
+  path:
+    dependency: "direct main"
+    description:
+      name: path
+      sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.9.1"
+  posix:
+    dependency: transitive
+    description:
+      name: posix
+      sha256: a0117dc2167805aa9125b82eee515cc891819bac2f538c83646d355b16f58b9a
+      url: "https://pub.dev"
+    source: hosted
+    version: "6.0.1"
+  postgrest:
+    dependency: transitive
+    description:
+      name: postgrest
+      sha256: "9f759ac497a24839addbed69d9569ea6d51d2e4834c672b8c2a73752fb6945c8"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.4.0"
+  realtime_client:
+    dependency: transitive
+    description:
+      name: realtime_client
+      sha256: "1bfcb7455fdcf15953bf18ac2817634ea5b8f7f350c7e8c9873141a3ee2c3e9c"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.4.1"
+  retry:
+    dependency: transitive
+    description:
+      name: retry
+      sha256: "822e118d5b3aafed083109c72d5f484c6dc66707885e07c0fbcb8b986bba7efc"
+      url: "https://pub.dev"
+    source: hosted
+    version: "3.1.2"
+  rxdart:
+    dependency: transitive
+    description:
+      name: rxdart
+      sha256: "5c3004a4a8dbb94bd4bf5412a4def4acdaa12e12f269737a5751369e12d1a962"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.28.0"
+  shelf:
+    dependency: "direct main"
+    description:
+      name: shelf
+      sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.4.2"
+  shelf_router:
+    dependency: "direct main"
+    description:
+      name: shelf_router
+      sha256: f5e5d492440a7fb165fe1e2e1a623f31f734d3370900070b2b1e0d0428d59864
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.1.4"
+  source_span:
+    dependency: transitive
+    description:
+      name: source_span
+      sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.10.1"
+  stack_trace:
+    dependency: transitive
+    description:
+      name: stack_trace
+      sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.12.1"
+  storage_client:
+    dependency: transitive
+    description:
+      name: storage_client
+      sha256: d80d34f0aa60e5199646bc301f5750767ee37310c2ecfe8d4bbdd29351e09ab0
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.3.0"
+  stream_channel:
+    dependency: transitive
+    description:
+      name: stream_channel
+      sha256: "4ac0537115a24d772c408a2520ecd0abb99bca2ea9c4e634ccbdbfae64fe17ec"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.1.3"
+  string_scanner:
+    dependency: transitive
+    description:
+      name: string_scanner
+      sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.4.1"
+  supabase:
+    dependency: "direct main"
+    description:
+      name: supabase
+      sha256: ea3daaf4fc76df9bf42ca00142f8d07b94400943a93d563e87f5575ea78f1c2c
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.6.1"
+  term_glyph:
+    dependency: transitive
+    description:
+      name: term_glyph
+      sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.2.2"
+  typed_data:
+    dependency: transitive
+    description:
+      name: typed_data
+      sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.4.0"
+  web:
+    dependency: transitive
+    description:
+      name: web
+      sha256: cd3543bd5798f6ad290ea73d210f423502e71900302dde696f8bff84bf89a1cb
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.1.0"
+  web_socket:
+    dependency: transitive
+    description:
+      name: web_socket
+      sha256: "3c12d96c0c9a4eec095246debcea7b86c0324f22df69893d538fcc6f1b8cce83"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.1.6"
+  web_socket_channel:
+    dependency: transitive
+    description:
+      name: web_socket_channel
+      sha256: "9f187088ed104edd8662ca07af4b124465893caf063ba29758f97af57e61da8f"
+      url: "https://pub.dev"
+    source: hosted
+    version: "3.0.1"
+  yet_another_json_isolate:
+    dependency: transitive
+    description:
+      name: yet_another_json_isolate
+      sha256: "56155e9e0002cc51ea7112857bbcdc714d4c35e176d43e4d3ee233009ff410c9"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.0.3"
+sdks:
+  dart: ">=3.5.0 <4.0.0"

pubspec.yaml

@@ -0,0 +1,16 @@
+name: file_upload_processor
+description: A file upload API server with ZIP processing
+version: 1.0.0
+
+environment:
+  sdk: '>=3.0.0 <4.0.0'
+
+dependencies:
+  shelf: #^1.4.1
+  shelf_router: #^1.1.4
+  path: #^1.9.0
+  http_parser: #^4.0.2
+  archive: #^3.4.10
+  mime: #^1.0.5
+  supabase: #^2.0.8
+  intl: #^0.19.0