Merge pull request #61 from PrivateAIM/59-upload-file-chunk

Use spooled upload files for submitting results to hub
PrivateAIM · Oct 18, 2024 · e9f248b · e9f248b
2 parents 815a56a + 5921321
commit e9f248b
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 3 deletions.
diff --git a/project/hub.py b/project/hub.py
@@ -1,4 +1,5 @@
 import time
+import typing
 import urllib.parse
 from datetime import datetime
 from io import BytesIO
@@ -595,7 +596,7 @@ def upload_to_bucket(
         self,
         bucket_id: str | UUID,
         file_name: str,
-        file: bytes | BytesIO,
+        file: typing.IO[bytes],
         content_type: str = "application/octet-stream",
     ) -> ResourceList[BucketFile]:
         """

diff --git a/project/routers/final.py b/project/routers/final.py
@@ -36,7 +36,7 @@ async def submit_final_result_to_hub(
     bucket_file_lst = storage_client.upload_to_bucket(
         analysis_bucket.external_id,
         file.filename,
-        await file.read(),  # TODO should be chunked as well
+        file.file,
         file.content_type or "application/octet-stream",
     )
 

diff --git a/project/routers/intermediate.py b/project/routers/intermediate.py
@@ -46,7 +46,7 @@ async def submit_intermediate_result_to_hub(
     bucket_file_lst = storage_client.upload_to_bucket(
         analysis_bucket.external_id,
         file.filename,
-        await file.read(),  # TODO should be chunked for large files
+        file.file,
         file.content_type or "application/octet-stream",
     )