allow the upload of files in a recursive way

isbkch · Oct 28, 2020 · b1f15b4 · b1f15b4
1 parent 0bd9987
commit b1f15b4
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 2 deletions.
diff --git a/ftp-to-S3/rootfs/usr/bin/ftp-to-s3/ftp-to-s3.py b/ftp-to-S3/rootfs/usr/bin/ftp-to-s3/ftp-to-s3.py
@@ -137,7 +137,7 @@ def upload_file(file: Path, s3_bucket: S3Bucket):
         logger.critical("Error listing contents of S3 bucket!")
         sys.exit(1)
 
-    local_files = [x.name for x in config.monitor_path.iterdir()
+    local_files = [x for x in config.monitor_path.glob('**/*')
                    if x.is_file()]
 
     for local_file in local_files:
@@ -154,6 +154,12 @@ def upload_file(file: Path, s3_bucket: S3Bucket):
                     f"Local file {file} does not match the file in S3")
                 try:
                     upload_file(file, s3_bucket)
+                    # delete after upload
+                    # if os.path.exists(file):
+                    #     os.remove(file)
+                    # else:
+                    #     logger.debug(
+                    #         f"Did not delete: File {file} does not exist")
                 except S3BucketError as err:
                     logger.exception(f"Error uploading file: {err}")
         else:

diff --git a/ftp-to-S3/rootfs/usr/bin/ftp-to-s3/s3bucket.py b/ftp-to-S3/rootfs/usr/bin/ftp-to-s3/s3bucket.py
@@ -53,14 +53,15 @@ def upload_file(self, file: str):
         Args:
             file (str): Full path of file to upload
         """
+        clean_file_name = file.lstrip("/")
         extra_args = {}
         extra_args["StorageClass"] = self.storage_class
 
         try:
             logger.info(f"Uploading file [{file}] to S3")
             self.s3_client.upload_file(Filename=file,
                                        Bucket=self.bucket_name,
-                                       Key=file,
+                                       Key=clean_file_name,
                                        ExtraArgs=extra_args)
             logger.info(
                 f"Uploaded file [{file}] to S3 bucket [{self.bucket_name}] using storage class [{self.storage_class}]")