diff --git a/teraserver/python/modules/FlaskModule/API/user/UserQueryAssetsArchive.py b/teraserver/python/modules/FlaskModule/API/user/UserQueryAssetsArchive.py index fb7dd674..2997c022 100644 --- a/teraserver/python/modules/FlaskModule/API/user/UserQueryAssetsArchive.py +++ b/teraserver/python/modules/FlaskModule/API/user/UserQueryAssetsArchive.py @@ -118,7 +118,7 @@ def get_assets_for_project(project: TeraProject, path: str = ''): p = TeraProject.get_project_by_id(args['id_project']) site = p.project_site - get_assets_for_project(p, f"/{site.site_name}") + get_assets_for_project(p, f"{site.site_name}") elif args['id_participant'] is not None: # Get all assets for participant @@ -129,7 +129,7 @@ def get_assets_for_project(project: TeraProject, path: str = ''): project = participant.participant_project site = project.project_site get_assets_for_participant(participant, - f"/{site.site_name}/{project.project_name}") + f"{site.site_name}/{project.project_name}") elif args['id_session'] is not None: # Get all assets for session @@ -146,7 +146,7 @@ def get_assets_for_project(project: TeraProject, path: str = ''): project = participant.participant_project site = project.project_site get_assets_for_session(sess, - f"/{site.site_name}/{project.project_name}/{participant.participant_name}") + f"{site.site_name}/{project.project_name}/{participant.participant_name}") else: return gettext('Missing required parameter'), 400 @@ -178,8 +178,6 @@ def get_assets_for_project(project: TeraProject, path: str = ''): file_transfer_service_host = file_transfer_service.service_hostname file_transfer_service_port = file_transfer_service.service_port - #archive_file_infos_url = f"https://{server_name}:{port}/file/api/archives/infos" - #archive_file_upload_url = f"https://{server_name}:{port}/file/api/archives" archive_file_infos_url = f"http://{file_transfer_service_host}:{file_transfer_service_port}/api/archives/infos" archive_file_upload_url = f"http://{file_transfer_service_host}:{file_transfer_service_port}/api/archives" @@ -232,8 +230,8 @@ def process_monitor_thread(process, job_id, flaskModule: FlaskModule): f"job: {job_id} finished with code: {return_code}. Duration: {duration_s} seconds.") # Join thread later - thread = threading.currentThread() - reactor.callFromThread(lambda: thread.join()) + current_thread = threading.current_thread() + reactor.callFromThread(lambda: current_thread.join()) thread = threading.Thread(target=process_monitor_thread, args=(process, job_id, self.module)) thread.start() diff --git a/teraserver/python/services/FileTransferService/API/QueryArchiveFile.py b/teraserver/python/services/FileTransferService/API/QueryArchiveFile.py index 0e4aca56..704d7c58 100644 --- a/teraserver/python/services/FileTransferService/API/QueryArchiveFile.py +++ b/teraserver/python/services/FileTransferService/API/QueryArchiveFile.py @@ -59,7 +59,8 @@ def get(self): src_dir = flask_app.config['UPLOAD_FOLDER'] # Log access - self.module.logger.log_info('FileTransferService.FlaskModule.QueryArchiveFile', f'User {requester_uuid} downloaded archive {archive.archive_uuid}') + self.module.logger.log_info('FileTransferService.FlaskModule.QueryArchiveFile', + f'User {requester_uuid} downloaded archive {archive.archive_uuid}') # Send file filename = archive.archive_original_filename @@ -144,8 +145,6 @@ def post(self): except Exception as e: return gettext('Error parsing archive information : ') + str(e), 400 - return gettext("Not implemented"), 501 - @api.expect(delete_parser, validate=True) @api.doc(description='Delete archive', responses={200: 'Success - archive deleted', diff --git a/teraserver/python/services/FileTransferService/API/QueryArchiveFileInfos.py b/teraserver/python/services/FileTransferService/API/QueryArchiveFileInfos.py index d9bedcd9..0be70856 100644 --- a/teraserver/python/services/FileTransferService/API/QueryArchiveFileInfos.py +++ b/teraserver/python/services/FileTransferService/API/QueryArchiveFileInfos.py @@ -109,8 +109,8 @@ def post(self): # Add Creation datetime archive.archive_creation_datetime = datetime.datetime.now(datetime.timezone.utc) - # Add Expiration datetime in 30 days - archive.archive_expiration_datetime = archive.archive_creation_datetime + datetime.timedelta(days=30) + # Add Expiration datetime in 7 days + archive.archive_expiration_datetime = archive.archive_creation_datetime + datetime.timedelta(days=7) # Make sure file name is secure archive.archive_original_filename = secure_filename(archive.archive_original_filename) diff --git a/teraserver/python/services/FileTransferService/FileTransferService.py b/teraserver/python/services/FileTransferService/FileTransferService.py index 8f61c332..ac59dec9 100644 --- a/teraserver/python/services/FileTransferService/FileTransferService.py +++ b/teraserver/python/services/FileTransferService/FileTransferService.py @@ -35,8 +35,7 @@ def __init__(self, config_man: ConfigManager, this_service_info): # Create cleaning task, every hour by default, can specify period (s) in arg. self.archive_cleaning_task = self.setup_file_archive_cleanup_task(3600.0) - - def setup_file_archive_cleanup_task(self, period_s:float = 3600.0) -> task.LoopingCall: + def setup_file_archive_cleanup_task(self, period_s: float = 3600.0) -> task.LoopingCall: loop = task.LoopingCall(self.archive_cleanup_task_callback) # Start looping every period_s seconds. @@ -77,7 +76,6 @@ def archive_cleanup_task_callback(self): self.logger.log_error('FileTransferService', f'Error deleting archive {archive.archive_uuid} : {e}.') continue - def cbArchiveLoopDone(self, result): """ Called when file archive cleanup task was stopped with success. @@ -91,7 +89,6 @@ def ebArchiveLoopFailed(self, failure): self.logger.log_error('FileTransferService', f'ebArchiveLoopFailed : {failure}.') print('ebArchiveLoopFailed', failure) - def verify_file_upload_directory(self, config: ConfigManager, create=True): file_upload_path = config.filetransfer_config['files_directory'] diff --git a/teraserver/python/workers/AssetsArchiveWorker.py b/teraserver/python/workers/AssetsArchiveWorker.py index 1e6a75de..a121f806 100644 --- a/teraserver/python/workers/AssetsArchiveWorker.py +++ b/teraserver/python/workers/AssetsArchiveWorker.py @@ -74,7 +74,7 @@ # zip_buffer = BytesIO() # Change status of the file transfer to 'in progress' - archive_info['archive_status'] = 1 # STATUS_INPROGRESS + archive_info['archive_status'] = 1 # STATUS_INPROGRESS response = requests.post(archive_file_infos_url, json={'archive': archive_info}, headers={'Authorization': 'OpenTera ' + job_info['service_token']}, timeout=30, verify=args.verify) @@ -83,8 +83,8 @@ zip_buffer = tempfile.NamedTemporaryFile(delete=True) # Create a ZipFile object to write to the in-memory stream - with zipfile.ZipFile(zip_buffer, 'w') as zip_file: - + with (zipfile.ZipFile(zip_buffer, 'w') as zip_file): + files_in_zip = {} for _, service_data in job_info['assets_map'].items(): for asset in service_data['service_assets']: @@ -108,7 +108,18 @@ if response.status_code == 200: # Add to zip file - file_path = pathlib.Path(path) / pathlib.Path(f"[{asset['asset_uuid']}]_{asset['asset_name']}") + # file_path = pathlib.Path(path) / pathlib.Path(f"[{asset['asset_uuid']}]_{asset['asset_name']}") + file_path = pathlib.Path(path) / pathlib.Path(f"{asset['asset_name']}") + + if file_path in files_in_zip: + files_in_zip[file_path] = files_in_zip[file_path] + 1 + # File with that name already exists - insert number + file_name_ext = '.' + asset['asset_name'].split('.')[-1] + file_name = asset['asset_name'].removesuffix(file_name_ext) + '(' + \ + str(files_in_zip[file_path]) + ')' + file_name_ext + file_path = pathlib.Path(path) / pathlib.Path(f"{file_name}") + else: + files_in_zip[file_path] = 1 # Compress zip_file.writestr(str(file_path), response.content)