diff --git a/storage_s3/README.md b/storage_s3/README.md index 0ff9391b..74ae9ffd 100644 --- a/storage_s3/README.md +++ b/storage_s3/README.md @@ -15,6 +15,11 @@ to local storage (but it would of course be possible to write a script for this) ## Changelog +### 3.3.1 + +- Adapt to Indico 3.3.5 changes +- Add file import handler for event export/import operations that require copying many files on S3 + ### 3.3 - Support (and require) Python 3.12 diff --git a/storage_s3/indico_storage_s3/import_event_files.py b/storage_s3/indico_storage_s3/import_event_files.py new file mode 100644 index 00000000..06ddfe2f --- /dev/null +++ b/storage_s3/indico_storage_s3/import_event_files.py @@ -0,0 +1,57 @@ +# This file is part of the Indico plugins. +# Copyright (C) 2002 - 2024 CERN +# +# The Indico plugins are free software; you can redistribute +# them and/or modify them under the terms of the MIT License; +# see the LICENSE file for more details. + +# XXX: This module is not as unused as it may appear. It can be referenced in +# the `indico event import-files` command when all files are on S3 and one wants +# to do a fast copy using rclone + +import subprocess +import sys + +import click + +from indico.core.storage.backend import Storage, get_storage + +from indico.util.console import verbose_iterator +from indico_storage_s3.storage import S3StorageBase + + +def copy_files(mapping, backends): + target_backend_names = list({x[1][0] for x in mapping}) + assert len(target_backend_names) == 1 + target_backend: Storage = get_storage(target_backend_names[0]) + click.echo(f'Target backend: {target_backend_names[0]} ({target_backend.name})') + if not isinstance(target_backend, S3StorageBase): + click.echo('Target backend is not S3; aborting') + sys.exit(1) + if not all(isinstance(x, S3StorageBase) for x in backends.values()): + click.echo('Not all source backends are S3; aborting') + sys.exit(1) + rclone_remote_name = click.prompt('rclone remote name') + mapping.sort() + iterator = verbose_iterator(mapping, len(mapping), get_title=lambda x: x[1][1], print_every=1, + print_total_time=True) + for (source_backend_name, source_file_id), (__, target_file_id) in iterator: + source_backend: S3StorageBase = backends[source_backend_name] + _run_rclone( + rclone_remote_name, + '/'.join(source_backend._parse_file_id(source_file_id)), + '/'.join(target_backend._parse_file_id(target_file_id)), + ) + + +def _run_rclone(remote, src, dst): + subprocess.check_call( + [ + 'rclone', + '--s3-no-check-bucket', + '--s3-acl=bucket-owner-full-control', + 'copyto', + f'{remote}:{src}', + f'{remote}:{dst}', + ] + ) diff --git a/storage_s3/indico_storage_s3/storage.py b/storage_s3/indico_storage_s3/storage.py index 34bee766..3e116f72 100644 --- a/storage_s3/indico_storage_s3/storage.py +++ b/storage_s3/indico_storage_s3/storage.py @@ -204,10 +204,10 @@ def _get_current_bucket_name(self): def _parse_file_id(self, file_id): return self.bucket_name, file_id - def save(self, name, content_type, filename, fileobj): + def save(self, name, content_type, filename, fileobj, *, dry_run=False): try: bucket = self._get_current_bucket_name() - checksum = self._save(bucket, name, content_type, fileobj) + checksum = self._save(bucket, name, content_type, fileobj) if not dry_run else None return name, checksum except Exception as exc: raise StorageError(f'Could not save "{name}": {exc}') from exc @@ -250,10 +250,10 @@ def _replace_bucket_placeholders(self, name, date): name = name.replace('', date.strftime('%m')) return name.replace('', date.strftime('%W')) - def save(self, name, content_type, filename, fileobj): + def save(self, name, content_type, filename, fileobj, *, dry_run=False): try: bucket = self._get_current_bucket_name() - checksum = self._save(bucket, name, content_type, fileobj) + checksum = self._save(bucket, name, content_type, fileobj) if not dry_run else None file_id = f'{bucket}//{name}' return file_id, checksum except Exception as exc: diff --git a/storage_s3/pyproject.toml b/storage_s3/pyproject.toml index 74a63088..9caea844 100644 --- a/storage_s3/pyproject.toml +++ b/storage_s3/pyproject.toml @@ -2,7 +2,7 @@ name = 'indico-plugin-storage-s3' description = 'S3 storage backend for Indico' readme = 'README.md' -version = '3.3' +version = '3.3.1' license = 'MIT' authors = [{ name = 'Indico Team', email = 'indico-team@cern.ch' }] classifiers = [