From 8a4e1d64d408d45a8d3cc28ba67f32e65a986106 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Tue, 30 Apr 2024 23:09:13 +0300 Subject: [PATCH 1/5] added various b2id, b2_bucket_uri parsers --- b2/_internal/_cli/b2args.py | 73 +++++++++++++++++++++++++++++++++++- b2/_internal/console_tool.py | 33 ++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/b2/_internal/_cli/b2args.py b/b2/_internal/_cli/b2args.py index 1e2c26cb..ba744f6f 100644 --- a/b2/_internal/_cli/b2args.py +++ b/b2/_internal/_cli/b2args.py @@ -13,7 +13,7 @@ import argparse import functools from os import environ -from typing import Optional, Tuple +from typing import Optional, Tuple, Union from b2._internal._cli.arg_parser_types import wrap_with_argument_type_error from b2._internal._cli.argcompleters import b2uri_file_completer, bucket_name_completer @@ -21,7 +21,38 @@ B2_APPLICATION_KEY_ENV_VAR, B2_APPLICATION_KEY_ID_ENV_VAR, ) -from b2._internal._utils.uri import B2URI, B2URIBase, parse_b2_uri, parse_uri +from b2._internal._utils.uri import B2URI, B2FileIdURI, B2URIBase, parse_b2_uri, parse_uri + + +def b2id_uri(value: str) -> B2FileIdURI: + b2_uri = parse_b2_uri(value) + if not isinstance(b2_uri, B2FileIdURI): + raise ValueError(f"B2 URI pointing to a file id is required, but {value} was provided") + return b2_uri + + +def b2_bucket_uri(value: str) -> B2URI: + b2_uri = parse_b2_uri(value) + if not isinstance(b2_uri, B2URI): + raise ValueError( + f"B2 URI pointing to a bucket object is required, but {value} was provided" + ) + if b2_uri.path: + raise ValueError( + f"B2 URI pointing to a bucket object is required, but {value!r} was provided which contains path part: {b2_uri.path!r}" + ) + return b2_uri + + +def b2id_or_b2_bucket_uri(value: str) -> Union[B2URI, B2FileIdURI]: + b2_uri = parse_b2_uri(value) + if isinstance(b2_uri, B2URI): + if b2_uri.path: + raise ValueError( + f"B2 URI pointing to a bucket object is required, but {value!r} was provided which contains path part: {b2_uri.path!r}" + ) + return b2_uri + return b2_uri def b2id_or_file_like_b2_uri(value: str) -> B2URIBase: @@ -47,7 +78,10 @@ def parse_bucket_name(value: str, allow_all_buckets: bool = False) -> str: return str(value) +B2ID_URI_ARG_TYPE = wrap_with_argument_type_error(b2id_uri) +B2_BUCKET_URI_ARG_TYPE = wrap_with_argument_type_error(b2_bucket_uri) B2ID_OR_B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri) +B2ID_OR_B2_BUCKET_URI_ARG_TYPE = wrap_with_argument_type_error(b2id_or_b2_bucket_uri) B2ID_OR_B2_URI_OR_ALL_BUCKETS_ARG_TYPE = wrap_with_argument_type_error( functools.partial(parse_b2_uri, allow_all_buckets=True) ) @@ -84,6 +118,33 @@ def add_b2_uri_argument( ).completer = b2uri_file_completer +def add_b2_bucket_uri_argument( + parser: argparse.ArgumentParser, + name="B2_URI", +): + """ + Add B2 URI as an argument to the parser. + + B2 URI can point to a bucket. + """ + parser.add_argument( + name, + type=B2_BUCKET_URI_ARG_TYPE, + help="B2 URI pointing to a bucket, e.g. b2://yourBucket", + ).completer = b2uri_file_completer + + +def add_b2id_uri_argument(parser: argparse.ArgumentParser, name="B2_URI"): + """ + Add B2 URI (b2id://) as an argument to the parser. + """ + parser.add_argument( + name, + type=B2ID_URI_ARG_TYPE, + help="B2 URI pointing to a file id. e.g. b2id://fileId", + ).completer = b2uri_file_completer + + def add_b2id_or_b2_uri_argument( parser: argparse.ArgumentParser, name="B2_URI", *, allow_all_buckets: bool = False ): @@ -114,6 +175,14 @@ def add_b2id_or_b2_uri_argument( argument_spec.completer = b2uri_file_completer +def add_b2id_or_b2_bucket_uri_argument(parser: argparse.ArgumentParser, name="B2_URI"): + parser.add_argument( + name, + type=B2ID_OR_B2_BUCKET_URI_ARG_TYPE, + help="B2 URI pointing to a bucket, or a file id. e.g. b2://yourBucket, or b2id://fileId", + ).completer = b2uri_file_completer + + def add_b2id_or_file_like_b2_uri_argument(parser: argparse.ArgumentParser, name="B2_URI"): """ Add a B2 URI pointing to a file as an argument to the parser. diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 7da72e73..149f884c 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -135,9 +135,12 @@ ) from b2._internal._cli.b2api import _get_b2api_for_profile, _get_inmemory_b2api from b2._internal._cli.b2args import ( + add_b2_bucket_uri_argument, add_b2_uri_argument, + add_b2id_or_b2_bucket_uri_argument, add_b2id_or_b2_uri_argument, add_b2id_or_file_like_b2_uri_argument, + add_b2id_uri_argument, add_bucket_name_argument, get_keyid_and_key_from_env_vars, ) @@ -762,6 +765,36 @@ def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URI | B2FileIdURI: return args.B2_URI +class B2IDOrB2BucketURIMixin: + @classmethod + def _setup_parser(cls, parser): + add_b2id_or_b2_bucket_uri_argument(parser) + super()._setup_parser(parser) + + def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URI | B2FileIdURI: + return args.B2_URI + + +class B2BucketURIMixin: + @classmethod + def _setup_parser(cls, parser): + add_b2_bucket_uri_argument(parser) + super()._setup_parser(parser) + + def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URI: + return args.B2_URI + + +class B2IDURIMixin: + @classmethod + def _setup_parser(cls, parser): + add_b2id_uri_argument(parser) + super()._setup_parser(parser) + + def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2FileIdURI: + return args.B2_URI + + class UploadModeMixin(Described): """ Use --incremental-mode to allow for incremental file uploads to safe bandwidth. This will only affect files, which From e675d85b36ecf12a18e1e21170ca69e949dd2e66 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Tue, 30 Apr 2024 22:54:16 +0300 Subject: [PATCH 2/5] added file large subcommand --- b2/_internal/console_tool.py | 172 +++++++++++++----- changelog.d/+command-file-large.added.md | 1 + changelog.d/+command-file-large.deprecated.md | 3 + 3 files changed, 126 insertions(+), 50 deletions(-) create mode 100644 changelog.d/+command-file-large.added.md create mode 100644 changelog.d/+command-file-large.deprecated.md diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 149f884c..96d3e3f4 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -1376,51 +1376,35 @@ def _get_user_requested_realm(cls, args) -> str | None: return os.environ.get(B2_ENVIRONMENT_ENV_VAR) -class CancelAllUnfinishedLargeFiles(Command): +class FileLargeUnfinishedCancelBase(Command): """ - Lists all large files that have been started but not - finished and cancels them. Any parts that have been - uploaded will be deleted. - - Requires capability: - - - **listFiles** - - **writeFiles** - """ - - @classmethod - def _setup_parser(cls, parser): - add_bucket_name_argument(parser) - super()._setup_parser(parser) - - def _run(self, args): - bucket = self.api.get_bucket_by_name(args.bucketName) - for file_version in bucket.list_unfinished_large_files(): - bucket.cancel_large_file(file_version.file_id) - self._print(file_version.file_id, 'canceled') - return 0 - - -class CancelLargeFile(Command): - """ - Cancels a large file upload. Used to undo a ``start-large-file``. - + When used with a b2id://fileId, cancels a large file upload. Cannot be used once the file is finished. After finishing, - using ``delete-file-version`` to delete the large file. + use ``rm`` to delete the large file. + + When used with a b2://bucketName, lists all large files that + have been started but not finished and cancels them. Any parts + that have been uploaded will be deleted. Requires capability: + - **listFiles** (if canceling a bucket) - **writeFiles** """ - @classmethod - def _setup_parser(cls, parser): - parser.add_argument('fileId') - super()._setup_parser(parser) - def _run(self, args): - self.api.cancel_large_file(args.fileId) - self._print(args.fileId, 'canceled') + b2_uri = self.get_b2_uri_from_arg(args) + if isinstance(b2_uri, B2FileIdURI): + self.api.cancel_large_file(b2_uri.file_id) + self._print(b2_uri.file_id, 'canceled') + elif isinstance(b2_uri, B2URI): + bucket = self.api.get_bucket_by_name(b2_uri.bucket_name) + for file_version in bucket.list_unfinished_large_files(): + bucket.cancel_large_file(file_version.file_id) + self._print(file_version.file_id, 'canceled') + else: + self._print_stderr(f'ERROR: unsupported URI "{b2_uri}"') + return 1 return 0 @@ -2290,7 +2274,7 @@ def timestamp_display(self, timestamp_or_none): return dt.strftime('%Y-%m-%d'), dt.strftime('%H:%M:%S') -class ListParts(Command): +class FileLargePartsBase(Command): """ Lists all of the parts that have been uploaded for the given large file, which must be a file that was started but not @@ -2301,18 +2285,14 @@ class ListParts(Command): - **writeFiles** """ - @classmethod - def _setup_parser(cls, parser): - parser.add_argument('largeFileId') - super()._setup_parser(parser) - def _run(self, args): - for part in self.api.list_parts(args.largeFileId): + b2_uri = self.get_b2_uri_from_arg(args) + for part in self.api.list_parts(b2_uri.file_id): self._print('%5d %9d %s' % (part.part_number, part.content_length, part.content_sha1)) return 0 -class ListUnfinishedLargeFiles(Command): +class FileLargeUnfinishedListBase(Command): """ Lists all of the large files in the bucket that were started, but not finished or canceled. @@ -2322,13 +2302,9 @@ class ListUnfinishedLargeFiles(Command): - **listFiles** """ - @classmethod - def _setup_parser(cls, parser): - add_bucket_name_argument(parser) - super()._setup_parser(parser) - def _run(self, args): - bucket = self.api.get_bucket_by_name(args.bucketName) + b2_uri = self.get_b2_uri_from_arg(args) + bucket = self.api.get_bucket_by_name(b2_uri.bucket_name) for unfinished in bucket.list_unfinished_large_files(): file_info_text = ' '.join( f'{k}={unfinished.file_info[k]}' for k in sorted(unfinished.file_info) @@ -5154,6 +5130,102 @@ class DeleteFileVersion(CmdReplacedByMixin, DeleteFileVersionBase): replaced_by_cmd = Rm +@File.subcommands_registry.register +class FileLarge(Command): + """ + Large file uploads management subcommands. + + For more information on each subcommand, use ``{NAME} file large SUBCOMMAND --help``. + + Examples: + + .. code-block:: + + {NAME} file large parts + {NAME} file large unfinished + """ + COMMAND_NAME = 'large' + subcommands_registry = ClassRegistry(attr_name='COMMAND_NAME') + + +@FileLarge.subcommands_registry.register +class FileLargeParts(B2IDURIMixin, FileLargePartsBase): + __doc__ = FileLargePartsBase.__doc__ + COMMAND_NAME = 'parts' + + +@FileLarge.subcommands_registry.register +class FileLargeUnfinished(Command): + """ + Large file unfinished uploads management subcommands. + + For more information on each subcommand, use ``{NAME} file large unfinished SUBCOMMAND --help``. + + Examples: + + .. code-block:: + + {NAME} file large unfinished list + {NAME} file large unfinished cancel + """ + COMMAND_NAME = 'unfinished' + subcommands_registry = ClassRegistry(attr_name='COMMAND_NAME') + + +@FileLargeUnfinished.subcommands_registry.register +class FileLargeUnfinishedList(B2BucketURIMixin, FileLargeUnfinishedListBase): + __doc__ = FileLargePartsBase.__doc__ + COMMAND_NAME = 'list' + + +@FileLargeUnfinished.subcommands_registry.register +class FileLargeUnfinishedCancel(B2IDOrB2BucketURIMixin, FileLargeUnfinishedCancelBase): + __doc__ = FileLargeUnfinishedCancelBase.__doc__ + COMMAND_NAME = 'cancel' + + +class ListParts(CmdReplacedByMixin, B2URIFileIDArgMixin, FileLargePartsBase): + __doc__ = FileLargePartsBase.__doc__ + replaced_by_cmd = (File, FileLarge, FileLargeParts) + + +class ListUnfinishedLargeFiles( + CmdReplacedByMixin, B2URIBucketArgMixin, FileLargeUnfinishedListBase +): + __doc__ = FileLargeUnfinishedListBase.__doc__ + replaced_by_cmd = (File, FileLarge, FileLargeUnfinished, FileLargeUnfinishedList) + + +class CancelAllUnfinishedLargeFiles( + CmdReplacedByMixin, B2URIBucketArgMixin, FileLargeUnfinishedCancelBase +): + """ + Lists all large files that have been started but not + finished and cancels them. Any parts that have been + uploaded will be deleted. + + Requires capability: + + - **listFiles** + - **writeFiles** + """ + replaced_by_cmd = (File, FileLarge, FileLargeUnfinished, FileLargeUnfinishedCancel) + + +class CancelLargeFile(CmdReplacedByMixin, B2URIFileIDArgMixin, FileLargeUnfinishedCancelBase): + """ + Cancels a large file upload. Used to undo a ``start-large-file``. + + Cannot be used once the file is finished. After finishing, + using ``delete-file-version`` to delete the large file. + + Requires capability: + + - **writeFiles** + """ + replaced_by_cmd = (File, FileLarge, FileLargeUnfinished, FileLargeUnfinishedCancel) + + class ConsoleTool: """ Implements the commands available in the B2 command-line tool diff --git a/changelog.d/+command-file-large.added.md b/changelog.d/+command-file-large.added.md new file mode 100644 index 00000000..d736529b --- /dev/null +++ b/changelog.d/+command-file-large.added.md @@ -0,0 +1 @@ +Add `file large {parts|unfinished list|unfinished cancel}` commands. \ No newline at end of file diff --git a/changelog.d/+command-file-large.deprecated.md b/changelog.d/+command-file-large.deprecated.md new file mode 100644 index 00000000..dd7a7c04 --- /dev/null +++ b/changelog.d/+command-file-large.deprecated.md @@ -0,0 +1,3 @@ +Deprecated `list-parts`, use `file large parts` instead. +Deprecated `list-unfinished-large-files`, use `file large unfinished list` instead. +Deprecated `cancel-large-file` amd `cancel-all-unfinished-large-files`, use `file large unfinished cancel` instead. \ No newline at end of file From 8568d63f67210786e88a265db221d13cbad45ecc Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Tue, 30 Apr 2024 23:02:07 +0300 Subject: [PATCH 3/5] updated file large tests --- test/unit/test_console_tool.py | 97 +++++++++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 6 deletions(-) diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 82f5e13c..82cc2eda 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -1706,7 +1706,11 @@ def test_get_download_auth_url_with_encoding(self): def test_list_unfinished_large_files_with_none(self): self._authorize_account() self._create_my_bucket() - self._run_command(['list-unfinished-large-files', 'my-bucket'], '', '', 0) + self._run_command( + ['list-unfinished-large-files', 'my-bucket'], '', + 'WARNING: `list-unfinished-large-files` command is deprecated. Use `file large unfinished list` instead.\n', + 0 + ) def test_upload_large_file(self): self._authorize_account() @@ -2992,7 +2996,18 @@ def setUp(self): def test_cancel_large_file(self): file = self.v1_bucket.start_large_file('file1', 'text/plain', {}) - self._run_command(['cancel-large-file', file.file_id], '9999 canceled\n', '', 0) + self._run_command( + ['file', 'large', 'unfinished', 'cancel', f'b2id://{file.file_id}'], '9999 canceled\n', + '', 0 + ) + + def test_cancel_large_file_deprecated(self): + file = self.v1_bucket.start_large_file('file1', 'text/plain', {}) + self._run_command( + ['cancel-large-file', file.file_id], '9999 canceled\n', + 'WARNING: `cancel-large-file` command is deprecated. Use `file large unfinished cancel` instead.\n', + 0 + ) def test_cancel_all_large_file(self): self.v1_bucket.start_large_file('file1', 'text/plain', {}) @@ -3003,15 +3018,58 @@ def test_cancel_all_large_file(self): ''' self._run_command( - ['cancel-all-unfinished-large-files', 'my-v1-bucket'], expected_stdout, '', 0 + ['file', 'large', 'unfinished', 'cancel', 'b2://my-v1-bucket'], expected_stdout, '', 0 + ) + + def test_cancel_all_large_file_deprecated(self): + self.v1_bucket.start_large_file('file1', 'text/plain', {}) + self.v1_bucket.start_large_file('file2', 'text/plain', {}) + expected_stdout = ''' + 9999 canceled + 9998 canceled + ''' + + self._run_command( + ['cancel-all-unfinished-large-files', 'my-v1-bucket'], expected_stdout, + 'WARNING: `cancel-all-unfinished-large-files` command is deprecated. Use `file large unfinished cancel` instead.\n', + 0 ) def test_list_parts_with_none(self): file = self.v1_bucket.start_large_file('file', 'text/plain', {}) - self._run_command(['list-parts', file.file_id], '', '', 0) + self._run_command(['file', 'large', 'parts', f'b2id://{file.file_id}'], '', '', 0) + + def test_list_parts_with_none_deprecated(self): + file = self.v1_bucket.start_large_file('file', 'text/plain', {}) + self._run_command( + ['list-parts', file.file_id], '', + 'WARNING: `list-parts` command is deprecated. Use `file large parts` instead.\n', 0 + ) def test_list_parts_with_parts(self): + bucket = self.b2_api.get_bucket_by_name('my-bucket') + file = self.v1_bucket.start_large_file('file', 'text/plain', {}) + content = b'hello world' + large_file_upload_state = mock.MagicMock() + large_file_upload_state.has_error.return_value = False + bucket.api.services.upload_manager._upload_part( + bucket.id_, file.file_id, UploadSourceBytes(content), 1, large_file_upload_state, None, + None + ) + bucket.api.services.upload_manager._upload_part( + bucket.id_, file.file_id, UploadSourceBytes(content), 3, large_file_upload_state, None, + None + ) + expected_stdout = ''' + 1 11 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed + 3 11 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed + ''' + self._run_command( + ['file', 'large', 'parts', f'b2id://{file.file_id}'], expected_stdout, '', 0 + ) + + def test_list_parts_with_parts_deprecated(self): bucket = self.b2_api.get_bucket_by_name('my-bucket') file = self.v1_bucket.start_large_file('file', 'text/plain', {}) content = b'hello world' @@ -3030,7 +3088,10 @@ def test_list_parts_with_parts(self): 3 11 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed ''' - self._run_command(['list-parts', file.file_id], expected_stdout, '', 0) + self._run_command( + ['list-parts', file.file_id], expected_stdout, + 'WARNING: `list-parts` command is deprecated. Use `file large parts` instead.\n', 0 + ) def test_list_unfinished_large_files_with_some(self): api_url = self.account_info.get_api_url() @@ -3048,7 +3109,31 @@ def test_list_unfinished_large_files_with_some(self): 9997 file3 application/json ''' - self._run_command(['list-unfinished-large-files', 'my-bucket'], expected_stdout, '', 0) + self._run_command( + ['file', 'large', 'unfinished', 'list', 'b2://my-bucket'], expected_stdout, '', 0 + ) + + def test_list_unfinished_large_files_with_some_deprecated(self): + api_url = self.account_info.get_api_url() + auth_token = self.account_info.get_account_auth_token() + self.raw_api.start_large_file(api_url, auth_token, 'bucket_0', 'file1', 'text/plain', {}) + self.raw_api.start_large_file( + api_url, auth_token, 'bucket_0', 'file2', 'text/plain', {'color': 'blue'} + ) + self.raw_api.start_large_file( + api_url, auth_token, 'bucket_0', 'file3', 'application/json', {} + ) + expected_stdout = ''' + 9999 file1 text/plain + 9998 file2 text/plain color=blue + 9997 file3 application/json + ''' + + self._run_command( + ['list-unfinished-large-files', 'my-bucket'], expected_stdout, + 'WARNING: `list-unfinished-large-files` command is deprecated. Use `file large unfinished list` instead.\n', + 0 + ) class TestRmConsoleTool(BaseConsoleToolTest): From d3d93567abfe18aa3f36192a582ed8de5a0e2174 Mon Sep 17 00:00:00 2001 From: adal-chiriliuc-reef <165330877+adal-chiriliuc-reef@users.noreply.github.com> Date: Wed, 1 May 2024 12:58:55 +0300 Subject: [PATCH 4/5] PR change request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Maciej Urbański <122983254+mjurbanski-reef@users.noreply.github.com> --- b2/_internal/console_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index 96d3e3f4..a7562bce 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -1388,7 +1388,7 @@ class FileLargeUnfinishedCancelBase(Command): Requires capability: - - **listFiles** (if canceling a bucket) + - **listFiles** (if canceling unfinished large files in a bucket) - **writeFiles** """ From 7a1b430934e74fb3fde627806e3a9663cc75ada1 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Wed, 1 May 2024 13:07:52 +0300 Subject: [PATCH 5/5] updated usage examples --- b2/_internal/console_tool.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index a7562bce..2d0238c6 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -5141,8 +5141,10 @@ class FileLarge(Command): .. code-block:: - {NAME} file large parts - {NAME} file large unfinished + {NAME} file large parts b2id://yourFileId + {NAME} file large unfinished list b2://yourBucket + {NAME} file large unfinished cancel b2://yourBucket + {NAME} file large unfinished cancel b2id://yourFileId """ COMMAND_NAME = 'large' subcommands_registry = ClassRegistry(attr_name='COMMAND_NAME') @@ -5165,8 +5167,9 @@ class FileLargeUnfinished(Command): .. code-block:: - {NAME} file large unfinished list - {NAME} file large unfinished cancel + {NAME} file large unfinished list b2://yourBucket + {NAME} file large unfinished cancel b2://yourBucket + {NAME} file large unfinished cancel b2id://yourFileId """ COMMAND_NAME = 'unfinished' subcommands_registry = ClassRegistry(attr_name='COMMAND_NAME')