diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48376151a..d4cc5c484 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,12 +81,13 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.10"] + # pypy version pin was required due 7.3.13 being broken https://foss.heptapod.net/pypy/pypy/-/issues/4021 + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.10-v7.3.12"] exclude: - os: "macos-latest" - python-version: "pypy-3.10" + python-version: "pypy-3.10-v7.3.12" - os: "windows-latest" - python-version: "pypy-3.10" + python-version: "pypy-3.10-v7.3.12" steps: - uses: actions/checkout@v3 with: @@ -103,7 +104,7 @@ jobs: run: nox -vs integration -- -m "not require_secrets" - name: Run integration tests (with secrets) # Limit CI workload by running integration tests with secrets only on edge Python versions. - if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && contains(fromJSON('["3.7", "pypy-3.10", "3.11"]'), matrix.python-version) }} + if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && contains(fromJSON('["3.7", "pypy-3.10-v7.3.12", "3.11"]'), matrix.python-version) }} run: nox -vs integration -- -m "require_secrets" --cleanup test-docker: needs: cleanup_buckets diff --git a/CHANGELOG.md b/CHANGELOG.md index 47da6b16e..eb62cfe4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -* Add linux/arm64 as a build platform for the official Docker image +* Add linux/arm64 platform support to the official Docker image +* Add `cat` command for downloading file contents directly to stdout ### Fixed * Emit `Using https://api.backblazeb2.com` message to stderr instead of stdout, therefor prevent JSON output corruption ### Changed * Stream `ls --json` JSON output instead of dumping it only after all objects have been fetched +* Alias `-` to stdout in `download-file-by-name` or `download-file-by-id` command ## [3.12.0] - 2023-10-28 @@ -61,7 +63,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Add s3 endpoint to `get-account-info` ### Deprecated -* Support of `-` as a valid filename in `upload-file` command. In future `-` will be an alias for standard input. +* Deprecate support of `-` as a valid filename in `upload-file` command. In the future `-` will always be interpreted as standard input ### Changed * Better help text for --corsRules diff --git a/b2/_cli/argcompleters.py b/b2/_cli/argcompleters.py index a4510a35e..c6fe94410 100644 --- a/b2/_cli/argcompleters.py +++ b/b2/_cli/argcompleters.py @@ -10,10 +10,10 @@ from functools import wraps from itertools import islice +from b2sdk.v2 import LIST_FILE_NAMES_MAX_LIMIT from b2sdk.v2.api import B2Api from b2._cli.b2api import _get_b2api_for_profile -from b2._cli.const import LIST_FILE_NAMES_MAX_LIMIT def _with_api(func): diff --git a/b2/_cli/const.py b/b2/_cli/const.py index 44d26f36b..64deb23e8 100644 --- a/b2/_cli/const.py +++ b/b2/_cli/const.py @@ -24,7 +24,4 @@ DEFAULT_THREADS = 10 # Constants used in the B2 API -# TODO B2-47 move API related constants to b2sdk CREATE_BUCKET_TYPES = ('allPublic', 'allPrivate') -DEFAULT_MIN_PART_SIZE = 5 * 1000 * 1000 # 5MB -LIST_FILE_NAMES_MAX_LIMIT = 10000 # https://www.backblaze.com/b2/docs/b2_list_file_names.html diff --git a/b2/_utils/filesystem.py b/b2/_utils/filesystem.py deleted file mode 100644 index 8e529bda6..000000000 --- a/b2/_utils/filesystem.py +++ /dev/null @@ -1,20 +0,0 @@ -###################################################################### -# -# File: b2/_utils/filesystem.py -# -# Copyright 2023 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### -import stat -from pathlib import Path - - -def points_to_fifo(path: Path) -> bool: - path = path.resolve() - try: - - return stat.S_ISFIFO(path.stat().st_mode) - except OSError: - return False diff --git a/b2/_utils/uri.py b/b2/_utils/uri.py new file mode 100644 index 000000000..537e3863a --- /dev/null +++ b/b2/_utils/uri.py @@ -0,0 +1,65 @@ +###################################################################### +# +# File: b2/_utils/uri.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from __future__ import annotations + +import dataclasses +import pathlib +import urllib +from pathlib import Path + + +class B2URIBase: + pass + + +@dataclasses.dataclass +class B2URI(B2URIBase): + bucket: str + path: str + + def __str__(self) -> str: + return f"b2://{self.bucket}{self.path}" + + +@dataclasses.dataclass +class B2FileIdURI(B2URIBase): + file_id: str + + def __str__(self) -> str: + return f"b2id://{self.file_id}" + + +def parse_uri(uri: str) -> Path | B2URI | B2FileIdURI: + parsed = urllib.parse.urlparse(uri) + if parsed.scheme == "": + return pathlib.Path(uri) + return _parse_b2_uri(uri, parsed) + + +def parse_b2_uri(uri: str) -> B2URI | B2FileIdURI: + parsed = urllib.parse.urlparse(uri) + return _parse_b2_uri(uri, parsed) + + +def _parse_b2_uri(uri, parsed: urllib.parse.ParseResult) -> B2URI | B2FileIdURI: + if parsed.scheme in ("b2", "b2id"): + if not parsed.netloc: + raise ValueError(f"Invalid B2 URI: {uri!r}") + elif parsed.password or parsed.username: + raise ValueError( + "Invalid B2 URI: credentials passed using `user@password:` syntax are not supported in URI" + ) + + if parsed.scheme == "b2": + return B2URI(bucket=parsed.netloc, path=parsed.path[1:]) + elif parsed.scheme == "b2id": + return B2FileIdURI(file_id=parsed.netloc) + else: + raise ValueError(f"Unsupported URI scheme: {parsed.scheme!r}") diff --git a/b2/arg_parser.py b/b2/arg_parser.py index 3f61ae3aa..9a0e0d4e4 100644 --- a/b2/arg_parser.py +++ b/b2/arg_parser.py @@ -9,6 +9,7 @@ ###################################################################### import argparse +import functools import locale import re import sys @@ -148,3 +149,18 @@ def parse_default_retention_period(s): 'default retention period must be in the form of "X days|years "' ) return RetentionPeriod(**{m.group('unit'): int(m.group('duration'))}) + + +def wrap_with_argument_type_error(func, translator=str, exc_type=ValueError): + """ + Wrap function that may raise an exception into a function that raises ArgumentTypeError error. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except exc_type as e: + raise argparse.ArgumentTypeError(translator(e)) + + return wrapper diff --git a/b2/console_tool.py b/b2/console_tool.py index c9f8d0eac..5c93bc9cc 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -48,11 +48,13 @@ B2_ACCOUNT_INFO_DEFAULT_FILE, B2_ACCOUNT_INFO_ENV_VAR, B2_ACCOUNT_INFO_PROFILE_FILE, + DEFAULT_MIN_PART_SIZE, DEFAULT_SCAN_MANAGER, NO_RETENTION_BUCKET_SETTING, REALM_URLS, SRC_LAST_MODIFIED_MILLIS, SSE_C_KEY_ID_FILE_INFO_KEY_NAME, + STDOUT_FILEPATH, UNKNOWN_KEY_ID, XDG_CONFIG_HOME_ENV_VAR, ApplicationKey, @@ -86,6 +88,7 @@ get_included_sources, make_progress_listener, parse_sync_folder, + points_to_fifo, ) from b2sdk.v2.exception import ( B2Error, @@ -116,18 +119,18 @@ B2_SOURCE_SSE_C_KEY_B64_ENV_VAR, B2_USER_AGENT_APPEND_ENV_VAR, CREATE_BUCKET_TYPES, - DEFAULT_MIN_PART_SIZE, DEFAULT_THREADS, ) from b2._cli.obj_loads import validated_loads from b2._cli.shell import detect_shell -from b2._utils.filesystem import points_to_fifo +from b2._utils.uri import B2URI, B2FileIdURI, B2URIBase, parse_b2_uri from b2.arg_parser import ( ArgumentParser, parse_comma_separated_list, parse_default_retention_period, parse_millis_from_float_timestamp, parse_range, + wrap_with_argument_type_error, ) from b2.json_encoder import B2CliJsonEncoder from b2.version import VERSION @@ -202,6 +205,9 @@ def local_path_to_b2_path(path): return path.replace(os.path.sep, '/') +B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri) + + def keyboard_interrupt_handler(signum, frame): raise KeyboardInterrupt() @@ -1412,18 +1418,27 @@ def _represent_legal_hold(cls, legal_hold: LegalHold): def _print_file_attribute(self, label, value): self._print((label + ':').ljust(20) + ' ' + value) + def get_local_output_filepath(self, filename: str) -> pathlib.Path: + if filename == '-': + return STDOUT_FILEPATH + return pathlib.Path(filename) + @B2.register_subcommand class DownloadFileById( - ThreadsMixin, ProgressMixin, SourceSseMixin, WriteBufferSizeMixin, SkipHashVerificationMixin, - MaxDownloadStreamsMixin, DownloadCommand + ThreadsMixin, + ProgressMixin, + SourceSseMixin, + WriteBufferSizeMixin, + SkipHashVerificationMixin, + MaxDownloadStreamsMixin, + DownloadCommand, ): """ Downloads the given file, and stores it in the given local file. {PROGRESSMIXIN} {THREADSMIXIN} - {THREADSMIXIN} {SOURCESSEMIXIN} {WRITEBUFFERSIZEMIXIN} {SKIPHASHVERIFICATIONMIXIN} @@ -1452,7 +1467,8 @@ def run(self, args): ) self._print_download_info(downloaded_file) - downloaded_file.save_to(args.localFileName) + output_filepath = self.get_local_output_filepath(args.localFileName) + downloaded_file.save_to(output_filepath) self._print('Download finished') return 0 @@ -1460,8 +1476,8 @@ def run(self, args): @B2.register_subcommand class DownloadFileByName( - ProgressMixin, ThreadsMixin, + ProgressMixin, SourceSseMixin, WriteBufferSizeMixin, SkipHashVerificationMixin, @@ -1503,12 +1519,67 @@ def run(self, args): ) self._print_download_info(downloaded_file) - downloaded_file.save_to(args.localFileName) + output_filepath = self.get_local_output_filepath(args.localFileName) + downloaded_file.save_to(output_filepath) self._print('Download finished') return 0 +@B2.register_subcommand +class Cat( + ProgressMixin, + SourceSseMixin, + WriteBufferSizeMixin, + SkipHashVerificationMixin, + DownloadCommand, +): + """ + Download content of a file identified by B2 URI directly to stdout. + + {PROGRESSMIXIN} + {SOURCESSEMIXIN} + {WRITEBUFFERSIZEMIXIN} + {SKIPHASHVERIFICATIONMIXIN} + + Requires capability: + + - **readFiles** + """ + + @classmethod + def _setup_parser(cls, parser): + parser.add_argument( + 'b2uri', + type=B2_URI_ARG_TYPE, + help= + "B2 URI identifying the file to print, e.g. b2://yourBucket/file.txt or b2id://fileId", + ) + super()._setup_parser(parser) + + def download_by_b2_uri( + self, b2_uri: B2URIBase, args: argparse.Namespace, local_filename: str + ) -> DownloadedFile: + progress_listener = make_progress_listener(local_filename, args.noProgress or args.quiet) + encryption_setting = self._get_source_sse_setting(args) + if isinstance(b2_uri, B2FileIdURI): + download = functools.partial(self.api.download_file_by_id, b2_uri.file_id) + elif isinstance(b2_uri, B2URI): + bucket = self.api.get_bucket_by_name(b2_uri.bucket) + download = functools.partial(bucket.download_file_by_name, b2_uri.path) + else: # This should never happen since there are no more subclasses of B2URIBase + raise ValueError(f'Unsupported B2 URI: {b2_uri!r}') + + return download(progress_listener=progress_listener, encryption=encryption_setting) + + def run(self, args): + super().run(args) + downloaded_file = self.download_by_b2_uri(args.b2uri, args, '-') + output_filepath = self.get_local_output_filepath('-') + downloaded_file.save_to(output_filepath) + return 0 + + @B2.register_subcommand class GetAccountInfo(Command): """ @@ -2913,7 +2984,7 @@ def get_input_stream(self, filename: str) -> 'str | int | io.BinaryIO': if filename == "-": if os.path.exists('-'): self._print_stderr( - "WARNING: Filename `-` won't be supported in the future and will be treated as stdin alias." + "WARNING: Filename `-` won't be supported in the future and will always be treated as stdin alias." ) else: return sys.stdin.buffer if platform.system() == "Windows" else sys.stdin.fileno() diff --git a/noxfile.py b/noxfile.py index 4b61124e6..23b3ba903 100644 --- a/noxfile.py +++ b/noxfile.py @@ -196,7 +196,7 @@ def run_integration_test(session, pytest_posargs): 'test/integration', '-s', '-n', - 'auto', + '2' if CI else 'auto', '--log-level', 'INFO', '-W', diff --git a/requirements.txt b/requirements.txt index c9f4b6b6d..68b52fc05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete>=2,<4 arrow>=1.0.2,<2.0.0 -b2sdk>=1.24.1,<2 +b2sdk>=1.25.0,<2 docutils>=0.18.1 idna~=3.4; platform_system == 'Java' importlib-metadata~=3.3; python_version < '3.8' diff --git a/test/integration/cleanup_buckets.py b/test/integration/cleanup_buckets.py index 45adf743a..51d216528 100644 --- a/test/integration/cleanup_buckets.py +++ b/test/integration/cleanup_buckets.py @@ -13,4 +13,4 @@ def test_cleanup_buckets(b2_api): # this is not a test, but it is intended to be called # via pytest because it reuses fixtures which have everything # set up - b2_api.clean_buckets() + pass # b2_api calls b2_api.clean_buckets() in its finalizer diff --git a/test/integration/conftest.py b/test/integration/conftest.py index ff336bdc8..c4b37b060 100755 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -7,8 +7,9 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations -import contextlib +import logging import os import pathlib import subprocess @@ -18,16 +19,38 @@ from tempfile import TemporaryDirectory import pytest -from b2sdk.exception import BadRequest, BucketIdNotFound -from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR +from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR, Bucket -from .helpers import Api, CommandLine, bucket_name_part +from .helpers import NODE_DESCRIPTION, RNG_SEED, Api, CommandLine, bucket_name_part, random_token + +logger = logging.getLogger(__name__) GENERAL_BUCKET_NAME_PREFIX = 'clitst' TEMPDIR = tempfile.gettempdir() ROOT_PATH = pathlib.Path(__file__).parent.parent.parent +@pytest.fixture(scope='session', autouse=True) +def summary_notes(request, worker_id): + capmanager = request.config.pluginmanager.getplugin("capturemanager") + with capmanager.global_and_fixture_disabled(): + log_handler = logging.StreamHandler(sys.stderr) + log_fmt = logging.Formatter(f'{worker_id} %(asctime)s %(levelname).1s %(message)s') + log_handler.setFormatter(log_fmt) + logger.addHandler(log_handler) + + class Notes: + def append(self, note): + logger.info(note) + + return Notes() + + +@pytest.fixture(scope='session', autouse=True) +def node_stats(summary_notes): + summary_notes.append(f"NODE={NODE_DESCRIPTION} seed={RNG_SEED}") + + @pytest.hookimpl def pytest_addoption(parser): parser.addoption( @@ -64,19 +87,19 @@ def realm() -> str: yield environ.get('B2_TEST_ENVIRONMENT', 'production') -@pytest.fixture(scope='function') -def bucket(b2_api) -> str: - try: - bucket = b2_api.create_bucket() - except BadRequest as e: - if e.code != 'too_many_buckets': - raise - num_buckets = b2_api.count_and_print_buckets() - print('current number of buckets:', num_buckets) - raise - yield bucket - with contextlib.suppress(BucketIdNotFound): - b2_api.clean_bucket(bucket) +@pytest.fixture +def bucket(bucket_factory) -> Bucket: + return bucket_factory() + + +@pytest.fixture +def bucket_factory(b2_api, schedule_bucket_cleanup): + def create_bucket(**kwargs): + new_bucket = b2_api.create_bucket(**kwargs) + schedule_bucket_cleanup(new_bucket.name, new_bucket.bucket_dict) + return new_bucket + + yield create_bucket @pytest.fixture(scope='function') @@ -86,7 +109,7 @@ def bucket_name(bucket) -> str: @pytest.fixture(scope='function') def file_name(bucket) -> str: - file_ = bucket.upload_bytes(b'test_file', f'{bucket_name_part(8)}.txt') + file_ = bucket.upload_bytes(b'test_file', f'{random_token(8)}.txt') yield file_.file_name @@ -111,44 +134,55 @@ def this_run_bucket_name_prefix() -> str: yield GENERAL_BUCKET_NAME_PREFIX + bucket_name_part(8) -@pytest.fixture(scope='module') -def monkey_patch(): - """ Module-scope monkeypatching (original `monkeypatch` is function-scope) """ - from _pytest.monkeypatch import MonkeyPatch - monkey = MonkeyPatch() - yield monkey - monkey.undo() +@pytest.fixture(scope='session') +def monkeysession(): + with pytest.MonkeyPatch.context() as mp: + yield mp -@pytest.fixture(scope='module', autouse=True) -def auto_change_account_info_dir(monkey_patch) -> dir: +@pytest.fixture(scope='session', autouse=True) +def auto_change_account_info_dir(monkeysession) -> dir: """ - Automatically for the whole module testing: + Automatically for the whole testing: 1) temporary remove B2_APPLICATION_KEY and B2_APPLICATION_KEY_ID from environment 2) create a temporary directory for storing account info database """ - monkey_patch.delenv('B2_APPLICATION_KEY_ID', raising=False) - monkey_patch.delenv('B2_APPLICATION_KEY', raising=False) + monkeysession.delenv('B2_APPLICATION_KEY_ID', raising=False) + monkeysession.delenv('B2_APPLICATION_KEY', raising=False) # make b2sdk use temp dir for storing default & per-profile account information with TemporaryDirectory() as temp_dir: - monkey_patch.setenv(B2_ACCOUNT_INFO_ENV_VAR, path.join(temp_dir, '.b2_account_info')) - monkey_patch.setenv(XDG_CONFIG_HOME_ENV_VAR, temp_dir) + monkeysession.setenv(B2_ACCOUNT_INFO_ENV_VAR, path.join(temp_dir, '.b2_account_info')) + monkeysession.setenv(XDG_CONFIG_HOME_ENV_VAR, temp_dir) yield temp_dir -@pytest.fixture(scope='module') -def b2_api(application_key_id, application_key, realm, this_run_bucket_name_prefix) -> Api: - yield Api( - application_key_id, application_key, realm, GENERAL_BUCKET_NAME_PREFIX, - this_run_bucket_name_prefix +@pytest.fixture(scope='session') +def b2_api( + application_key_id, + application_key, + realm, + this_run_bucket_name_prefix, + auto_change_account_info_dir, + summary_notes, +) -> Api: + api = Api( + application_key_id, + application_key, + realm, + general_bucket_name_prefix=GENERAL_BUCKET_NAME_PREFIX, + this_run_bucket_name_prefix=this_run_bucket_name_prefix, ) + yield api + api.clean_buckets() + summary_notes.append(f"Buckets names used during this tests: {api.bucket_name_log!r}") @pytest.fixture(scope='module') def global_b2_tool( - request, application_key_id, application_key, realm, this_run_bucket_name_prefix + request, application_key_id, application_key, realm, this_run_bucket_name_prefix, b2_api, + auto_change_account_info_dir ) -> CommandLine: tool = CommandLine( request.config.getoption('--sut'), @@ -157,9 +191,10 @@ def global_b2_tool( realm, this_run_bucket_name_prefix, request.config.getoption('--env-file-cmd-placeholder'), + api_wrapper=b2_api, ) tool.reauthorize(check_key_capabilities=True) # reauthorize for the first time (with check) - return tool + yield tool @pytest.fixture(scope='function') @@ -169,13 +204,38 @@ def b2_tool(global_b2_tool): return global_b2_tool +@pytest.fixture +def schedule_bucket_cleanup(global_b2_tool): + """ + Explicitly ask for buckets cleanup after the test + + This should be only used when testing `create-bucket` command; otherwise use `bucket_factory` fixture. + """ + buckets_to_clean = {} + + def add_bucket_to_cleanup(bucket_name, bucket_dict: dict | None = None): + buckets_to_clean[bucket_name] = bucket_dict + + yield add_bucket_to_cleanup + if buckets_to_clean: + global_b2_tool.reauthorize( + check_key_capabilities=False + ) # test may have mangled authorization + global_b2_tool.cleanup_buckets(buckets_to_clean) + + @pytest.fixture(autouse=True, scope='session') -def sample_file(): +def sample_filepath(): """Copy the README.md file to /tmp so that docker tests can access it""" - tmp_readme = pathlib.Path(f'{TEMPDIR}/README.md') + tmp_readme = pathlib.Path(TEMPDIR) / 'README.md' if not tmp_readme.exists(): tmp_readme.write_text((ROOT_PATH / 'README.md').read_text()) - return str(tmp_readme) + return tmp_readme + + +@pytest.fixture(autouse=True, scope='session') +def sample_file(sample_filepath): + return str(sample_filepath) @pytest.fixture(scope='session') @@ -218,12 +278,12 @@ def b2_in_path(tmp_path_factory): @pytest.fixture(scope="module") -def env(b2_in_path, homedir, monkey_patch, is_running_on_docker): +def env(b2_in_path, homedir, monkeysession, is_running_on_docker): """Get ENV for running b2 command from shell level.""" if not is_running_on_docker: - monkey_patch.setenv('PATH', b2_in_path) - monkey_patch.setenv('HOME', str(homedir)) - monkey_patch.setenv('SHELL', "/bin/bash") # fix for running under github actions + monkeysession.setenv('PATH', b2_in_path) + monkeysession.setenv('HOME', str(homedir)) + monkeysession.setenv('SHELL', "/bin/bash") # fix for running under github actions yield os.environ diff --git a/test/integration/helpers.py b/test/integration/helpers.py index 3d6893855..264f15045 100755 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -7,6 +7,9 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations + +import dataclasses import json import logging import os @@ -14,24 +17,26 @@ import platform import random import re +import secrets import shutil import string import subprocess import sys import threading import time +import warnings from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from os import environ, linesep, path from pathlib import Path from tempfile import gettempdir, mkdtemp, mktemp -from typing import List, Optional, Union from unittest.mock import MagicMock import backoff -from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound from b2sdk.v2 import ( ALL_CAPABILITIES, + BUCKET_NAME_CHARS_UNIQ, + BUCKET_NAME_LENGTH_RANGE, NO_RETENTION_FILE_SETTING, B2Api, Bucket, @@ -47,63 +52,80 @@ fix_windows_path_limit, ) from b2sdk.v2.exception import ( + BadRequest, BucketIdNotFound, - DuplicateBucketName, FileNotPresent, TooManyRequests, + v3BucketIdNotFound, ) from b2.console_tool import Command, current_time_millis logger = logging.getLogger(__name__) -BUCKET_CLEANUP_PERIOD_MILLIS = 0 +# A large period is set here to avoid issues related to clock skew or other time-related issues under CI +BUCKET_CLEANUP_PERIOD_MILLIS = timedelta(days=1).total_seconds() * 1000 ONE_HOUR_MILLIS = 60 * 60 * 1000 ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24 -BUCKET_NAME_LENGTH = 50 -BUCKET_NAME_CHARS = string.ascii_letters + string.digits + '-' +BUCKET_NAME_LENGTH = BUCKET_NAME_LENGTH_RANGE[1] BUCKET_CREATED_AT_MILLIS = 'created_at_millis' +NODE_DESCRIPTION = f"{platform.node()}: {platform.platform()}" + + +def get_seed(): + """ + Get seed for random number generator. + + GH Actions machines seem to offer a very limited entropy pool + """ + return b''.join( + ( + secrets.token_bytes(32), + str(time.time_ns()).encode(), + NODE_DESCRIPTION.encode(), + str(os.getpid()).encode(), # needed due to pytest-xdist + str(environ).encode('utf8', errors='ignore' + ), # especially helpful under GitHub (and similar) CI + ) + ) + + +RNG = random.Random(get_seed()) +RNG_SEED = RNG.randint(0, 2 << 31) +RNG_COUNTER = 0 + +if sys.version_info < (3, 9): + RNG.randbytes = lambda n: RNG.getrandbits(n * 8).to_bytes(n, 'little') + SSE_NONE = EncryptionSetting(mode=EncryptionMode.NONE,) SSE_B2_AES = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) +_SSE_KEY = RNG.randbytes(32) SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, - key=EncryptionKey(secret=os.urandom(32), key_id='user-generated-key-id') + key=EncryptionKey(secret=_SSE_KEY, key_id='user-generated-key-id') ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, - key=EncryptionKey(secret=os.urandom(32), key_id='another-user-generated-key-id') -) - -RNG_SEED = '_'.join( - [ - os.getenv('GITHUB_REPOSITORY', ''), - os.getenv('GITHUB_SHA', ''), - os.getenv('GITHUB_RUN_ID', ''), - os.getenv('GITHUB_RUN_ATTEMPT', ''), - os.getenv('GITHUB_JOB', ''), - os.getenv('GITHUB_ACTION', ''), - str(os.getpid()), # for local runs with xdist - str(time.time()), - ] + key=EncryptionKey(secret=_SSE_KEY, key_id='another-user-generated-key-id') ) -RNG = random.Random(RNG_SEED) -RNG_COUNTER = 0 +def random_token(length: int, chars=string.ascii_letters) -> str: + return ''.join(RNG.choice(chars) for _ in range(length)) def bucket_name_part(length: int) -> str: assert length >= 1 global RNG_COUNTER RNG_COUNTER += 1 - name_part = ''.join(RNG.choice(BUCKET_NAME_CHARS) for _ in range(length)) + name_part = random_token(length, BUCKET_NAME_CHARS_UNIQ) logger.info('RNG_SEED: %s', RNG_SEED) logger.info('RNG_COUNTER: %i, length: %i', RNG_COUNTER, length) logger.info('name_part: %s', name_part) @@ -119,6 +141,7 @@ class Api: this_run_bucket_name_prefix: str api: B2Api = None + bucket_name_log: list[str] = dataclasses.field(default_factory=list) def __post_init__(self): info = InMemoryAccountInfo() @@ -129,55 +152,61 @@ def __post_init__(self): self.this_run_bucket_name_prefix ) > 5, self.this_run_bucket_name_prefix - def create_bucket(self) -> Bucket: - for _ in range(10): - bucket_name = self.this_run_bucket_name_prefix + bucket_name_part( - BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix) - ) - print('Creating bucket:', bucket_name) - try: - return self.api.create_bucket( - bucket_name, - 'allPublic', - bucket_info={BUCKET_CREATED_AT_MILLIS: str(current_time_millis())}, - ) - except DuplicateBucketName: - pass - print() - - raise ValueError('Failed to create bucket due to name collision') + def new_bucket_name(self) -> str: + bucket_name = self.this_run_bucket_name_prefix + bucket_name_part( + BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix) + ) + self.bucket_name_log.append(bucket_name) + return bucket_name + + def new_bucket_info(self) -> dict: + return { + BUCKET_CREATED_AT_MILLIS: str(current_time_millis()), + "created_by": NODE_DESCRIPTION, + } + + def create_bucket(self, bucket_type: str = 'allPublic', **kwargs) -> Bucket: + bucket_name = self.new_bucket_name() + return self.api.create_bucket( + bucket_name, + bucket_type=bucket_type, + bucket_info=self.new_bucket_info(), + **kwargs, + ) - def _should_remove_bucket(self, bucket: Bucket): + def _should_remove_bucket(self, bucket: Bucket) -> tuple[bool, str]: if bucket.name.startswith(self.this_run_bucket_name_prefix): return True, 'it is a bucket for this very run' - OLD_PATTERN = 'test-b2-cli-' - if bucket.name.startswith(self.general_bucket_name_prefix) or bucket.name.startswith(OLD_PATTERN): # yapf: disable + if bucket.name.startswith(self.general_bucket_name_prefix): if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info: delete_older_than = current_time_millis() - BUCKET_CLEANUP_PERIOD_MILLIS - this_bucket_creation_time = bucket.bucket_info[BUCKET_CREATED_AT_MILLIS] - if int(this_bucket_creation_time) < delete_older_than: + this_bucket_creation_time = int(bucket.bucket_info[BUCKET_CREATED_AT_MILLIS]) + if this_bucket_creation_time < delete_older_than: return True, f"this_bucket_creation_time={this_bucket_creation_time} < delete_older_than={delete_older_than}" + return False, f"this_bucket_creation_time={this_bucket_creation_time} >= delete_older_than={delete_older_than}" else: return True, 'undefined ' + BUCKET_CREATED_AT_MILLIS - return False, '' + return False, f'does not start with {self.general_bucket_name_prefix!r}' - def clean_buckets(self): - buckets = self.api.list_buckets() + def clean_buckets(self, quick=False): + # even with use_cache=True, if cache is empty API call will be made + buckets = self.api.list_buckets(use_cache=quick) print('Total bucket count:', len(buckets)) + remaining_buckets = [] for bucket in buckets: should_remove, why = self._should_remove_bucket(bucket) if not should_remove: - print(f'Skipping bucket removal: "{bucket.name}"') + print(f'Skipping bucket removal {bucket.name!r} because {why}') + remaining_buckets.append(bucket) continue print('Trying to remove bucket:', bucket.name, 'because', why) try: self.clean_bucket(bucket) - except (BucketIdNotFound, v3BucketIdNotFound): + except BucketIdNotFound: print(f'It seems that bucket {bucket.name} has already been removed') - buckets = self.api.list_buckets() - print('Total bucket count after cleanup:', len(buckets)) - for bucket in buckets: + print('Total bucket count after cleanup:', len(remaining_buckets)) + for bucket in remaining_buckets: print(bucket) @backoff.on_exception( @@ -185,10 +214,18 @@ def clean_buckets(self): TooManyRequests, max_tries=8, ) - def clean_bucket(self, bucket: Union[Bucket, str]): + def clean_bucket(self, bucket: Bucket | str): if isinstance(bucket, str): bucket = self.api.get_bucket_by_name(bucket) + # try optimistic bucket removal first, since it is completely free (as opposed to `ls` call) + try: + return self.api.delete_bucket(bucket) + except (BucketIdNotFound, v3BucketIdNotFound): + return # bucket was already removed + except BadRequest as exc: + assert exc.code == 'cannot_delete_non_empty_bucket' + files_leftover = False file_versions = bucket.ls(latest_only=False, recursive=True) @@ -352,8 +389,14 @@ class CommandLine: ] def __init__( - self, command, account_id, application_key, realm, bucket_name_prefix, - env_file_cmd_placeholder + self, + command, + account_id, + application_key, + realm, + bucket_name_prefix, + env_file_cmd_placeholder, + api_wrapper: Api, ): self.command = command self.account_id = account_id @@ -362,14 +405,15 @@ def __init__( self.bucket_name_prefix = bucket_name_prefix self.env_file_cmd_placeholder = env_file_cmd_placeholder self.env_var_test_context = EnvVarTestContext(SqliteAccountInfo().filename) - self.account_info_file_name = SqliteAccountInfo().filename + self.api_wrapper = api_wrapper def generate_bucket_name(self): - return self.bucket_name_prefix + bucket_name_part( - BUCKET_NAME_LENGTH - len(self.bucket_name_prefix) - ) + return self.api_wrapper.new_bucket_name() - def run_command(self, args, additional_env: Optional[dict] = None): + def get_bucket_info_args(self) -> tuple[str, str]: + return '--bucketInfo', json.dumps(self.api_wrapper.new_bucket_info()) + + def run_command(self, args, additional_env: dict | None = None): """ Runs the command with the given arguments, returns a tuple in form of (succeeded, stdout) @@ -379,9 +423,9 @@ def run_command(self, args, additional_env: Optional[dict] = None): def should_succeed( self, - args: Optional[List[str]], - expected_pattern: Optional[str] = None, - additional_env: Optional[dict] = None, + args: list[str] | None, + expected_pattern: str | None = None, + additional_env: dict | None = None, ) -> str: """ Runs the command-line with the given arguments. Raises an exception @@ -396,12 +440,6 @@ def should_succeed( assert any(p.match(line) for p in self.EXPECTED_STDERR_PATTERNS), \ f'Unexpected stderr line: {repr(line)}' - if platform.python_implementation().lower() == 'pypy': - # TODO: remove after pypy removes the leftover print and resolve - # https://github.com/Backblaze/B2_Command_Line_Tool/issues/936 - while stdout.startswith('/'): - stdout = stdout.split('\n', 1)[-1] - if expected_pattern is not None: assert re.search(expected_pattern, stdout), \ f'did not match pattern="{expected_pattern}", stdout="{stdout}"' @@ -409,7 +447,7 @@ def should_succeed( return stdout @classmethod - def prepare_env(self, additional_env: Optional[dict] = None): + def prepare_env(self, additional_env: dict | None = None): environ['PYTHONPATH'] = '.' environ['PYTHONIOENCODING'] = 'utf-8' env = environ.copy() @@ -436,8 +474,8 @@ def parse_command(self, env): def execute( self, - args: Optional[List[Union[str, Path, int]]] = None, - additional_env: Optional[dict] = None, + args: list[str | Path | int] | None = None, + additional_env: dict | None = None, ): """ :param cmd: a command to run @@ -450,7 +488,7 @@ def execute( env = self.prepare_env(additional_env) command = self.parse_command(env) - args: List[str] = [str(arg) for arg in args] if args else [] + args: list[str] = [str(arg) for arg in args] if args else [] command.extend(args) print('Running:', ' '.join(command)) @@ -481,7 +519,7 @@ def execute( print_output(p.returncode, stdout_decoded, stderr_decoded) return p.returncode, stdout_decoded, stderr_decoded - def should_succeed_json(self, args, additional_env: Optional[dict] = None): + def should_succeed_json(self, args, additional_env: dict | None = None): """ Runs the command-line with the given arguments. Raises an exception if there was an error; otherwise, treats the stdout as JSON and returns @@ -494,7 +532,7 @@ def should_succeed_json(self, args, additional_env: Optional[dict] = None): raise ValueError(f'{result} is not a valid json') return loaded_result - def should_fail(self, args, expected_pattern, additional_env: Optional[dict] = None): + def should_fail(self, args, expected_pattern, additional_env: dict | None = None): """ Runs the command-line with the given args, expecting the given pattern to appear in stderr. @@ -502,12 +540,6 @@ def should_fail(self, args, expected_pattern, additional_env: Optional[dict] = N status, stdout, stderr = self.execute(args, additional_env) assert status != 0, 'ERROR: should have failed' - if platform.python_implementation().lower() == 'pypy': - # TODO: remove after pypy removes the leftover print and resolve - # https://github.com/Backblaze/B2_Command_Line_Tool/issues/936 - while stdout.startswith('/'): - stdout = stdout.split('\n', 1)[-1] - assert re.search(expected_pattern, stdout + stderr), \ f'did not match pattern="{expected_pattern}", stdout="{stdout}", stderr="{stderr}"' @@ -530,16 +562,40 @@ def reauthorize(self, check_key_capabilities=False): def list_file_versions(self, bucket_name): return self.should_succeed_json(['ls', '--json', '--recursive', '--versions', bucket_name]) + def cleanup_buckets(self, buckets: dict[str, dict | None]) -> None: + for bucket_name, bucket_dict in buckets.items(): + self.cleanup_bucket(bucket_name, bucket_dict) + + def cleanup_bucket(self, bucket_name: str, bucket_dict: dict | None = None) -> None: + """ + Cleanup bucket + + Since bucket was being handled by the tool, it is safe to assume it is cached in its cache and we don't + need to call C class API list_buckets endpoint to get it. + """ + if not bucket_dict: + try: + bucket_dict = self.should_succeed_json(['get-bucket', bucket_name]) + except (ValueError, AssertionError): # bucket doesn't exist + return + + bucket = self.api_wrapper.api.BUCKET_CLASS( + api=self.api_wrapper.api, + id_=bucket_dict['bucketId'], + name=bucket_name, + ) + self.api_wrapper.clean_bucket(bucket) + class TempDir: def __init__(self): + warnings.warn( + 'TempDir is deprecated; use pytest tmp_path fixture instead', + DeprecationWarning, + stacklevel=2, + ) self.dirpath = None - def get_dir(self): - assert self.dirpath is not None, \ - "can't call get_dir() before entering the context manager" - return self.dirpath - def __enter__(self): self.dirpath = mkdtemp() return Path(self.dirpath) @@ -548,31 +604,23 @@ def __exit__(self, exc_type, exc_val, exc_tb): shutil.rmtree(fix_windows_path_limit(self.dirpath)) -def read_file(path: Union[str, Path]): - if isinstance(path, Path): - path = str(path) +def read_file(path: str | Path): with open(path, 'rb') as f: return f.read() -def write_file(path: Union[str, Path], contents: bytes): - if isinstance(path, Path): - path = str(path) +def write_file(path: str | Path, contents: bytes): with open(path, 'wb') as f: f.write(contents) -def file_mod_time_millis(path: Union[str, Path]): - if isinstance(path, Path): - path = str(path) +def file_mod_time_millis(path: str | Path) -> int: return int(os.path.getmtime(path) * 1000) -def set_file_mod_time_millis(path: Union[str, Path], time): - if isinstance(path, Path): - path = str(path) +def set_file_mod_time_millis(path: str | Path, time): os.utime(path, (os.path.getatime(path), time / 1000)) def random_hex(length): - return ''.join(random.choice('0123456789abcdef') for _ in range(length)) + return ''.join(RNG.choice('0123456789abcdef') for _ in range(length)) diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index ea458203d..2cb24c8f7 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -8,8 +8,10 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations import base64 +import contextlib import hashlib import itertools import json @@ -19,7 +21,6 @@ import sys import time from pathlib import Path -from typing import Optional, Tuple import pytest from b2sdk.v2 import ( @@ -38,7 +39,6 @@ from ..helpers import skip_on_windows from .helpers import ( - BUCKET_CREATED_AT_MILLIS, ONE_DAY_MILLIS, ONE_HOUR_MILLIS, SSE_B2_AES, @@ -55,27 +55,33 @@ ) -def get_bucketinfo() -> Tuple[str, str]: - return '--bucketInfo', json.dumps({BUCKET_CREATED_AT_MILLIS: str(current_time_millis())}), +@pytest.fixture +def uploaded_sample_file(b2_tool, bucket_name, sample_filepath): + return b2_tool.should_succeed_json( + ['upload-file', '--quiet', bucket_name, + str(sample_filepath), 'sample_file'] + ) -def test_download(b2_tool, bucket_name, sample_file): +def test_download(b2_tool, bucket_name, sample_filepath, uploaded_sample_file, tmp_path): + output_a = tmp_path / 'a' + b2_tool.should_succeed( + [ + 'download-file-by-name', '--quiet', bucket_name, uploaded_sample_file['fileName'], + str(output_a) + ] + ) + assert output_a.read_text() == sample_filepath.read_text() - uploaded_a = b2_tool.should_succeed_json( - ['upload-file', '--quiet', bucket_name, sample_file, 'a'] + output_b = tmp_path / 'b' + b2_tool.should_succeed( + ['download-file-by-id', '--quiet', uploaded_sample_file['fileId'], + str(output_b)] ) - with TempDir() as dir_path: - b2_tool.should_succeed( - ['download-file-by-name', '--quiet', bucket_name, 'a', dir_path / 'a'] - ) - assert read_file(dir_path / 'a') == read_file(sample_file) - b2_tool.should_succeed( - ['download-file-by-id', '--quiet', uploaded_a['fileId'], dir_path / 'b'] - ) - assert read_file(dir_path / 'b') == read_file(sample_file) + assert output_b.read_text() == sample_filepath.read_text() -def test_basic(b2_tool, bucket_name, sample_file, is_running_on_docker): +def test_basic(b2_tool, bucket_name, sample_file, tmp_path): file_mod_time_str = str(file_mod_time_millis(sample_file)) @@ -121,13 +127,9 @@ def test_basic(b2_tool, bucket_name, sample_file, is_running_on_docker): should_equal(['rm1'], [f['fileName'] for f in list_of_files]) b2_tool.should_succeed(['rm', '--recursive', '--withWildcard', bucket_name, 'rm1']) - with TempDir() as dir_path: - b2_tool.should_succeed( - [ - 'download-file-by-name', '--noProgress', '--quiet', bucket_name, 'b/1', - dir_path / 'a' - ] - ) + b2_tool.should_succeed( + ['download-file-by-name', '--noProgress', '--quiet', bucket_name, 'b/1', tmp_path / 'a'] + ) b2_tool.should_succeed(['hide-file', bucket_name, 'c']) @@ -194,13 +196,16 @@ def test_basic(b2_tool, bucket_name, sample_file, is_running_on_docker): 'any-file-name', ), ) # \r? is for Windows, as $ doesn't match \r\n + + +def test_debug_logs(b2_tool, is_running_on_docker, tmp_path): to_be_removed_bucket_name = b2_tool.generate_bucket_name() b2_tool.should_succeed( [ 'create-bucket', to_be_removed_bucket_name, 'allPublic', - *get_bucketinfo(), + *b2_tool.get_bucket_info_args(), ], ) b2_tool.should_succeed(['delete-bucket', to_be_removed_bucket_name],) @@ -251,7 +256,10 @@ def test_bucket(b2_tool, bucket_name): "fileNamePrefix": "" }""" output = b2_tool.should_succeed_json( - ['update-bucket', '--lifecycleRule', rule, bucket_name, 'allPublic', *get_bucketinfo()], + [ + 'update-bucket', '--lifecycleRule', rule, bucket_name, 'allPublic', + *b2_tool.get_bucket_info_args() + ], ) ########## // doesn't happen on production, but messes up some tests \\ ########## @@ -270,10 +278,7 @@ def test_bucket(b2_tool, bucket_name): ] -def test_key_restrictions(b2_api, b2_tool, bucket_name, sample_file): - - second_bucket_name = b2_tool.generate_bucket_name() - b2_tool.should_succeed(['create-bucket', second_bucket_name, 'allPublic', *get_bucketinfo()],) +def test_key_restrictions(b2_tool, bucket_name, sample_file, bucket_factory): # A single file for rm to fail on. b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, sample_file, 'test']) @@ -292,6 +297,7 @@ def test_key_restrictions(b2_api, b2_tool, bucket_name, sample_file): ) b2_tool.should_succeed(['get-bucket', bucket_name],) + second_bucket_name = bucket_factory().name b2_tool.should_succeed(['get-bucket', second_bucket_name],) key_two_name = 'clt-testKey-02' + random_hex(6) @@ -336,20 +342,27 @@ def test_key_restrictions(b2_api, b2_tool, bucket_name, sample_file): b2_tool.application_key ] ) - b2_api.clean_bucket(second_bucket_name) b2_tool.should_succeed(['delete-key', key_one_id]) b2_tool.should_succeed(['delete-key', key_two_id]) -def test_account(b2_tool, bucket_name): - # actually a high level operations test - we run bucket tests here since this test doesn't use it +def test_delete_bucket(b2_tool, bucket_name): b2_tool.should_succeed(['delete-bucket', bucket_name]) + b2_tool.should_fail( + ['delete-bucket', bucket_name], re.compile(r'^ERROR: Bucket with id=\w* not found\s*$') + ) + + +def test_rapid_bucket_operations(b2_tool): new_bucket_name = b2_tool.generate_bucket_name() + bucket_info_args = b2_tool.get_bucket_info_args() # apparently server behaves erratically when we delete a bucket and recreate it right away - b2_tool.should_succeed(['create-bucket', new_bucket_name, 'allPrivate', *get_bucketinfo()]) + b2_tool.should_succeed(['create-bucket', new_bucket_name, 'allPrivate', *bucket_info_args]) b2_tool.should_succeed(['update-bucket', new_bucket_name, 'allPublic']) b2_tool.should_succeed(['delete-bucket', new_bucket_name]) + +def test_account(b2_tool): with b2_tool.env_var_test_context: b2_tool.should_succeed(['clear-account']) bad_application_key = random_hex(len(b2_tool.application_key)) @@ -387,7 +400,9 @@ def test_account(b2_tool, bucket_name): os.environ['B2_ENVIRONMENT'] = b2_tool.realm bucket_name = b2_tool.generate_bucket_name() - b2_tool.should_succeed(['create-bucket', bucket_name, 'allPrivate', *get_bucketinfo()]) + b2_tool.should_succeed( + ['create-bucket', bucket_name, 'allPrivate', *b2_tool.get_bucket_info_args()] + ) b2_tool.should_succeed(['delete-bucket', bucket_name]) assert os.path.exists(new_creds), 'sqlite file not created' @@ -826,13 +841,15 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryp ) -def test_sync_copy(b2_api, b2_tool, bucket_name, sample_file): - prepare_and_run_sync_copy_tests(b2_api, b2_tool, bucket_name, 'sync', sample_file=sample_file) +def test_sync_copy(bucket_factory, b2_tool, bucket_name, sample_file): + prepare_and_run_sync_copy_tests( + bucket_factory, b2_tool, bucket_name, 'sync', sample_file=sample_file + ) -def test_sync_copy_no_prefix_default_encryption(b2_api, b2_tool, bucket_name, sample_file): +def test_sync_copy_no_prefix_default_encryption(bucket_factory, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, '', @@ -842,9 +859,9 @@ def test_sync_copy_no_prefix_default_encryption(b2_api, b2_tool, bucket_name, sa ) -def test_sync_copy_no_prefix_no_encryption(b2_api, b2_tool, bucket_name, sample_file): +def test_sync_copy_no_prefix_no_encryption(bucket_factory, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, '', @@ -854,9 +871,9 @@ def test_sync_copy_no_prefix_no_encryption(b2_api, b2_tool, bucket_name, sample_ ) -def test_sync_copy_no_prefix_sse_b2(b2_api, b2_tool, bucket_name, sample_file): +def test_sync_copy_no_prefix_sse_b2(bucket_factory, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, '', @@ -866,9 +883,9 @@ def test_sync_copy_no_prefix_sse_b2(b2_api, b2_tool, bucket_name, sample_file): ) -def test_sync_copy_no_prefix_sse_c(b2_api, b2_tool, bucket_name, sample_file): +def test_sync_copy_no_prefix_sse_c(bucket_factory, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, '', @@ -912,7 +929,7 @@ def test_sync_copy_sse_c_single_bucket(b2_tool, bucket_name, sample_file): def prepare_and_run_sync_copy_tests( - b2_api, + bucket_factory, b2_tool, bucket_name, folder_in_bucket, @@ -928,10 +945,7 @@ def prepare_and_run_sync_copy_tests( else: b2_file_prefix = '' - other_bucket_name = b2_tool.generate_bucket_name() - success, _ = b2_tool.run_command( - ['create-bucket', other_bucket_name, 'allPublic', *get_bucketinfo()] - ) + other_bucket_name = bucket_factory().name other_b2_sync_point = 'b2:%s' % other_bucket_name if folder_in_bucket: @@ -967,8 +981,6 @@ def prepare_and_run_sync_copy_tests( file_version_summary_with_encryption(file_versions), ) - b2_api.clean_bucket(other_bucket_name) - def run_sync_copy_with_basic_checks( b2_tool, @@ -1085,7 +1097,7 @@ def test_sync_long_path(b2_tool, bucket_name): should_equal(['+ ' + long_path], file_version_summary(file_versions)) -def test_default_sse_b2(b2_api, b2_tool, bucket_name): +def test_default_sse_b2__update_bucket(b2_tool, bucket_name, schedule_bucket_cleanup): # Set default encryption via update-bucket bucket_info = b2_tool.should_succeed_json(['get-bucket', bucket_name]) bucket_default_sse = {'mode': 'none'} @@ -1107,15 +1119,18 @@ def test_default_sse_b2(b2_api, b2_tool, bucket_name): } should_equal(bucket_default_sse, bucket_info['defaultServerSideEncryption']) + +def test_default_sse_b2__create_bucket(b2_tool, schedule_bucket_cleanup): # Set default encryption via create-bucket second_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(second_bucket_name) b2_tool.should_succeed( [ 'create-bucket', '--defaultServerSideEncryption=SSE-B2', second_bucket_name, 'allPublic', - *get_bucketinfo(), + *b2_tool.get_bucket_info_args(), ] ) second_bucket_info = b2_tool.should_succeed_json(['get-bucket', second_bucket_name]) @@ -1124,10 +1139,9 @@ def test_default_sse_b2(b2_api, b2_tool, bucket_name): 'mode': 'SSE-B2', } should_equal(second_bucket_default_sse, second_bucket_info['defaultServerSideEncryption']) - b2_api.clean_bucket(second_bucket_name) -def test_sse_b2(b2_tool, bucket_name, sample_file): +def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path): b2_tool.should_succeed( [ 'upload-file', '--destinationServerSideEncryption=SSE-B2', '--quiet', bucket_name, @@ -1135,16 +1149,16 @@ def test_sse_b2(b2_tool, bucket_name, sample_file): ] ) b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'not_encrypted']) - with TempDir() as dir_path: - b2_tool.should_succeed( - ['download-file-by-name', '--quiet', bucket_name, 'encrypted', dir_path / 'encrypted'] - ) - b2_tool.should_succeed( - [ - 'download-file-by-name', '--quiet', bucket_name, 'not_encrypted', - dir_path / 'not_encypted' - ] - ) + + b2_tool.should_succeed( + ['download-file-by-name', '--quiet', bucket_name, 'encrypted', tmp_path / 'encrypted'] + ) + b2_tool.should_succeed( + [ + 'download-file-by-name', '--quiet', bucket_name, 'not_encrypted', + tmp_path / 'not_encypted' + ] + ) list_of_files = b2_tool.should_succeed_json(['ls', '--json', '--recursive', bucket_name]) should_equal( @@ -1194,7 +1208,7 @@ def test_sse_b2(b2_tool, bucket_name, sample_file): should_equal({'mode': 'none'}, file_info['serverSideEncryption']) -def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file): +def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path): sse_c_key_id = 'user-generated-key-id \nąóźćż\nœøΩ≈ç\nßäöü' if is_running_on_docker: @@ -1254,7 +1268,7 @@ def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file): expected_pattern='ERROR: Wrong or no SSE-C key provided when reading a file.', additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode()} ) - with TempDir() as dir_path: + with contextlib.nullcontext(tmp_path) as dir_path: b2_tool.should_succeed( [ 'download-file-by-name', @@ -1563,16 +1577,11 @@ def test_license(b2_tool, with_packages): SOFTWARE.""" in license_text.replace(os.linesep, '\n'), repr(license_text[-2000:]) -def test_file_lock(b2_tool, application_key_id, application_key, b2_api, sample_file): - lock_disabled_bucket_name = b2_tool.generate_bucket_name() - b2_tool.should_succeed( - [ - 'create-bucket', - lock_disabled_bucket_name, - 'allPrivate', - *get_bucketinfo(), - ], - ) +def test_file_lock( + b2_tool, application_key_id, application_key, sample_file, bucket_factory, + schedule_bucket_cleanup +): + lock_disabled_bucket_name = bucket_factory(bucket_type='allPrivate').name now_millis = current_time_millis() @@ -1616,13 +1625,14 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api, sample_ ], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' ) lock_enabled_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(lock_enabled_bucket_name) b2_tool.should_succeed( [ 'create-bucket', lock_enabled_bucket_name, 'allPrivate', '--fileLockEnabled', - *get_bucketinfo(), + *b2_tool.get_bucket_info_args(), ], ) updated_bucket = b2_tool.should_succeed_json( @@ -1852,17 +1862,6 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api, sample_ b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key, sample_file ) - # ---- perform test cleanup ---- - b2_tool.should_succeed( - ['authorize-account', '--environment', b2_tool.realm, application_key_id, application_key], - ) - buckets = [ - bucket for bucket in b2_api.api.list_buckets() - if bucket.name in {lock_enabled_bucket_name, lock_disabled_bucket_name} - ] - for bucket in buckets: - b2_api.clean_bucket(bucket) - def make_lock_disabled_key(b2_tool): key_name = 'no-perms-for-file-lock' + random_hex(6) @@ -2109,7 +2108,7 @@ def test_profile_switch(b2_tool): os.environ[B2_ACCOUNT_INFO_ENV_VAR] = B2_ACCOUNT_INFO -def test_replication_basic(b2_api, b2_tool, bucket_name): +def test_replication_basic(b2_tool, bucket_name, schedule_bucket_cleanup): key_one_name = 'clt-testKey-01' + random_hex(6) created_key_stdout = b2_tool.should_succeed( [ @@ -2193,6 +2192,7 @@ def test_replication_basic(b2_api, b2_tool, bucket_name): # create a source bucket and set up replication to destination bucket source_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(source_bucket_name) b2_tool.should_succeed( [ 'create-bucket', @@ -2200,7 +2200,7 @@ def test_replication_basic(b2_api, b2_tool, bucket_name): 'allPublic', '--replication', source_replication_configuration_json, - *get_bucketinfo(), + *b2_tool.get_bucket_info_args(), ] ) source_bucket = b2_tool.should_succeed_json(['get-bucket', source_bucket_name]) @@ -2258,18 +2258,18 @@ def test_replication_basic(b2_api, b2_tool, bucket_name): b2_tool.should_succeed(['delete-key', key_one_id]) b2_tool.should_succeed(['delete-key', key_two_id]) - b2_api.clean_bucket(source_bucket_name) -def test_replication_setup(b2_api, b2_tool, bucket_name): +def test_replication_setup(b2_tool, bucket_name, schedule_bucket_cleanup): source_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(source_bucket_name) b2_tool.should_succeed( [ 'create-bucket', source_bucket_name, 'allPublic', '--fileLockEnabled', - *get_bucketinfo(), + *b2_tool.get_bucket_info_args(), ] ) destination_bucket_name = bucket_name @@ -2314,13 +2314,12 @@ def test_replication_setup(b2_api, b2_tool, bucket_name): 'sourceToDestinationKeyMapping'].items(): b2_tool.should_succeed(['delete-key', key_one_id]) b2_tool.should_succeed(['delete-key', key_two_id]) - b2_api.clean_bucket(source_bucket_name) assert destination_bucket_old['replication']['asReplicationDestination'][ 'sourceToDestinationKeyMapping'] == destination_bucket['replication'][ 'asReplicationDestination']['sourceToDestinationKeyMapping'] -def test_replication_monitoring(b2_tool, bucket_name, b2_api, sample_file): +def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_bucket_cleanup): # ---------------- set up keys ---------------- key_one_name = 'clt-testKey-01' + random_hex(6) @@ -2400,6 +2399,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api, sample_file): # create a source bucket and set up replication to destination bucket source_bucket_name = b2_tool.generate_bucket_name() + schedule_bucket_cleanup(source_bucket_name) b2_tool.should_succeed( [ 'create-bucket', @@ -2408,7 +2408,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api, sample_file): '--fileLockEnabled', '--replication', source_replication_configuration_json, - *get_bucketinfo(), + *b2_tool.get_bucket_info_args(), ] ) @@ -2549,10 +2549,8 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api, sample_file): } for first, second in itertools.product(['FAILED', 'PENDING'], ['FAILED', 'PENDING']) ] - b2_api.clean_bucket(source_bucket_name) - -def test_enable_file_lock_first_retention_second(b2_tool, b2_api, bucket_name): +def test_enable_file_lock_first_retention_second(b2_tool, bucket_name): # enable file lock only b2_tool.should_succeed(['update-bucket', bucket_name, '--fileLockEnabled']) @@ -2567,10 +2565,8 @@ def test_enable_file_lock_first_retention_second(b2_tool, b2_api, bucket_name): # attempt to re-enable should be a noop b2_tool.should_succeed(['update-bucket', bucket_name, '--fileLockEnabled']) - b2_api.clean_bucket(bucket_name) - -def test_enable_file_lock_and_set_retention_at_once(b2_tool, b2_api, bucket_name): +def test_enable_file_lock_and_set_retention_at_once(b2_tool, bucket_name): # attempt setting retention without file lock enabled b2_tool.should_fail( [ @@ -2590,15 +2586,13 @@ def test_enable_file_lock_and_set_retention_at_once(b2_tool, b2_api, bucket_name # attempt to re-enable should be a noop b2_tool.should_succeed(['update-bucket', bucket_name, '--fileLockEnabled']) - b2_api.clean_bucket(bucket_name) - def _assert_file_lock_configuration( b2_tool, file_id, - retention_mode: Optional['RetentionMode'] = None, - retain_until: Optional[int] = None, - legal_hold: Optional[LegalHold] = None + retention_mode: RetentionMode | None = None, + retain_until: int | None = None, + legal_hold: LegalHold | None = None ): file_version = b2_tool.should_succeed_json(['get-file-info', file_id]) @@ -2619,7 +2613,7 @@ def _assert_file_lock_configuration( assert legal_hold == actual_legal_hold -def test_cut(b2_tool, bucket_name, sample_file): +def test_upload_file__custom_upload_time(b2_tool, bucket_name, sample_file): file_data = read_file(sample_file) cut = 12345 cut_printable = '1970-01-01 00:00:12' @@ -2677,3 +2671,22 @@ def test_upload_unbound_stream__redirect_operator( f'b2 upload-unbound-stream {bucket_name} <(echo -n {content}) {request.node.name}.txt' ) assert hashlib.sha1(content.encode()).hexdigest() in run.stdout + + +def test_download_file_stdout( + b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file +): + assert b2_tool.should_succeed( + ['download-file-by-name', '--quiet', bucket_name, uploaded_sample_file['fileName'], '-'], + ).replace("\r", "") == sample_filepath.read_text() + assert b2_tool.should_succeed( + ['download-file-by-id', '--quiet', uploaded_sample_file['fileId'], '-'], + ).replace("\r", "") == sample_filepath.read_text() + + +def test_cat(b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file): + assert b2_tool.should_succeed( + ['cat', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}"], + ).replace("\r", "") == sample_filepath.read_text() + assert b2_tool.should_succeed(['cat', f"b2id://{uploaded_sample_file['fileId']}" + ],).replace("\r", "") == sample_filepath.read_text() diff --git a/test/unit/_utils/test_filesystem.py b/test/unit/_utils/test_filesystem.py deleted file mode 100644 index 5e2738cd3..000000000 --- a/test/unit/_utils/test_filesystem.py +++ /dev/null @@ -1,32 +0,0 @@ -###################################################################### -# -# File: test/unit/_utils/test_filesystem.py -# -# Copyright 2023 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### -import os -from test.helpers import skip_on_windows - -from b2._utils.filesystem import points_to_fifo - - -def test_points_to_fifo__doesnt_exist(tmp_path): - non_existent = tmp_path / 'non-existent' - assert not non_existent.exists() - assert not points_to_fifo(non_existent) - - -@skip_on_windows -def test_points_to_fifo__named_pipe(tmp_path): - named_pipe = tmp_path / 'fifo' - os.mkfifo(str(named_pipe)) - assert points_to_fifo(named_pipe) - - -def test_points_to_fifo__regular_file(tmp_path): - regular_file = tmp_path / 'regular' - regular_file.write_text('hello') - assert not points_to_fifo(regular_file) diff --git a/test/unit/_utils/test_uri.py b/test/unit/_utils/test_uri.py new file mode 100644 index 000000000..ef34d8a33 --- /dev/null +++ b/test/unit/_utils/test_uri.py @@ -0,0 +1,62 @@ +###################################################################### +# +# File: test/unit/_utils/test_uri.py +# +# Copyright 2023 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from pathlib import Path + +import pytest + +from b2._utils.uri import B2URI, B2FileIdURI, parse_uri + + +def test_b2pathuri_str(): + uri = B2URI(bucket="testbucket", path="/path/to/file") + assert str(uri) == "b2://testbucket/path/to/file" + + +def test_b2fileuri_str(): + uri = B2FileIdURI(file_id="file123") + assert str(uri) == "b2id://file123" + + +@pytest.mark.parametrize( + "uri,expected", + [ + ("some/local/path", Path("some/local/path")), + ("./some/local/path", Path("some/local/path")), + ("b2://bucket/path/to/dir/", B2URI(bucket="bucket", path="path/to/dir/")), + ("b2id://file123", B2FileIdURI(file_id="file123")), + ], +) +def test_parse_uri(uri, expected): + assert parse_uri(uri) == expected + + +@pytest.mark.parametrize( + "uri, expected_exception_message", + [ + # Test cases for invalid B2 URIs (missing netloc part) + ("b2://", "Invalid B2 URI: 'b2://'"), + ("b2id://", "Invalid B2 URI: 'b2id://'"), + # Test cases for B2 URIs with credentials + ( + "b2://user@password:bucket/path", + "Invalid B2 URI: credentials passed using `user@password:` syntax are not supported in URI", + ), + ( + "b2id://user@password:file123", + "Invalid B2 URI: credentials passed using `user@password:` syntax are not supported in URI", + ), + # Test cases for unsupported URI schemes + ("unknown://bucket/path", "Unsupported URI scheme: 'unknown'"), + ], +) +def test_parse_uri_exceptions(uri, expected_exception_message): + with pytest.raises(ValueError) as exc_info: + parse_uri(uri) + assert expected_exception_message in str(exc_info.value) diff --git a/test/unit/conftest.py b/test/unit/conftest.py index e7894c900..c21344b75 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -7,6 +7,7 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from test.unit.helpers import RunOrDieExecutor from unittest import mock import pytest @@ -17,3 +18,10 @@ def mock_realm_urls(): with mock.patch.dict(REALM_URLS, {'production': 'http://production.example.com'}): yield + + +@pytest.fixture +def bg_executor(): + """Executor for running background tasks in tests""" + with RunOrDieExecutor() as executor: + yield executor diff --git a/test/unit/console_tool/test_download_file.py b/test/unit/console_tool/test_download_file.py index 63bdc60d7..717b05648 100644 --- a/test/unit/console_tool/test_download_file.py +++ b/test/unit/console_tool/test_download_file.py @@ -8,22 +8,24 @@ # ###################################################################### import os +import pathlib +from test.helpers import skip_on_windows import pytest @pytest.fixture -def test_file_setup(tmpdir): +def local_file(tmp_path): """Set up a test file and return its path.""" filename = 'file1.txt' content = 'hello world' - local_file = tmpdir.join(filename) - local_file.write(content) + local_file = tmp_path / filename + local_file.write_text(content) mod_time = 1500111222 os.utime(local_file, (mod_time, mod_time)) - return local_file, content + return local_file EXPECTED_STDOUT_DOWNLOAD = ''' @@ -41,58 +43,127 @@ def test_file_setup(tmpdir): ''' -def upload_file(b2_cli, local_file, filename='file1.txt'): - """Helper function to upload a file.""" - b2_cli.run(['upload-file', 'my-bucket', str(local_file), filename]) - +@pytest.fixture +def uploaded_file(b2_cli, bucket, local_file): + filename = 'file1.txt' + b2_cli.run(['upload-file', bucket, str(local_file), filename]) + return { + 'bucket': bucket, + 'fileName': filename, + 'content': local_file.read_text(), + } -def test_download_file_by_name(b2_cli, bucket, test_file_setup): - local_file, content = test_file_setup - upload_file(b2_cli, local_file) +def test_download_file_by_name(b2_cli, local_file, uploaded_file, tmp_path): + output_path = tmp_path / 'output.txt' b2_cli.run( - ['download-file-by-name', '--noProgress', 'my-bucket', 'file1.txt', - str(local_file)], + [ + 'download-file-by-name', '--noProgress', uploaded_file['bucket'], + uploaded_file['fileName'], + str(output_path) + ], expected_stdout=EXPECTED_STDOUT_DOWNLOAD ) - assert local_file.read() == content - + assert output_path.read_text() == uploaded_file['content'] -def test_download_file_by_name_quietly(b2_cli, bucket, test_file_setup): - local_file, content = test_file_setup - upload_file(b2_cli, local_file) +def test_download_file_by_name_quietly(b2_cli, uploaded_file, tmp_path): + output_path = tmp_path / 'output.txt' b2_cli.run( - ['download-file-by-name', '--quiet', 'my-bucket', 'file1.txt', - str(local_file)], + [ + 'download-file-by-name', '--quiet', uploaded_file['bucket'], uploaded_file['fileName'], + str(output_path) + ], expected_stdout='' ) - assert local_file.read() == content + assert output_path.read_text() == uploaded_file['content'] -def test_download_file_by_id(b2_cli, bucket, test_file_setup): - local_file, content = test_file_setup - - upload_file(b2_cli, local_file) +def test_download_file_by_id(b2_cli, uploaded_file, tmp_path): + output_path = tmp_path / 'output.txt' b2_cli.run( ['download-file-by-id', '--noProgress', '9999', - str(local_file)], # <-- Here's the change + str(output_path)], expected_stdout=EXPECTED_STDOUT_DOWNLOAD ) - assert local_file.read() == content + assert output_path.read_text() == uploaded_file['content'] + + +def test_download_file_by_id_quietly(b2_cli, uploaded_file, tmp_path): + output_path = tmp_path / 'output.txt' + b2_cli.run(['download-file-by-id', '--quiet', '9999', str(output_path)], expected_stdout='') + assert output_path.read_text() == uploaded_file['content'] -def test_download_file_by_id_quietly(b2_cli, bucket, test_file_setup): - local_file, content = test_file_setup - upload_file(b2_cli, local_file) +@skip_on_windows(reason='os.mkfifo is not supported on Windows') +def test_download_file_by_name__named_pipe( + b2_cli, local_file, uploaded_file, tmp_path, bg_executor +): + output_path = tmp_path / 'output.txt' + os.mkfifo(output_path) + + output_string = None + + def reader(): + nonlocal output_string + output_string = output_path.read_text() + + reader_future = bg_executor.submit(reader) b2_cli.run( - ['download-file-by-id', '--quiet', '9999', - str(local_file)], # <-- Here's the change - expected_stdout='' + [ + 'download-file-by-name', '--noProgress', uploaded_file['bucket'], + uploaded_file['fileName'], + str(output_path) + ], + expected_stdout=EXPECTED_STDOUT_DOWNLOAD ) - assert local_file.read() == content + reader_future.result(timeout=1) + assert output_string == uploaded_file['content'] + + +@pytest.fixture +def uploaded_stdout_txt(b2_cli, bucket, local_file, tmp_path): + local_file.write_text('non-mocked /dev/stdout test ignore me') + b2_cli.run(['upload-file', bucket, str(local_file), 'stdout.txt']) + return { + 'bucket': bucket, + 'fileName': 'stdout.txt', + 'content': local_file.read_text(), + } + + +def test_download_file_by_name__to_stdout_by_alias( + b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd +): + """Test download_file_by_name stdout alias support""" + b2_cli.run( + ['download-file-by-name', '--noProgress', bucket, uploaded_stdout_txt['fileName'], '-'], + ) + assert capfd.readouterr().out == uploaded_stdout_txt['content'] + assert not pathlib.Path('-').exists() + + +def test_cat__b2_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): + """Test download_file_by_name stdout alias support""" + b2_cli.run(['cat', '--noProgress', f"b2://{bucket}/{uploaded_stdout_txt['fileName']}"],) + assert capfd.readouterr().out == uploaded_stdout_txt['content'] + + +def test_cat__b2_uri__invalid(b2_cli, capfd): + b2_cli.run( + ['cat', "nothing/meaningful"], + expected_stderr=None, + expected_status=2, + ) + assert "argument b2uri: Unsupported URI scheme: ''" in capfd.readouterr().err + + +def test_cat__b2id_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): + """Test download_file_by_name stdout alias support""" + b2_cli.run(['cat', '--noProgress', "b2id://9999"],) + assert capfd.readouterr().out == uploaded_stdout_txt['content'] diff --git a/test/unit/console_tool/test_upload_file.py b/test/unit/console_tool/test_upload_file.py index ff49a1af2..c9da4d322 100644 --- a/test/unit/console_tool/test_upload_file.py +++ b/test/unit/console_tool/test_upload_file.py @@ -9,7 +9,6 @@ ###################################################################### import os from test.helpers import skip_on_windows -from test.unit.helpers import run_in_background import b2 @@ -41,13 +40,13 @@ def test_upload_file__file_info_src_last_modified_millis(b2_cli, bucket, tmpdir) @skip_on_windows -def test_upload_file__named_pipe(b2_cli, bucket, tmpdir): +def test_upload_file__named_pipe(b2_cli, bucket, tmpdir, bg_executor): """Test upload_file supports named pipes""" filename = 'named_pipe.txt' content = 'hello world' local_file1 = tmpdir.join('file1.txt') os.mkfifo(str(local_file1)) - writer = run_in_background( + writer = bg_executor.submit( local_file1.write, content ) # writer will block until content is read @@ -66,13 +65,13 @@ def test_upload_file__named_pipe(b2_cli, bucket, tmpdir): remove_version=True, expected_part_of_stdout=expected_stdout, ) - writer.join() + writer.result(timeout=1) def test_upload_file__hyphen_file_instead_of_stdin(b2_cli, bucket, tmpdir, monkeypatch): """Test upload_file will upload file named `-` instead of stdin by default""" # TODO remove this in v4 - assert b2.__version__ < '4', "`-` file upload should not be supported in next major version of CLI" + assert b2.__version__ < '4', "`-` filename should not be supported in next major version of CLI" filename = 'stdin.txt' content = "I'm very rare creature, a file named '-'" monkeypatch.chdir(str(tmpdir)) @@ -92,7 +91,7 @@ def test_upload_file__hyphen_file_instead_of_stdin(b2_cli, bucket, tmpdir, monke remove_version=True, expected_part_of_stdout=expected_stdout, expected_stderr= - "WARNING: Filename `-` won't be supported in the future and will be treated as stdin alias.\n", + "WARNING: Filename `-` won't be supported in the future and will always be treated as stdin alias.\n", ) diff --git a/test/unit/console_tool/test_upload_unbound_stream.py b/test/unit/console_tool/test_upload_unbound_stream.py index 9c9015325..665908d63 100644 --- a/test/unit/console_tool/test_upload_unbound_stream.py +++ b/test/unit/console_tool/test_upload_unbound_stream.py @@ -9,19 +9,18 @@ ###################################################################### import os from test.helpers import skip_on_windows -from test.unit.helpers import run_in_background -from b2._cli.const import DEFAULT_MIN_PART_SIZE +from b2sdk.v2 import DEFAULT_MIN_PART_SIZE @skip_on_windows -def test_upload_unbound_stream__named_pipe(b2_cli, bucket, tmpdir): +def test_upload_unbound_stream__named_pipe(b2_cli, bucket, tmpdir, bg_executor): """Test upload_unbound_stream supports named pipes""" filename = 'named_pipe.txt' content = 'hello world' fifo_file = tmpdir.join('fifo_file.txt') os.mkfifo(str(fifo_file)) - writer = run_in_background(fifo_file.write, content) # writer will block until content is read + writer = bg_executor.submit(fifo_file.write, content) # writer will block until content is read expected_stdout = f'URL by file name: http://download.example.com/file/my-bucket/{filename}' expected_json = { @@ -37,7 +36,7 @@ def test_upload_unbound_stream__named_pipe(b2_cli, bucket, tmpdir): remove_version=True, expected_part_of_stdout=expected_stdout, ) - writer.join() + writer.result(timeout=1) def test_upload_unbound_stream__stdin(b2_cli, bucket, tmpdir, mock_stdin): @@ -64,7 +63,9 @@ def test_upload_unbound_stream__stdin(b2_cli, bucket, tmpdir, mock_stdin): @skip_on_windows -def test_upload_unbound_stream__with_part_size_options(b2_cli, bucket, tmpdir, mock_stdin): +def test_upload_unbound_stream__with_part_size_options( + b2_cli, bucket, tmpdir, mock_stdin, bg_executor +): """Test upload_unbound_stream with part size options""" part_size = DEFAULT_MIN_PART_SIZE expected_size = part_size + 500 # has to be bigger to force multipart upload @@ -72,8 +73,8 @@ def test_upload_unbound_stream__with_part_size_options(b2_cli, bucket, tmpdir, m filename = 'named_pipe.txt' fifo_file = tmpdir.join('fifo_file.txt') os.mkfifo(str(fifo_file)) - writer = run_in_background( - fifo_file.write, "x" * expected_size + writer = bg_executor.submit( + lambda: fifo_file.write("x" * expected_size) ) # writer will block until content is read expected_stdout = f'URL by file name: http://download.example.com/file/my-bucket/{filename}' @@ -99,7 +100,7 @@ def test_upload_unbound_stream__with_part_size_options(b2_cli, bucket, tmpdir, m remove_version=True, expected_part_of_stdout=expected_stdout, ) - writer.join() + writer.result(timeout=1) def test_upload_unbound_stream__regular_file(b2_cli, bucket, tmpdir): diff --git a/test/unit/helpers.py b/test/unit/helpers.py index 62bfa3220..b7f1b7893 100644 --- a/test/unit/helpers.py +++ b/test/unit/helpers.py @@ -7,10 +7,34 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### -import threading +import concurrent.futures +import sys -def run_in_background(func, *args, **kwargs) -> threading.Thread: - thread = threading.Thread(target=func, args=args, kwargs=kwargs) - thread.start() - return thread +class RunOrDieExecutor(concurrent.futures.ThreadPoolExecutor): + """ + Deadly ThreadPoolExecutor, which ensures all task are quickly closed before exiting. + + Only really usable in tests. + """ + + def __exit__(self, exc_type, exc_val, exc_tb): + self.shutdown(wait=False, cancel_futures=True) + return super().__exit__(exc_type, exc_val, exc_tb) + + if sys.version_info < (3, 9): # shutdown(cancel_futures=True) is Python 3.9+ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._futures = [] + + def shutdown(self, wait=True, cancel_futures=False): + if cancel_futures: + for future in self._futures: + future.cancel() + super().shutdown(wait=wait) + + def submit(self, *args, **kwargs): + future = super().submit(*args, **kwargs) + self._futures.append(future) + return future diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 57aa6a606..285ba3312 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -134,6 +134,8 @@ def _remove_api_version_number(self, s): return re.sub(self.RE_API_VERSION, '/vx/', s) def _normalize_expected_output(self, text, format_vars=None): + if text is None: + return None format_vars = format_vars or {} return self._trim_leading_spaces(text).format( account_id=self.account_id, master_key=self.master_key, **format_vars @@ -213,7 +215,7 @@ def _run_command( ) print('EXPECTED TO FIND IN STDOUT:', repr(expected_part_of_stdout)) print('ACTUAL STDOUT: ', repr(actual_stdout)) - if expected_stderr != actual_stderr: + if expected_stderr is not None and expected_stderr != actual_stderr: print('EXPECTED STDERR:', repr(expected_stderr)) print('ACTUAL STDERR: ', repr(actual_stderr)) print(actual_stderr) @@ -235,8 +237,10 @@ def _run_command( self.assertIn(expected_part_of_stdout, actual_stdout) if unexpected_part_of_stdout is not None: self.assertNotIn(unexpected_part_of_stdout, actual_stdout) - self.assertEqual(expected_stderr, actual_stderr, 'stderr') + if expected_stderr is not None: + self.assertEqual(expected_stderr, actual_stderr, 'stderr') self.assertEqual(expected_status, actual_status, 'exit status code') + return actual_status, actual_stdout, actual_stderr @classmethod def _upload_multiple_files(cls, bucket):