Merge pull request #1061 from Backblaze/upload_time

Upload time tweaks

b2/_internal/console_tool.py

@@ -3007,6 +3007,8 @@ class Sync(
     Ignored files or file versions will not be taken for consideration during sync.
     The time should be given as a seconds timestamp (e.g. "1367900664")
     If you need milliseconds precision, put it after the comma (e.g. "1367900664.152")
+    Alternatively you can specify ``--exclude-if-uploaded-after`` to use the server-side
+    object creation timestamp rather than the modification time declared by the client.
 
     Files are considered to be the same if they have the same name
     and modification time.  This behaviour can be changed using the
@@ -3124,6 +3126,13 @@ def _setup_parser(cls, parser):
             default=None,
             metavar='TIMESTAMP'
         )
+        add_normalized_argument(
+            parser,
+            '--exclude-if-uploaded-after',
+            type=parse_millis_from_float_timestamp,
+            default=None,
+            metavar='TIMESTAMP'
+        )
         super()._setup_parser(parser)  # add parameters from the mixins, and the parent class
         parser.add_argument('source')
         parser.add_argument('destination')
@@ -3217,6 +3226,7 @@ def get_policies_manager_from_args(self, args):
             include_file_regexes=args.include_regex,
             exclude_all_symlinks=args.exclude_all_symlinks,
             exclude_modified_after=args.exclude_if_modified_after,
+            exclude_uploaded_after=args.exclude_if_uploaded_after,
         )
 
     def get_synchronizer_from_args(

changelog.d/+9092be80.added.md

@@ -0,0 +1 @@
+Add `--exclude-if-uploaded-after` to `sync`.

test/integration/test_b2_command_line.py

@@ -1197,12 +1197,15 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryp
             upload_encryption_args,
             additional_env=upload_additional_env,
         )
+
+        # Sync all the files
         b2_tool.should_succeed(
             ['sync', b2_sync_point, local_path] + sync_encryption_args,
             additional_env=sync_additional_env,
         )
         should_equal(['a', 'b'], sorted(os.listdir(local_path)))
 
+        # Put another file in B2
         b2_tool.should_succeed(
             ['file', 'upload', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'c'] +
             upload_encryption_args,
@@ -1219,14 +1222,35 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryp
             additional_env=sync_additional_env,
         )
         should_equal(['a', 'b'], sorted(os.listdir(local_path)))
+
+        # Put another file in B2 with a custom upload timestamp
+        b2_tool.should_succeed(
+            [
+                'file', 'upload', '--no-progress', '--custom-upload-timestamp', '1367900664152',
+                bucket_name, sample_file, b2_file_prefix + 'd'
+            ] + upload_encryption_args,
+            additional_env=upload_additional_env,
+        )
+
+        # Sync the files with one file being excluded because of upload timestamp
+        b2_tool.should_succeed(
+            [
+                'sync', '--no-progress', '--exclude-if-uploaded-after', '1367900664142',
+                b2_sync_point, local_path
+            ] + sync_encryption_args,
+            additional_env=sync_additional_env,
+        )
+        should_equal(['a', 'b'], sorted(os.listdir(local_path)))
+
         # Sync all the files
         b2_tool.should_succeed(
             ['sync', '--no-progress', b2_sync_point, local_path] + sync_encryption_args,
             additional_env=sync_additional_env,
         )
-        should_equal(['a', 'b', 'c'], sorted(os.listdir(local_path)))
-    with TempDir() as new_local_path:
-        if encryption and encryption.mode == EncryptionMode.SSE_C:
+        should_equal(['a', 'b', 'c', 'd'], sorted(os.listdir(local_path)))
+
+    if encryption and encryption.mode == EncryptionMode.SSE_C:
+        with TempDir() as new_local_path:
             b2_tool.should_fail(
                 ['sync', '--no-progress', b2_sync_point, new_local_path] + sync_encryption_args,
                 expected_pattern='ValueError: Using SSE-C requires providing an encryption key via '

test/unit/test_console_tool.py

@@ -15,6 +15,7 @@
 from functools import lru_cache
 from io import StringIO
 from itertools import chain, product
+from tempfile import TemporaryDirectory
 from test.helpers import skip_on_windows
 from typing import List, Optional
 from unittest import mock
@@ -2210,6 +2211,52 @@ def test_sync_exclude_if_modified_after_exact(self):
             ]
             self._run_command(command, expected_stdout, '', 0)
 
+    def test_sync_exclude_if_uploaded_after_in_range(self):
+        self._authorize_account()
+        self._create_my_bucket()
+
+        with TemporaryDirectory() as temp_dir:
+            for file, utime in (('test.txt', 1367900664152), ('test2.txt', 1367600664152)):
+                file_path = self._make_local_file(temp_dir, file)
+                command = [
+                    'file', 'upload', '--no-progress', '--custom-upload-timestamp',
+                    str(utime), 'my-bucket', file_path, file
+                ]
+                self._run_command(command, expected_status=0)
+
+        with TemporaryDirectory() as temp_dir:
+            command = [
+                'sync', '--no-progress', '--exclude-if-uploaded-after', '1367700664.152',
+                'b2://my-bucket', temp_dir
+            ]
+            expected_stdout = '''
+            dnload test2.txt
+            '''
+            self._run_command(command, expected_stdout, '', 0)
+
+    def test_sync_exclude_if_uploaded_after_exact(self):
+        self._authorize_account()
+        self._create_my_bucket()
+
+        with TemporaryDirectory() as temp_dir:
+            for file, utime in (('test.txt', 1367900664152), ('test2.txt', 1367600664152)):
+                file_path = self._make_local_file(temp_dir, file)
+                command = [
+                    'file', 'upload', '--no-progress', '--custom-upload-timestamp',
+                    str(utime), 'my-bucket', file_path, file
+                ]
+                self._run_command(command, expected_status=0)
+
+        with TemporaryDirectory() as temp_dir:
+            command = [
+                'sync', '--no-progress', '--exclude-if-uploaded-after', '1367600664.152',
+                'b2://my-bucket', temp_dir
+            ]
+            expected_stdout = '''
+            dnload test2.txt
+            '''
+            self._run_command(command, expected_stdout, '', 0)
+
     def _test_sync_threads(
         self,
         threads=None,