feat(params): use binary file-upload if file is large

CHANGELOG/CHANGELOG.md

@@ -1,3 +1,68 @@
+# [1.13.0](https://github.com/rapyuta-robotics/rapyuta-io-sdk/compare/v1.12.0...v1.13.0) (2023-12-13)
+
+
+### Features
+
+* **device:** accepts feature config in toggle_features ([#70](https://github.com/rapyuta-robotics/rapyuta-io-sdk/issues/70)) ([a293787](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/a2937874d500633c79497cfc8414ed5f09e32137))
+
+# [1.12.0](https://github.com/rapyuta-robotics/rapyuta-io-sdk/compare/v1.11.1...v1.12.0) (2023-09-26)
+
+
+### Features
+
+* **usergroup:** supports user group role in projects ([4f6c34f](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/4f6c34fd713cb493c6b44225d566e90afe8f88fa))
+
+## [1.11.1](https://github.com/rapyuta-robotics/rapyuta-io-sdk/compare/v1.11.0...v1.11.1) (2023-07-27)
+
+
+### Bug Fixes
+
+* **package:** makes dependent deployment ready phase configurable ([e0df4ec](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/e0df4ecab55a2defd47cae6446dda7f48e7e9f42))
+
+# [1.11.0](https://github.com/rapyuta-robotics/rapyuta-io-sdk/compare/v1.10.0...v1.11.0) (2023-07-14)
+
+### Features
+
+* **deployment**: adds update_deployment API ([51c14af74c3377585c56c025cb99af12bc90910d](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/51c14af74c3377585c56c025cb99af12bc90910d))
+* **deployment**: made poll deployment till ready phase configurable ([763716c](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/763716c47c426a513f4f9eb522419086a37349bc))
+* **usergroup**: implements usergroup APIS ([6d9037e](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/6d9037eb162e8854e0595dcf4caf3caf11c7980a))
+
+# [1.10.0](https://github.com/rapyuta-robotics/rapyuta-io-sdk/compare/v1.9.0...v1.10.0) (2023-05-31)
+
+
+### Features
+
+* **rip:** adds support for token_level in get_auth_token ([88fcc69](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/88fcc697842de856a031a370f8ab630047532f49))
+
+# [1.9.0](https://github.com/rapyuta-robotics/rapyuta-io-sdk/compare/v1.8.0...v1.9.0) (2023-04-19)
+
+
+### Bug Fixes
+
+* **metrics:** tags schema and its unit tests ([6cf652b](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/6cf652b177fb7c722a037ff9498c845ee12a6ff1))
+* **rosbags:** integration-test for throttling ([3cd2cd6](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/3cd2cd6d1ed1ca3a6b4beb050865dcedb56b4c45))
+
+
+### Features
+
+* **device:** adds API to toggle features ([39860fc](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/39860fc05a7a261862a2738ca1d9c7af546a0f6f))
+* **network:** allows custom resource limits for cloud networks ([e58f0c5](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/e58f0c5c33ddd0f860210cc9a74cb6e6cf9ef6b9))
+* **parameters:** adds the option to upload arbitrary directories as config ([4c95e96](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/4c95e96db054bbc5fba615edafd586d65a9b7a5a))
+
+# [1.8.0](https://github.com/rapyuta-robotics/rapyuta-io-sdk/compare/v1.7.0...v1.8.0) (2023-02-07)
+
+
+### Features
+
+* **rosbag:** fetches rosbag job by guid ([3d93c6e](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/3d93c6e2f8c1d4e6bda7de2815b1ec3a1001bdc9))
+
+# [1.7.0](https://github.com/rapyuta-robotics/rapyuta-io-sdk/compare/v1.6.1...v1.7.0) (2022-12-14)
+
+
+### Features
+
+* **rosbags:** adds support for splitting rosbags by duration ([3770e00](https://github.com/rapyuta-robotics/rapyuta-io-sdk/commit/3770e00a7d95d6ca8699185a96bf6941a29c3875))
+
 # [1.6.0](https://github.com/rapyuta-robotics/rapyuta-io-sdk/compare/v1.5.0...v1.6.0) (2022-10-27)
 ### Features
 - **device:** Added support for throttling and latching of rosbags

rapyuta_io/__init__.py

@@ -15,4 +15,4 @@
 from .clients.rosbag import UploadOptions
 from .clients.user_group import UserGroup
 
-__version__ = "1.6.0"
+__version__ = "1.13.0"

rapyuta_io/clients/paramserver.py

@@ -18,7 +18,6 @@
     validate_list_of_strings
 import six
 
-
 class _Node(str, enum.Enum):
 
     def __str__(self):
@@ -36,6 +35,8 @@ class _ParamserverClient:
     yaml_content_type = 'text/yaml'
     json_content_type = 'application/json'
     default_binary_content_type = "application/octet-stream"
+    max_non_binary_size = 128 * 1024
+
 
     def __init__(self, auth_token, project, core_api_host):
         self._auth_token = auth_token
@@ -69,6 +70,11 @@ def create_binary_file(self, tree_path, file_path, retry_limit=0):
             if guessed_content_type[1]:
                 headers['Content-Encoding'] = guessed_content_type[1]
 
+        # Override Content-Type for JSON and YAML to allow creating Binary files.
+        # This is required for large YAML/JSON files.
+        if content_type in (self.json_content_type, self.yaml_content_type):
+            content_type = self.default_binary_content_type
+
         headers.update({'X-Rapyuta-Params-Version': "0",
                         'Content-Type': content_type})
 
@@ -129,7 +135,10 @@ def process_dir(self, executor, rootdir, tree_path, level, dir_futures, file_fut
                 future = executor.submit(func, new_tree_path)
                 dir_futures[future] = (new_tree_path, level + 1)
             elif not in_attribute_dir:  # ignore files in attribute directories
+                file_stat = os.stat(full_path)
                 file_name = os.path.basename(full_path)
+                if file_stat.st_size > self.max_non_binary_size:
+                    future = executor.submit(self.create_binary_file, new_tree_path, full_path)
                 if file_name.endswith('.yaml'):
                     with open(full_path, 'r') as f:
                         data = f.read()
@@ -151,8 +160,11 @@ def process_folder(self, executor, rootdir, tree_path, level, dir_futures, file_
                 future = executor.submit(self.create_folder, new_tree_path)
                 dir_futures[future] = (new_tree_path, level + 1)
             else:
+                file_stat = os.stat(full_path)
                 file_name = os.path.basename(full_path)
-                if file_name.endswith('.yaml'):
+                if file_stat.st_size > self.max_non_binary_size:
+                    future = executor.submit(self.create_binary_file, new_tree_path, full_path)
+                elif file_name.endswith('.yaml'):
                     with open(full_path, 'r') as f:
                         data = f.read()
                     future = executor.submit(self.create_file, new_tree_path, data)