feat(params): use binary file-upload if file is large

rapyuta_io/clients/paramserver.py

@@ -18,7 +18,6 @@
     validate_list_of_strings
 import six
 
-
 class _Node(str, enum.Enum):
 
     def __str__(self):
@@ -36,6 +35,8 @@ class _ParamserverClient:
     yaml_content_type = 'text/yaml'
     json_content_type = 'application/json'
     default_binary_content_type = "application/octet-stream"
+    max_non_binary_size = 128 * 1024
+
 
     def __init__(self, auth_token, project, core_api_host):
         self._auth_token = auth_token
@@ -69,6 +70,11 @@ def create_binary_file(self, tree_path, file_path, retry_limit=0):
             if guessed_content_type[1]:
                 headers['Content-Encoding'] = guessed_content_type[1]
 
+        # Override Content-Type for JSON and YAML to allow creating Binary files.
+        # This is required for large YAML/JSON files.
+        if content_type in (self.json_content_type, self.yaml_content_type):
+            content_type = self.default_binary_content_type
+
         headers.update({'X-Rapyuta-Params-Version': "0",
                         'Content-Type': content_type})
 
@@ -129,7 +135,10 @@ def process_dir(self, executor, rootdir, tree_path, level, dir_futures, file_fut
                 future = executor.submit(func, new_tree_path)
                 dir_futures[future] = (new_tree_path, level + 1)
             elif not in_attribute_dir:  # ignore files in attribute directories
+                file_stat = os.stat(full_path)
                 file_name = os.path.basename(full_path)
+                if file_stat.st_size > self.max_non_binary_size:
+                    future = executor.submit(self.create_binary_file, new_tree_path, full_path)
                 if file_name.endswith('.yaml'):
                     with open(full_path, 'r') as f:
                         data = f.read()
@@ -151,8 +160,11 @@ def process_folder(self, executor, rootdir, tree_path, level, dir_futures, file_
                 future = executor.submit(self.create_folder, new_tree_path)
                 dir_futures[future] = (new_tree_path, level + 1)
             else:
+                file_stat = os.stat(full_path)
                 file_name = os.path.basename(full_path)
-                if file_name.endswith('.yaml'):
+                if file_stat.st_size > self.max_non_binary_size:
+                    future = executor.submit(self.create_binary_file, new_tree_path, full_path)
+                elif file_name.endswith('.yaml'):
                     with open(full_path, 'r') as f:
                         data = f.read()
                     future = executor.submit(self.create_file, new_tree_path, data)