feat(params): use binary file-upload if file is large

rapyuta_io/clients/paramserver.py

@@ -18,7 +18,6 @@
     validate_list_of_strings
 import six
 
-
 class _Node(str, enum.Enum):
 
     def __str__(self):
@@ -30,6 +29,8 @@ def __str__(self):
     Folder = 'FolderNode'
 
 class _ParamserverClient:
+    MaxNonBinarySize = 128 * 1024
+
     """
     Internal client for paramserver. Not for public use.
     """
@@ -151,8 +152,11 @@ def process_folder(self, executor, rootdir, tree_path, level, dir_futures, file_
                 future = executor.submit(self.create_folder, new_tree_path)
                 dir_futures[future] = (new_tree_path, level + 1)
             else:
+                file_stat = os.stat(full_path)
                 file_name = os.path.basename(full_path)
-                if file_name.endswith('.yaml'):
+                if file_stat.st_size > self.MaxNonBinarySize:
+                    future = executor.submit(self.create_binary_file, new_tree_path, full_path)
+                elif file_name.endswith('.yaml'):
                     with open(full_path, 'r') as f:
                         data = f.read()
                     future = executor.submit(self.create_file, new_tree_path, data)