Add script to validate remote object store paths. (mosaicml#2667)

composer/utils/file_helpers.py

@@ -400,6 +400,44 @@ def maybe_create_remote_uploader_downloader_from_uri(
                                   'one of the supported RemoteUploaderDownloader object stores')
 
 
+def list_remote_objects(remote_path: str) -> List[str]:
+    """List objects at the remote path.
+
+    Args:
+        remote_path (str): Remote object store path.
+
+    Returns:
+        A list of objects at the remote path.
+    """
+    object_store = maybe_create_object_store_from_uri(remote_path)
+    if object_store is None:
+        raise ValueError(f'Failed to create object store. The given path {remote_path} is a local path.')
+    _, _, prefix = parse_uri(remote_path)
+    objects = object_store.list_objects(prefix)
+    return objects
+
+
+def validate_remote_path():
+    """Entry point to composer_validate_remote_path cli command.
+
+    Validates a remote path.
+    If the remote path is valid, prints a list of objects at the path.
+    Otherwise, raises an error.
+    """
+    import sys
+    args = sys.argv
+    if len(args) == 1:
+        raise ValueError('Please provide a remote path.')
+    if len(args) > 2:
+        raise ValueError('Extra arguments found. Please provide only one remote path.')
+    remote_path = sys.argv[1]
+    objects = list_remote_objects(remote_path)
+    if len(objects) == 0:
+        raise ValueError(f'No objects at path {remote_path} found. Please check your path and your access credentials.')
+    objects_str = '\n'.join(objects)
+    print(f'Found {len(objects)} objects at {remote_path} \n{objects_str}')
+
+
 def get_file(path: str,
              destination: str,
              object_store: Optional[Union[ObjectStore, LoggerDestination]] = None,

setup.py

@@ -266,6 +266,7 @@ def package_files(prefix: str, directory: str, extension: str):
           'console_scripts': [
               'composer = composer.cli.launcher:main',
               'composer_collect_env = composer.utils.collect_env:main',
+              'composer_validate_remote_path = composer.utils.file_helpers:validate_remote_path',
           ],
       },
       extras_require=extra_deps,