refactor: publisher and downloader args

docs/client_guide.md

@@ -43,12 +43,11 @@ If the client provides a free query auth token, the download will use the free q
 
 ### Quick Start CLI example
 
-Download a single file (`--manifest-type`) into local file system (`local-files`) with free query auth token (`--free-query-auth-token`)
+Download a single file into local file system (`local-files`) with free query auth token (`--free-query-auth-token`)
 
 ```
 $ file-exchange downloader \
    --ipfs-hash QmHash \
-   --manifest-type "File" \
    --indexer-endpoints http://localhost:5678,http://localhost:5677 \
    --free-query-auth-token 'Bearer auth_token' \
    --network-subgraph https://api.thegraph.com/subgraphs/name/graphprotocol/graph-network-arbitrum-sepolia \

docs/publisher_guide.md

@@ -21,22 +21,18 @@ Publisher must have read access to all files contained in the Bundle. The publis
 
 The publisher must provide a name for the bundle, filenames, file type, version, and path for read-access of the files.
 
-Publishing files stored in the local file system
+Publishing separate files stored in the local file system
 ```
 $ file-exchange publisher \
-  --bundle-name "blah" \
-  --file-names example0017686312.dbin,example-create-17686085.dbin \
-  --file-type flatfiles \
-  --file-version 0.0.0 \
-  --description "random flatfiles" \
-  local-files --output-dir ./example-file/
+  --filenames example0017686312.dbin,example-create-17686085.dbin \
+  local-files --main-dir ./example-file/
 ```
 
-Publishing files/objects stored in a remote s3 bucket
+Publishing files/objects stored in a remote s3 bucket into a bundle, provide s3 and bundle configurations.
 ```
 $ file-exchange publisher \
-  --bundle-name "blah" \
   --file-names example0017686312.dbin,example-create-17686085.dbin \
+  --bundle-name "blah" \
   --file-type flatfiles \
   --file-version 0.0.0 \
   --description "random flatfiles" \
@@ -46,9 +42,60 @@ $ file-exchange publisher \
    --secret-key "secretttttttttt" \
    --endpoint "https://ams3.digitaloceanspaces.com" 
 ```
+
 For more information 
 ```
 $ file-exchange --help
-```
 
-To publish files and not gather everything into a bundle, add additional argument of `--manifest-type file`.
+Publisher takes the files, generate bundle manifest,
+and publish to IPFS
+
+Usage: file-exchange publisher [OPTIONS] <COMMAND>
+
+Commands:
+  local-files     
+  object-storage  
+  help            Print this message or the help
+                      of the given subcommand(s)
+
+Options:
+      --yaml-store <YAML_STORE_DIR>
+          Path to the directory to store the generated
+          yaml file for bundle [env: YAML_STORE_DIR=]
+          [default: ./example-file/bundle.yaml]
+      --chunk-size <CHUNK_SIZE>
+          Chunk size in bytes to split files (Default:
+          1048576 bytes = 1MiB) [env: CHUNK_SIZE=]
+          [default: 1048576]
+      --filenames <FILE_NAMES>
+          Name for the files to publish [env:
+          FILE_NAMES=]
+      --bundle-name <BUNDLE_NAME>
+          Name for the bundle (later this can be
+          interactive) [env: BUNDLE_NAME=]
+      --file-type <FILE_TYPE>
+          Type of the file (e.g., sql_snapshot,
+          flatfiles) [env: FILE_TYPE=]
+      --bundle-version <FILE_VERSION>
+          Bundle versioning [env: FILE_VERSION=]
+      --identifier <IDENTIFIER>
+          Identifier of the file given its type
+          (chain-id for firehose flatfiles, subgraph
+          deployment hash for subgraph snapshots)
+          [env: IDENTIFIER=]
+      --start-block <START_BLOCK>
+          Start block for flatfiles [env:
+          START_BLOCK=]
+      --end-block <END_BLOCK>
+          End block for sql snapshot or flatfiles
+          [env: END_BLOCK=]
+      --description <DESCRIPTION>
+          Describe bundle content [env: DESCRIPTION=]
+          [default: ]
+      --chain-id <NETWORK>
+          Network represented in CCIP ID (Ethereum
+          mainnet: 1, goerli: 5, arbitrum-one: 42161,
+          sepolia: 58008 [env: NETWORK=] [default: 1]
+  -h, --help
+          Print help
+```

file-exchange/src/config.rs

@@ -52,8 +52,11 @@ impl Cli {
 #[derive(Clone, Debug, Subcommand, Serialize, Deserialize)]
 #[group(required = false, multiple = true)]
 pub enum Role {
+    #[clap(name = "downloader")]
     Downloader(DownloaderArgs),
+    #[clap(name = "publisher")]
     Publisher(PublisherArgs),
+    #[clap(name = "wallet")]
     Wallet(OnChainArgs),
 }
 
@@ -202,15 +205,6 @@ pub struct DownloaderArgs {
         help = "IPFS hash for the target bundle or file manifest"
     )]
     pub ipfs_hash: String,
-    #[arg(
-        long,
-        value_enum,
-        value_name = "manifest_type",
-        env = "manifest_type",
-        default_value_t = ManifestType::Bundle,
-        help = "Download type available: file, bundle (default)"
-    )]
-    pub manifest_type: ManifestType,
     #[arg(
         long,
         value_name = "GATEWAY_URL",
@@ -317,54 +311,6 @@ pub struct PublisherArgs {
         help = "Path to the directory to store the generated yaml file for bundle"
     )]
     pub yaml_store: String,
-    #[clap(subcommand)]
-    pub storage_method: StorageMethod,
-    #[arg(
-        long,
-        value_name = "BUNDLE_NAME",
-        env = "BUNDLE_NAME",
-        help = "Name for the bundle (later this can be interactive)"
-    )]
-    pub bundle_name: Option<String>,
-    #[arg(
-        long,
-        value_enum,
-        value_name = "manifest_type",
-        env = "manifest_type",
-        default_value_t = ManifestType::Bundle,
-        help = "Publish type available: file, bundle (default)"
-    )]
-    pub manifest_type: ManifestType,
-    #[arg(
-        long,
-        value_name = "FILE_NAMES",
-        value_delimiter = ',',
-        env = "FILE_NAMES",
-        help = "Name for the files to be included in bundle (later this can be interactive)"
-    )]
-    pub file_names: Vec<String>,
-    #[arg(
-        long,
-        value_name = "FILE_TYPE",
-        value_enum,
-        env = "FILE_TYPE",
-        help = "Type of the file (e.g., sql_snapshot, flatfiles)"
-    )]
-    pub file_type: String,
-    #[arg(
-        long,
-        value_name = "FILE_VERSION",
-        env = "FILE_VERSION",
-        help = "Bundle versioning"
-    )]
-    pub bundle_version: String,
-    #[arg(
-        long,
-        value_name = "IDENTIFIER",
-        env = "IDENTIFIER",
-        help = "Identifier of the file given its type (chain-id for firehose flatfiles, subgraph deployment hash for subgraph snapshots)"
-    )]
-    pub identifier: Option<String>,
     #[arg(
         long,
         value_name = "CHUNK_SIZE",
@@ -373,43 +319,18 @@ pub struct PublisherArgs {
         help = "Chunk size in bytes to split files (Default: 1048576 bytes = 1MiB)"
     )]
     pub chunk_size: u64,
+    #[clap(subcommand)]
+    pub storage_method: StorageMethod,
     #[arg(
         long,
-        value_name = "START_BLOCK",
-        env = "START_BLOCK",
-        help = "Start block for flatfiles"
-    )]
-    pub start_block: Option<u64>,
-    #[arg(
-        long,
-        value_name = "END_BLOCK",
-        env = "END_BLOCK",
-        help = "End block for sql snapshot or flatfiles"
-    )]
-    pub end_block: Option<u64>,
-    #[arg(
-        long,
-        value_name = "PUBLISHER_URL",
-        env = "PUBLISHER_URL",
-        help = "Self promoting endpoint to record inside the bundle (TODO: can update to be a github repository link)"
-    )]
-    pub publisher_url: Option<String>,
-    #[arg(
-        long,
-        value_name = "DESCRIPTION",
-        env = "DESCRIPTION",
-        default_value = "",
-        help = "Describe bundle content"
-    )]
-    pub description: String,
-    #[arg(
-        long,
-        value_name = "NETWORK",
-        env = "NETWORK",
-        default_value = "1",
-        help = "Network represented in CCIP ID (Ethereum mainnet: 1, goerli: 5, arbitrum-one: 42161, sepolia: 58008"
+        value_name = "FILE_NAMES",
+        value_delimiter = ',',
+        env = "FILE_NAMES",
+        help = "Name for the files to publish"
     )]
-    pub chain_id: String,
+    pub filenames: Vec<String>,
+    #[clap(flatten)]
+    pub bundle: Option<BundleArgs>,
 }
 
 #[derive(Clone, Debug, Args, Serialize, Deserialize, Default)]
@@ -521,12 +442,68 @@ pub struct ApproveArgs {
     pub tokens: U256,
 }
 
-#[allow(unused)]
-#[derive(ValueEnum, Clone, Debug, Serialize, Deserialize, Default)]
-pub enum ManifestType {
-    #[default]
-    Bundle,
-    File,
+#[derive(Clone, Debug, Args, Serialize, Deserialize, Default)]
+#[group(required = false, multiple = true)]
+pub struct BundleArgs {
+    #[arg(
+        long,
+        value_name = "BUNDLE_NAME",
+        env = "BUNDLE_NAME",
+        help = "Name for the bundle (later this can be interactive)"
+    )]
+    pub bundle_name: Option<String>,
+    #[arg(
+        long,
+        value_name = "FILE_TYPE",
+        value_enum,
+        env = "FILE_TYPE",
+        help = "Type of the file (e.g., sql_snapshot, flatfiles)"
+    )]
+    pub file_type: Option<String>,
+    #[arg(
+        long,
+        value_name = "FILE_VERSION",
+        env = "FILE_VERSION",
+        help = "Bundle versioning"
+    )]
+    pub bundle_version: Option<String>,
+    #[arg(
+        long,
+        value_name = "IDENTIFIER",
+        env = "IDENTIFIER",
+        help = "Identifier of the file given its type (chain-id for firehose flatfiles, subgraph deployment hash for subgraph snapshots)"
+    )]
+    pub identifier: Option<String>,
+    #[arg(
+        long,
+        value_name = "START_BLOCK",
+        env = "START_BLOCK",
+        help = "Start block for flatfiles"
+    )]
+    pub start_block: Option<u64>,
+    #[arg(
+        long,
+        value_name = "END_BLOCK",
+        env = "END_BLOCK",
+        help = "End block for sql snapshot or flatfiles"
+    )]
+    pub end_block: Option<u64>,
+    #[arg(
+        long,
+        value_name = "DESCRIPTION",
+        env = "DESCRIPTION",
+        default_value = "",
+        help = "Describe bundle content"
+    )]
+    pub description: Option<String>,
+    #[arg(
+        long,
+        value_name = "NETWORK",
+        env = "NETWORK",
+        default_value = "1",
+        help = "Network represented in CCIP ID (Ethereum mainnet: 1, goerli: 5, arbitrum-one: 42161, sepolia: 58008"
+    )]
+    pub chain_id: Option<String>,
 }
 
 #[allow(unused)]

file-exchange/src/download_client/mod.rs

@@ -87,25 +87,22 @@ pub struct OnChainSigner {
 
 impl Downloader {
     pub async fn new(ipfs_client: IpfsClient, args: DownloaderArgs) -> Self {
-        let target_manifest = match args.manifest_type {
-            crate::config::ManifestType::Bundle => {
-                let bundle = read_bundle(&ipfs_client, &args.ipfs_hash)
-                    .await
-                    .expect("Read bundle manifest");
-                TargetManifest::BundleManifest(bundle)
-            }
-            crate::config::ManifestType::File => {
-                let file_manifest = fetch_file_manifest_from_ipfs(&ipfs_client, &args.ipfs_hash)
-                    .await
-                    .expect("Read File manifest");
-                TargetManifest::FileManifest(FileManifestMeta {
-                    meta_info: FileMetaInfo {
-                        name: args.ipfs_hash.clone(),
-                        hash: args.ipfs_hash.clone(),
-                    },
-                    file_manifest,
-                })
-            }
+        // Automatically decipher if the desired download is a bundle or a file
+        let as_bundle = read_bundle(&ipfs_client, &args.ipfs_hash).await;
+
+        let target_manifest = if let Ok(bundle) = as_bundle {
+            TargetManifest::BundleManifest(bundle)
+        } else {
+            let file_manifest = fetch_file_manifest_from_ipfs(&ipfs_client, &args.ipfs_hash)
+                .await
+                .expect("The download ipfs_hash doesn't fit the manifest types");
+            TargetManifest::FileManifest(FileManifestMeta {
+                meta_info: FileMetaInfo {
+                    name: args.ipfs_hash.clone(),
+                    hash: args.ipfs_hash.clone(),
+                },
+                file_manifest,
+            })
         };
 
         let payment = if let Some(token) = &args.free_query_auth_token {
@@ -318,34 +315,7 @@ impl Downloader {
         // If storage method is the local file system, directly write the ranges
         // If remote object storage, first write ranges to a tmp file to complete the object
         // Open the output file
-        let file = match &self.store.storage_method {
-            StorageMethod::LocalFiles(directory) => {
-                let file = File::create(Path::new(
-                    &(directory.main_dir.clone() + "/" + &meta.meta_info.name),
-                ))
-                .unwrap_or_else(|_| {
-                    panic!(
-                        "Cannot create file for writing the output at directory {}",
-                        &directory.main_dir
-                    )
-                });
-
-                Arc::new(Mutex::new(file))
-            }
-            StorageMethod::ObjectStorage(store) => {
-                let file = File::create(Path::new(
-                    &("tmp/".to_owned() + &store.bucket + "/" + &meta.meta_info.name),
-                ))
-                .unwrap_or_else(|_| {
-                    panic!(
-                        "Cannot create file for writing the output at tmp/{}",
-                        &store.bucket.clone()
-                    )
-                });
-                tracing::debug!("Created tmp directory");
-                Arc::new(Mutex::new(file))
-            }
-        };
+        let file = self.store.prepare_write(&meta.meta_info.name);
 
         while !self.remaining_chunks(&meta.meta_info.hash).is_empty() {
             // Wait for all chunk tasks to complete and collect the results