refactor: split a subfile_server package

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
     "subfile-exchange", 
+    "subfile-service", 
 ]
 resolver = "2"
 

docs/client_guide.md

@@ -26,9 +26,12 @@ After determining the subfile CID, client should supply a local path for writing
 ### CLI example
 ```
 ➜  subfile-exchange git:(main) ✗ cargo run -p subfile-exchange downloader \
-   --ipfs-hash QmakV6VEwnydfe7PXFR3TRxHbhVm7mQRXqVHdsizhTRrGw \
-   --indexer-endpoints http://localhost:5678 \
-   --free-query-auth-token 'Bearer imfreeee'
+   --ipfs-hash QmeaPp764FjQjPB66M9ijmQKmLhwBpHQhA7dEbH2FA1j3v \
+   --indexer-endpoints http://localhost:5678,http://localhost:5677 \
+   --free-query-auth-token 'Bearer imfreeee' \
+   --mnemonic "sheriff obscure trick beauty army fat wink legal flee leader section suit" \
+   --chain-id 421614 \
+   --verifier 0xfC24cE7a4428A6B89B52645243662A02BA734ECF
 ```
 
 

subfile-exchange/src/config.rs

@@ -2,14 +2,11 @@ use clap::{arg, ValueEnum};
 use clap::{command, Args, Parser, Subcommand};
 use serde::{Deserialize, Serialize};
 use std::fmt;
-use std::path::PathBuf;
-use std::str::FromStr;
+
 use tracing::subscriber::SetGlobalDefaultError;
 use tracing_subscriber::EnvFilter;
 use tracing_subscriber::FmtSubscriber;
 
-use crate::errors::Error;
-
 #[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 #[command(
     name = "subfile-exchange",
@@ -370,53 +367,3 @@ impl fmt::Display for LogFormat {
         }
     }
 }
-
-/// Validate the subfile configurations at initialization
-pub fn validate_subfile_entries(entries: Vec<String>) -> Result<Vec<(String, PathBuf)>, Error> {
-    let mut results = Vec::new();
-
-    for entry in entries {
-        results.push(validate_subfile_entry(entry)?);
-    }
-
-    Ok(results)
-}
-
-/// Subfile entry must be in the format of "valid_ipfs_hash:valid_local_path"
-pub fn validate_subfile_entry(entry: String) -> Result<(String, PathBuf), Error> {
-    let parts: Vec<&str> = entry.split(':').collect();
-    if parts.len() != 2 {
-        return Err(Error::InvalidConfig(format!(
-            "Invalid format for entry: {}",
-            entry
-        )));
-    }
-
-    let ipfs_hash = parts[0];
-    let local_path = parts[1];
-
-    // Validate IPFS hash (this is a placeholder, you'll need to define what a valid IPFS hash is)
-    if !is_valid_ipfs_hash(ipfs_hash) {
-        return Err(Error::InvalidConfig(format!(
-            "Invalid IPFS hash: {}",
-            ipfs_hash
-        )));
-    }
-
-    // Validate local path
-    let path = PathBuf::from_str(local_path).map_err(|e| Error::InvalidConfig(e.to_string()))?;
-    if !path.exists() {
-        return Err(Error::InvalidConfig(format!(
-            "Path do not exist: {}",
-            local_path
-        )));
-    }
-
-    Ok((ipfs_hash.to_string(), path))
-}
-
-pub fn is_valid_ipfs_hash(hash: &str) -> bool {
-    // Basic validation for IPFS hash
-    // Note: This is a simplified check and may not cover all cases.
-    hash.starts_with("Qm") && hash.len() == 46
-}

subfile-exchange/src/ipfs.rs

@@ -8,7 +8,12 @@ use serde::Deserialize;
 use std::time::Duration;
 use std::{str::FromStr, sync::Arc};
 
-/// Reference type, clones will share the connection pool.
+pub fn is_valid_ipfs_hash(hash: &str) -> bool {
+    // Basic validation for IPFS hash
+    // Note: This is a simplified check and may not cover all cases.
+    hash.starts_with("Qm") && hash.len() == 46
+}
+
 #[derive(Clone, Debug)]
 pub struct IpfsClient {
     base: Arc<Uri>,

subfile-exchange/src/lib.rs

@@ -8,5 +8,5 @@ pub mod subfile;
 pub mod subfile_client;
 pub mod subfile_finder;
 pub mod subfile_reader;
-pub mod subfile_server;
 pub mod test_util;
+pub mod util;

subfile-exchange/src/main.rs

@@ -1,11 +1,10 @@
-use subfile_exchange::{subfile_client::SubfileDownloader, subfile_server::init_server};
-
 use dotenv::dotenv;
 
 use subfile_exchange::{
     config::{Cli, Role},
     ipfs::IpfsClient,
     publisher::SubfilePublisher,
+    subfile_client::SubfileDownloader,
 };
 
 #[tokio::main]
@@ -57,10 +56,8 @@ async fn main() {
         Role::Server(server_args) => {
             tracing::info!(
                 server = tracing::field::debug(&server_args),
-                "Tracker request"
+                "Use subfile-service crate"
             );
-
-            let _ = init_server(&client, server_args).await;
         }
     }
 }

subfile-exchange/src/subfile.rs

@@ -1,11 +1,15 @@
-use std::path::{Path, PathBuf};
+use std::{
+    path::{Path, PathBuf},
+    str::FromStr,
+};
 
 use serde::{Deserialize, Serialize};
 
 use crate::{
     errors::Error,
     file_hasher::{hash_chunk, verify_chunk},
     file_reader::{chunk_file, format_path, read_chunk},
+    ipfs::is_valid_ipfs_hash,
 };
 
 /* Public Manifests */
@@ -161,6 +165,49 @@ impl Subfile {
     }
 }
 
+/// Validate the subfile configurations at initialization
+pub fn validate_subfile_entries(entries: Vec<String>) -> Result<Vec<(String, PathBuf)>, Error> {
+    let mut results = Vec::new();
+
+    for entry in entries {
+        results.push(validate_subfile_entry(entry)?);
+    }
+
+    Ok(results)
+}
+
+/// Subfile entry must be in the format of "valid_ipfs_hash:valid_local_path"
+pub fn validate_subfile_entry(entry: String) -> Result<(String, PathBuf), Error> {
+    let parts: Vec<&str> = entry.split(':').collect();
+    if parts.len() != 2 {
+        return Err(Error::InvalidConfig(format!(
+            "Invalid format for entry: {}",
+            entry
+        )));
+    }
+
+    let ipfs_hash = parts[0];
+    let local_path = parts[1];
+
+    if !is_valid_ipfs_hash(ipfs_hash) {
+        return Err(Error::InvalidConfig(format!(
+            "Invalid IPFS hash: {}",
+            ipfs_hash
+        )));
+    }
+
+    // Validate local path
+    let path = PathBuf::from_str(local_path).map_err(|e| Error::InvalidConfig(e.to_string()))?;
+    if !path.exists() {
+        return Err(Error::InvalidConfig(format!(
+            "Path do not exist: {}",
+            local_path
+        )));
+    }
+
+    Ok((ipfs_hash.to_string(), path))
+}
+
 #[cfg(test)]
 mod tests {
     use crate::test_util::simple_subfile;

subfile-exchange/src/subfile_client/mod.rs

@@ -21,7 +21,7 @@ use crate::ipfs::IpfsClient;
 use crate::subfile::{ChunkFileMeta, Subfile};
 use crate::subfile_finder::{IndexerEndpoint, SubfileFinder};
 use crate::subfile_reader::read_subfile;
-use crate::subfile_server::util::build_wallet;
+use crate::util::build_wallet;
 
 use self::signer::ReceiptSigner;
 

subfile-exchange/src/subfile_finder/mod.rs

@@ -1,4 +1,5 @@
 use futures::{stream, StreamExt};
+use serde::{Deserialize, Serialize};
 
 use std::collections::HashMap;
 
@@ -12,7 +13,6 @@ use crate::errors::Error;
 use crate::ipfs::IpfsClient;
 
 use crate::subfile_reader::{fetch_subfile_from_ipfs, read_subfile};
-use crate::subfile_server::util::Operator;
 
 // Pair indexer operator address and indexer service endpoint (operator, indexer_url)
 // persumeably this should not be handled by clients themselves
@@ -149,7 +149,7 @@ impl SubfileFinder {
             }
         }
 
-        match unavailble_files(&file_map).await {
+        match unavailable_files(&file_map).await {
             files if !files.is_empty() => {
                 return Err(Error::DataUnavilable(format!(
                     "File availability incomplete, missing files: {:#?}",
@@ -217,7 +217,7 @@ impl SubfileFinder {
 }
 
 /// Check if there is a key in target_hashes where the corresponding availability is empty
-pub async fn unavailble_files(file_map: &FileAvailbilityMap) -> Vec<String> {
+pub async fn unavailable_files(file_map: &FileAvailbilityMap) -> Vec<String> {
     let mut missing_file = vec![];
     let hashes = file_map.lock().await;
     for (key, inner_map_arc) in hashes.iter() {
@@ -228,3 +228,10 @@ pub async fn unavailble_files(file_map: &FileAvailbilityMap) -> Vec<String> {
     }
     missing_file
 }
+
+//TODO: directly access the field instead
+#[derive(Serialize, Deserialize)]
+pub struct Operator {
+    #[serde(alias = "publicKey")]
+    pub public_key: String,
+}

subfile-exchange/src/util.rs

@@ -0,0 +1,16 @@
+use ethers::signers::{
+    coins_bip39::English, LocalWallet, MnemonicBuilder, Signer, Wallet, WalletError,
+};
+use ethers_core::k256::ecdsa::SigningKey;
+
+/// Build Wallet from Private key or Mnemonic
+pub fn build_wallet(value: &str) -> Result<Wallet<SigningKey>, WalletError> {
+    value
+        .parse::<LocalWallet>()
+        .or(MnemonicBuilder::<English>::default().phrase(value).build())
+}
+
+/// Get wallet public address to String
+pub fn wallet_address(wallet: &Wallet<SigningKey>) -> String {
+    format!("{:?}", wallet.address())
+}

subfile-exchange/tests/discovery.rs

@@ -4,7 +4,7 @@ mod tests {
 
     use subfile_exchange::{
         ipfs::IpfsClient,
-        subfile_finder::{unavailble_files, FileAvailbilityMap, IndexerEndpoint, SubfileFinder},
+        subfile_finder::{unavailable_files, FileAvailbilityMap, IndexerEndpoint, SubfileFinder},
         test_util::server_ready,
     };
 
@@ -188,12 +188,12 @@ mod tests {
             )
             .await
             .unwrap();
-        let unavailble_files = unavailble_files(&map).await;
-        assert!(unavailble_files.len() == 2);
-        assert!(unavailble_files.contains(&String::from(
+        let unavailable_files = unavailable_files(&map).await;
+        assert!(unavailable_files.len() == 2);
+        assert!(unavailable_files.contains(&String::from(
             "QmSydRNSzjozo5d7W4AyCK8BkgfpEU8KQp9kvSHzf2Ch4g"
         )));
-        assert!(unavailble_files.contains(&String::from(
+        assert!(unavailable_files.contains(&String::from(
             "QmSuyvzDpuDBoka2rCimRXPmX2icL7Vu6RUxoFWFQD7YBb"
         )));
 

subfile-service/Cargo.toml

@@ -0,0 +1,69 @@
+[package]
+name = "subfile-service"
+version = "0.0.1"
+edition = "2021"
+authors = ["hopeyen <[email protected]>"]
+rust-version = "1.72"
+description = "Subfile data service"
+readme = "README.md"
+license = "Apache-2.0"
+
+[[bin]]
+name = "subfile-service"
+path = "src/main.rs"
+
+[dependencies]
+subfile-exchange = { path = "../subfile-exchange" }
+alloy-sol-types = { version = "0.5.0", features = ["eip712-serde"] }
+alloy-primitives = { version = "0.5.0", features = ["serde"] }
+anyhow = "1.0"
+base64 = "0.21"
+build-info = "0.0.34"
+bytes = "1.0"
+chrono = "0.4.31"
+clap = { version = "4.4", features = ["cargo", "unstable-doc"] }
+confy = "0.5"
+derive-getters = "0.3.0"
+dotenv = "0.15"
+ethers = "2.0.10"
+ethers-core = "2.0.10"
+futures = { version = "0.3", features = ["compat"] }
+hex = "0.4.3"
+http = "0.2"
+hyper = { version = "0.14.27", features = [ "server" ]}
+ipfs-api-backend-hyper = "0.6"
+ipfs-api-prelude = "0.6"
+merkle-cbt = "0.3.2"
+rand = "0.8.4"
+reqwest = { version = "0.11", features = ["json", "stream", "multipart"] }
+rustls = "0.21.8"
+rustls-pemfile = "1.0.3"
+secp256k1 = "0.28.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+serde_yaml = "0.9"
+sha2 = "0.10.8"
+tap_core = { version = "0.7.0", git = "https://github.com/semiotic-ai/timeline-aggregation-protocol" }
+tempfile = "3.2.0"
+tokio = { version = "1.28", features = ["time", "sync", "macros", "test-util", "rt-multi-thread"] }
+tokio-retry = "0.3"
+toml = "0.7.4"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = [
+    "env-filter",
+    "ansi",
+    "fmt",
+    "std",
+    "json",
+] }
+
+[dev-dependencies]
+criterion = "0.5"
+
+[dev-dependencies.cargo-husky]
+version = "1"
+default-features = false
+features = ["precommit-hook", "run-cargo-fmt", "run-cargo-clippy"]
+
+[build-dependencies]
+build-info-build = "0.0.34"

subfile-service/build.rs

@@ -0,0 +1,5 @@
+use build_info_build::DependencyDepth;
+
+fn main() {
+    build_info_build::build_script().collect_dependencies(DependencyDepth::Depth(1));
+}