diff --git a/Cargo.lock b/Cargo.lock index c3c7832..c44ae9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -135,9 +135,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" [[package]] name = "arrayref" @@ -237,9 +237,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", @@ -468,9 +468,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.15" +version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" dependencies = [ "shlex", ] @@ -526,9 +526,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -536,9 +536,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstream", "anstyle", @@ -625,9 +625,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -1715,9 +1715,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", @@ -1864,14 +1864,14 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "iroh" -version = "0.23.0" -source = "git+https://github.com/izihawa/iroh?branch=main#6a145f381171b228c4463789b5494f3b078370a6" +version = "0.24.0" +source = "git+https://github.com/izihawa/iroh?branch=main#922db9642f1bff297156025383e5d71d1dbf5d97" dependencies = [ "anyhow", "async-channel", @@ -1914,8 +1914,8 @@ dependencies = [ [[package]] name = "iroh-base" -version = "0.23.0" -source = "git+https://github.com/izihawa/iroh?branch=main#6a145f381171b228c4463789b5494f3b078370a6" +version = "0.24.0" +source = "git+https://github.com/izihawa/iroh?branch=main#922db9642f1bff297156025383e5d71d1dbf5d97" dependencies = [ "aead", "anyhow", @@ -1955,8 +1955,8 @@ dependencies = [ [[package]] name = "iroh-blobs" -version = "0.23.0" -source = "git+https://github.com/izihawa/iroh?branch=main#6a145f381171b228c4463789b5494f3b078370a6" +version = "0.24.0" +source = "git+https://github.com/izihawa/iroh?branch=main#922db9642f1bff297156025383e5d71d1dbf5d97" dependencies = [ "anyhow", "async-channel", @@ -1997,8 +1997,8 @@ dependencies = [ [[package]] name = "iroh-docs" -version = "0.23.0" -source = "git+https://github.com/izihawa/iroh?branch=main#6a145f381171b228c4463789b5494f3b078370a6" +version = "0.24.0" +source = "git+https://github.com/izihawa/iroh?branch=main#922db9642f1bff297156025383e5d71d1dbf5d97" dependencies = [ "anyhow", "async-channel", @@ -2035,8 +2035,8 @@ dependencies = [ [[package]] name = "iroh-gossip" -version = "0.23.0" -source = "git+https://github.com/izihawa/iroh?branch=main#6a145f381171b228c4463789b5494f3b078370a6" +version = "0.24.0" +source = "git+https://github.com/izihawa/iroh?branch=main#922db9642f1bff297156025383e5d71d1dbf5d97" dependencies = [ "anyhow", "async-channel", @@ -2062,9 +2062,9 @@ dependencies = [ [[package]] name = "iroh-io" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d1047ad5ca29ab4ff316b6830d86e7ea52cea54325e4d4a849692e1274b498" +checksum = "17e302c5ad649c6a7aa9ae8468e1c4dc2469321af0c6de7341c1be1bdaab434b" dependencies = [ "bytes", "futures-lite 2.3.0", @@ -2075,8 +2075,8 @@ dependencies = [ [[package]] name = "iroh-metrics" -version = "0.23.0" -source = "git+https://github.com/izihawa/iroh?branch=main#6a145f381171b228c4463789b5494f3b078370a6" +version = "0.24.0" +source = "git+https://github.com/izihawa/iroh?branch=main#922db9642f1bff297156025383e5d71d1dbf5d97" dependencies = [ "anyhow", "erased_set", @@ -2095,8 +2095,8 @@ dependencies = [ [[package]] name = "iroh-net" -version = "0.23.0" -source = "git+https://github.com/izihawa/iroh?branch=main#6a145f381171b228c4463789b5494f3b078370a6" +version = "0.24.0" +source = "git+https://github.com/izihawa/iroh?branch=main#922db9642f1bff297156025383e5d71d1dbf5d97" dependencies = [ "anyhow", "backoff", @@ -2138,7 +2138,6 @@ dependencies = [ "pkarr", "postcard", "rand 0.8.5", - "rand_core 0.6.4", "rcgen", "reqwest", "ring", @@ -2780,9 +2779,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -2840,9 +2839,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.11" +version = "2.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +checksum = "9c73c26c01b8c87956cea613c907c9d6ecffd8d18a2a5908e5de0adfaa185cea" dependencies = [ "memchr", "thiserror", @@ -2851,9 +2850,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.11" +version = "2.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +checksum = "664d22978e2815783adbdd2c588b455b1bd625299ce36b2a99881ac9627e6d8d" dependencies = [ "pest", "pest_generator", @@ -2861,9 +2860,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.11" +version = "2.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +checksum = "a2d5487022d5d33f4c30d91c22afa240ce2a644e87fe08caad974d4eab6badbe" dependencies = [ "pest", "pest_meta", @@ -2874,9 +2873,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.11" +version = "2.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +checksum = "0091754bbd0ea592c4deb3a122ce8ecbb0753b738aa82bc055fcc2eccc8d8174" dependencies = [ "once_cell", "pest", @@ -3203,9 +3202,9 @@ dependencies = [ [[package]] name = "quic-rpc" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cb85690ab1688eade9a5de4d94545a9ceef60639b3370f5e1a28f525eb5589" +checksum = "77b3ae032988c83ac7ce072be3c2605e90671e8830a34d6ebf24095add68b4d5" dependencies = [ "anyhow", "bincode", @@ -3228,9 +3227,9 @@ dependencies = [ [[package]] name = "quic-rpc-derive" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6150a9fd3cf6c34d25730fe55a247b99d1c6e4fad6e7b7843f729a431a57e919" +checksum = "59e222c3c9829e6e0de27e5a8fb34047cb31c39093fc00746d30457167a44252" dependencies = [ "proc-macro2", "quic-rpc", @@ -3246,9 +3245,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quinn" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", @@ -3264,9 +3263,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand 0.8.5", @@ -3281,15 +3280,15 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3668,9 +3667,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.35" +version = "0.38.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" +checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" dependencies = [ "bitflags 2.6.0", "errno", @@ -3793,11 +3792,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3858,9 +3857,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -3895,9 +3894,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -3906,9 +3905,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", "memchr", @@ -4518,9 +4517,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -4569,9 +4568,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index caace45..6a205e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,8 +17,8 @@ async-stream = "0.3" axum = "0.7" bisection = "0.1" clap = { version = "4.5", features = ["derive"] } -iroh = { version = "0.23.0", branch = "main", git = "https://github.com/izihawa/iroh", features = [ "metrics" ] } -iroh-base = { version = "0.23.0", branch = "main", git = "https://github.com/izihawa/iroh" } +iroh = { version = "0.24.0", branch = "main", git = "https://github.com/izihawa/iroh", features = [ "metrics" ] } +iroh-base = { version = "0.24.0", branch = "main", git = "https://github.com/izihawa/iroh" } md5 = "0.7" percent-encoding = "2.3.1" rand = "0.9.0-alpha.1" @@ -37,7 +37,7 @@ lru = "0.12" tokio-task-pool = "0.1" url = "2.5.0" mime_guess = "2.0.4" -bytes = "1.6.0" -hyper = "1.3.1" +bytes = "1.7.1" +hyper = "1.4.1" headers = "0.4.0" range-collections = "0.4.5" diff --git a/src/config.rs b/src/config.rs index dd5a3f4..b0cef0d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,6 +7,13 @@ use std::path::{Path, PathBuf}; fn return_0() -> u32 { 0 } +fn return_default_address() -> String { + "0.0.0.0".to_string() +} + +fn return_false() -> bool { + false +} #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct StorageEngineConfig { @@ -34,6 +41,8 @@ pub struct TableConfig { #[serde(default = "DownloadPolicy::default")] pub download_policy: DownloadPolicy, pub storage_name: Option, + #[serde(default = "return_false")] + pub safe_mode: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -57,6 +66,8 @@ pub struct IrohConfig { #[serde(default = "HashMap::new")] pub tables: HashMap, pub path: PathBuf, + #[serde(default = "return_default_address")] + pub bind_address: String, pub bind_port: u16, #[serde(default = "HashMap::new")] pub storages: HashMap, @@ -78,6 +89,7 @@ impl Config { author: None, tables: Default::default(), path: base_path.join("iroh").to_path_buf(), + bind_address: "0.0.0.0".to_string(), bind_port: 11204, storages: HashMap::from_iter(vec![( "default".to_string(), diff --git a/src/iroh_node.rs b/src/iroh_node.rs index e6a757c..85d3829 100644 --- a/src/iroh_node.rs +++ b/src/iroh_node.rs @@ -14,6 +14,7 @@ use iroh_base::hash::Hash; use iroh_base::node_addr::NodeAddr; use std::collections::HashMap; use std::fmt::{Debug, Formatter}; +use std::net::{Ipv4Addr, SocketAddrV4}; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; @@ -71,7 +72,10 @@ impl IrohNode { .await .map_err(Error::node_create)? .relay_mode(relay_mode) - .bind_port(config_lock.iroh.bind_port); + .bind_addr_v4(SocketAddrV4::new( + Ipv4Addr::from_str(&config_lock.iroh.bind_address).map_err(Error::node_create)?, + config_lock.iroh.bind_port, + )); if let Some(gc_interval_secs) = config_lock.iroh.gc_interval_secs { node_builder = @@ -85,7 +89,7 @@ impl IrohNode { .list() .await .map_err(Error::io_error)? - .map(|x| x.expect("Can't extratc document").0) + .map(|x| x.expect("Can't extract document").0) .collect::>() .await { @@ -131,7 +135,6 @@ impl IrohNode { .set_download_policy(table_config.download_policy.clone()) .await .map_err(Error::doc)?; - iroh_doc.start_sync(vec![]).await.map_err(Error::doc)?; let table = Table::new( &table_name, @@ -199,6 +202,7 @@ impl IrohNode { id: iroh_doc.id().to_string(), download_policy: DownloadPolicy::default(), storage_name: storage_name.clone(), + safe_mode: false, }; let table = Table::new( table_name, @@ -276,6 +280,7 @@ impl IrohNode { id: iroh_doc.id().to_string(), download_policy, storage_name: storage_name.clone(), + safe_mode: false, }; let storage_engine = Table::new( table_name, @@ -382,7 +387,7 @@ impl IrohNode { .tables .get(table_name) .ok_or_else(|| Error::missing_table(table_name))?; - table.get(key).await + table.get_with_import(key).await } pub async fn table_delete(&self, table_name: &str, key: &str) -> Result { diff --git a/src/table.rs b/src/table.rs index cd8b7da..84315b8 100644 --- a/src/table.rs +++ b/src/table.rs @@ -238,33 +238,7 @@ impl Table { if import_threads_task_tracker0.is_closed() { return } - let join_handle = import_threads_task_tracker0.spawn(async move { - let iroh_doc = iroh_doc.clone(); - let import_progress = match iroh_doc - .import_file( - table.author_id, - key, - &entry.path(), - true, - ) - .await - .map_err(Error::doc) { - Ok(import_progress) => import_progress, - Err(error) => { - error!(error = ?error, path = ?entry.path(), key = ?entry.file_name(), "import_progress_error"); - return; - } - }; - if let Err(error) = import_progress.finish().await.map_err(Error::storage) { - error!( - error = ?error, - path = ?entry.path(), - key = ?entry.file_name(), - "import_progress_error" - ); - } - info!(action = "imported", key = ?entry.file_name()) - }.instrument(info_span!(parent: None, "import_missing", shard = ?base_path))); + let join_handle = import_threads_task_tracker0.spawn(Table::import_existing_file(iroh_doc.clone(), table.author_id.clone(), key, entry.path()).instrument(info_span!(parent: None, "import_missing", shard = ?base_path))); if let Err(error) = join_handle.await { error!( error = ?error, @@ -289,6 +263,35 @@ impl Table { Ok(table) } + async fn import_existing_file( + iroh_doc: iroh::client::Doc, + author_id: AuthorId, + key_bytes: Bytes, + path: PathBuf, + ) -> bool { + let import_progress = match iroh_doc + .import_file(author_id, key_bytes, &path, true) + .await + .map_err(Error::doc) + { + Ok(import_progress) => import_progress, + Err(error) => { + error!(error = ?error, path = ?path, "import_existing_file_progress_error"); + return false; + } + }; + if let Err(error) = import_progress.finish().await.map_err(Error::storage) { + error!( + error = ?error, + path = ?path, + "import_existing_file_progress_error" + ); + return false; + } + info!(action = "imported_existing_file"); + return true; + } + async fn process_remote_entry(&self, key: &str, entry: &Entry) -> Result> { let Some(storage) = &self.storage else { return Ok(None); @@ -427,6 +430,29 @@ impl Table { &self.iroh_doc } + pub async fn get_with_import(&self, key: &str) -> Result> { + match ( + self.get(key).await, + &self.storage, + self.table_config.safe_mode, + ) { + (Ok(None), Some(storage), true) => { + if Table::import_existing_file( + self.iroh_doc.clone(), + self.author_id.clone(), + key_to_bytes(key), + storage.get_path(key).unwrap(), + ) + .await + { + return self.get(key).await; + } + return Ok(None); + } + (r, _, _) => r, + } + } + pub async fn get(&self, key: &str) -> Result> { let entry = self .iroh_doc @@ -435,7 +461,6 @@ impl Table { ) .await .map_err(Error::entry)?; - if let Some(entry) = entry { let db_entry = self .node