diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e63209f4f0..a147b81421 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,9 +103,9 @@ jobs: baseline-rev: ${{ env.HEAD_COMMIT_SHA }} use-cache: false - check_fmt_and_docs: + check_fmt: timeout-minutes: 30 - name: Checking fmt and docs + name: Checking fmt runs-on: ubuntu-latest env: RUSTC_WRAPPER: "sccache" @@ -123,8 +123,25 @@ jobs: - name: fmt run: cargo fmt --all -- --check + check_docs: + timeout-minutes: 30 + name: Checking docs + runs-on: ubuntu-latest + env: + RUSTC_WRAPPER: "sccache" + SCCACHE_GHA_ENABLED: "on" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2024-05-02 + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.4 + - name: Docs run: cargo doc --workspace --all-features --no-deps --document-private-items + env: + RUSTDOCFLAGS: --cfg docsrs clippy_check: timeout-minutes: 30 diff --git a/Cargo.lock b/Cargo.lock index 14b11ba85c..8fb2f0ec7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,47 +78,48 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.13" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -411,9 +412,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64-url" @@ -547,9 +548,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd" [[package]] name = "cfg-if" @@ -680,9 +681,9 @@ checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" [[package]] name = "colored" @@ -708,9 +709,9 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ "crossbeam-utils", ] @@ -1029,7 +1030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -1037,9 +1038,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "der" @@ -1442,9 +1443,9 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fastrand" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fd-lock" @@ -1792,9 +1793,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", @@ -1806,7 +1807,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -2232,7 +2233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "serde", ] @@ -2758,6 +2759,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itertools" version = "0.10.5" @@ -2793,9 +2800,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "libm" @@ -2847,7 +2854,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -3420,7 +3427,7 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "serde", ] @@ -4013,9 +4020,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.0.1" +version = "11.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +checksum = "e29830cbb1290e404f24c73af91c5d8d631ce7e128691e9477556b540cd01ecd" dependencies = [ "bitflags 2.5.0", ] @@ -4367,9 +4374,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.8", @@ -4404,7 +4411,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "rustls-pki-types", ] @@ -4568,9 +4575,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.198" +version = "1.0.200" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f" dependencies = [ "serde_derive", ] @@ -4605,9 +4612,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.198" +version = "1.0.200" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb" dependencies = [ "proc-macro2", "quote", @@ -4667,11 +4674,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c85f8e96d1d6857f13768fcbd895fcb06225510022a2774ed8b5150581847b0" +checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", @@ -4685,9 +4692,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8b3a576c4eb2924262d5951a3b737ccaf16c931e39a2810c36f9a7e25575557" +checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" dependencies = [ "darling", "proc-macro2", @@ -4815,9 +4822,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -5025,7 +5032,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0adebf9fb8fba5c39ee34092b0383f247e4d1255b98fcffec94b4b797b85b677" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "bounded-integer", "byteorder", "crc", @@ -5421,7 +5428,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "pin-project-lite", "slab", "tokio", @@ -5471,7 +5478,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.6", + "winnow 0.6.7", ] [[package]] @@ -5691,9 +5698,9 @@ checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" [[package]] name = "unicode-xid" @@ -6220,9 +6227,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c976aaaa0e1f90dbb21e9587cdaf1d9679a1cde8875c0d6bd83ab96a208352" +checksum = "14b9415ee827af173ebb3f15f9083df5a122eb93572ec28741fb153356ea2578" dependencies = [ "memchr", ] diff --git a/iroh-base/Cargo.toml b/iroh-base/Cargo.toml index f729ec5c1f..e571677d42 100644 --- a/iroh-base/Cargo.toml +++ b/iroh-base/Cargo.toml @@ -27,7 +27,7 @@ thiserror = "1" # key module aead = { version = "0.5.2", features = ["bytes"], optional = true } -derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from_str"], optional = true } +derive_more = { version = "1.0.0-beta.6", features = ["debug", "display", "from_str"], optional = true } ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"], optional = true } once_cell = { version = "1.18.0", optional = true } rand = { version = "0.8", optional = true } @@ -50,3 +50,6 @@ hash = ["dep:bao-tree", "dep:data-encoding", "dep:postcard", "dep:derive_more"] base32 = ["dep:data-encoding"] redb = ["dep:redb"] key = ["dep:ed25519-dalek", "dep:once_cell", "dep:rand", "dep:rand_core", "dep:ssh-key", "dep:ttl_cache", "dep:aead", "dep:crypto_box", "dep:zeroize", "dep:url", "dep:derive_more"] + +[package.metadata.docs.rs] +all-features = true diff --git a/iroh-bytes/Cargo.toml b/iroh-bytes/Cargo.toml index 90cc268ffe..42cd1b6787 100644 --- a/iroh-bytes/Cargo.toml +++ b/iroh-bytes/Cargo.toml @@ -20,7 +20,7 @@ anyhow = { version = "1" } bao-tree = { version = "0.13", features = ["tokio_fsm"], default-features = false } bytes = { version = "1.4", features = ["serde"] } chrono = "0.4.31" -derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "deref", "deref_mut", "from", "try_into", "into"] } +derive_more = { version = "1.0.0-beta.6", features = ["debug", "display", "deref", "deref_mut", "from", "try_into", "into"] } flume = "0.11" futures-buffered = "0.2.4" futures-lite = "2.3" @@ -72,6 +72,9 @@ fs-store = ["dep:reflink-copy", "redb", "dep:redb_v1", "dep:tempfile"] metrics = ["dep:iroh-metrics"] redb = ["dep:redb"] +[package.metadata.docs.rs] +all-features = true + [[example]] name = "provide-bytes" diff --git a/iroh-bytes/src/lib.rs b/iroh-bytes/src/lib.rs index baad1bc7a0..d482faf215 100644 --- a/iroh-bytes/src/lib.rs +++ b/iroh-bytes/src/lib.rs @@ -1,4 +1,4 @@ -//! Send data over the internet. +//! Blobs layer for iroh. #![deny(missing_docs, rustdoc::broken_intra_doc_links)] #![recursion_limit = "256"] diff --git a/iroh-cli/Cargo.toml b/iroh-cli/Cargo.toml index faf2794c58..bb6eaec942 100644 --- a/iroh-cli/Cargo.toml +++ b/iroh-cli/Cargo.toml @@ -29,7 +29,7 @@ clap = { version = "4", features = ["derive"] } colored = "2.0.4" comfy-table = "7.0.1" console = "0.15.5" -derive_more = { version = "1.0.0-beta.1", features = ["display"] } +derive_more = { version = "1.0.0-beta.6", features = ["display"] } dialoguer = { version = "0.11.0", default-features = false } dirs-next = "2.0.0" flume = "0.11.0" diff --git a/iroh-cli/src/commands.rs b/iroh-cli/src/commands.rs index b2fd8c42e4..9fd279be12 100644 --- a/iroh-cli/src/commands.rs +++ b/iroh-cli/src/commands.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::{ensure, Context, Result}; use clap::Parser; use derive_more::FromStr; -use iroh::client::quic::Iroh as IrohRpc; +use iroh::client::QuicIroh; use crate::config::{ConsoleEnv, NodeConfig}; @@ -127,7 +127,7 @@ impl Cli { .await } else { crate::logging::init_terminal_logging()?; - let iroh = IrohRpc::connect(data_dir).await.context("rpc connect")?; + let iroh = QuicIroh::connect(data_dir).await.context("rpc connect")?; console::run(&iroh, &env).await } } @@ -144,7 +144,7 @@ impl Cli { .await } else { crate::logging::init_terminal_logging()?; - let iroh = IrohRpc::connect(data_dir).await.context("rpc connect")?; + let iroh = QuicIroh::connect(data_dir).await.context("rpc connect")?; command.run(&iroh, &env).await } } diff --git a/iroh-cli/src/commands/author.rs b/iroh-cli/src/commands/author.rs index 64a02dd603..eed5e8bab0 100644 --- a/iroh-cli/src/commands/author.rs +++ b/iroh-cli/src/commands/author.rs @@ -4,8 +4,8 @@ use derive_more::FromStr; use futures_lite::StreamExt; use iroh::base::base32::fmt_short; +use iroh::client::{Iroh, RpcService}; use iroh::sync::{Author, AuthorId}; -use iroh::{client::Iroh, rpc_protocol::ProviderService}; use quic_rpc::ServiceConnection; use crate::config::ConsoleEnv; @@ -34,7 +34,7 @@ pub enum AuthorCommands { impl AuthorCommands { pub async fn run(self, iroh: &Iroh, env: &ConsoleEnv) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { match self { Self::Switch { author } => { diff --git a/iroh-cli/src/commands/blob.rs b/iroh-cli/src/commands/blob.rs index 875b703cbb..ed26789877 100644 --- a/iroh-cli/src/commands/blob.rs +++ b/iroh-cli/src/commands/blob.rs @@ -13,25 +13,26 @@ use indicatif::{ HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressState, ProgressStyle, }; -use iroh::net::{key::PublicKey, relay::RelayUrl, NodeAddr}; use iroh::{ base::node_addr::AddrInfoOptions, + base::ticket::BlobTicket, bytes::{ get::{db::DownloadProgress, progress::BlobProgress, Stats}, provider::AddProgress, store::{ ConsistencyCheckProgress, ExportFormat, ExportMode, ReportLevel, ValidateProgress, }, + util::SetTagOption, BlobFormat, Hash, HashAndFormat, Tag, }, -}; -use iroh::{ - client::{BlobStatus, Iroh}, - rpc_protocol::{ - BlobDownloadRequest, BlobListCollectionsResponse, BlobListIncompleteResponse, - BlobListResponse, DownloadMode, ProviderService, SetTagOption, WrapOption, + client::{ + blobs::{ + BlobInfo, BlobStatus, CollectionInfo, DownloadMode, DownloadOptions, + IncompleteBlobInfo, WrapOption, + }, + Iroh, RpcService, }, - ticket::BlobTicket, + net::{key::PublicKey, relay::RelayUrl, NodeAddr}, }; use quic_rpc::ServiceConnection; use tokio::io::AsyncWriteExt; @@ -183,7 +184,7 @@ impl std::str::FromStr for TicketOrHash { impl BlobCommands { pub async fn run(self, iroh: &Iroh) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { match self { Self::Get { @@ -262,13 +263,15 @@ impl BlobCommands { let mut stream = iroh .blobs - .download(BlobDownloadRequest { + .download_with_opts( hash, - format, - nodes: vec![node_addr], - tag, - mode, - }) + DownloadOptions { + format, + nodes: vec![node_addr], + tag, + mode, + }, + ) .await?; show_download_progress(hash, &mut stream).await?; @@ -446,27 +449,27 @@ pub enum ListCommands { impl ListCommands { pub async fn run(self, iroh: &Iroh) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { match self { Self::Blobs => { let mut response = iroh.blobs.list().await?; while let Some(item) = response.next().await { - let BlobListResponse { path, hash, size } = item?; + let BlobInfo { path, hash, size } = item?; println!("{} {} ({})", path, hash, HumanBytes(size)); } } Self::IncompleteBlobs => { let mut response = iroh.blobs.list_incomplete().await?; while let Some(item) = response.next().await { - let BlobListIncompleteResponse { hash, size, .. } = item?; + let IncompleteBlobInfo { hash, size, .. } = item?; println!("{} ({})", hash, HumanBytes(size)); } } Self::Collections => { let mut response = iroh.blobs.list_collections().await?; while let Some(item) = response.next().await { - let BlobListCollectionsResponse { + let CollectionInfo { tag, hash, total_blobs_count, @@ -506,7 +509,7 @@ pub enum DeleteCommands { impl DeleteCommands { pub async fn run(self, iroh: &Iroh) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { match self { Self::Blob { hash } => { @@ -539,7 +542,7 @@ fn apply_report_level(text: String, level: ReportLevel) -> console::StyledObject pub async fn consistency_check(iroh: &Iroh, verbose: u8, repair: bool) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { let mut response = iroh.blobs.consistency_check(repair).await?; let verbosity = get_report_level(verbose); @@ -583,7 +586,7 @@ where pub async fn validate(iroh: &Iroh, verbose: u8, repair: bool) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { let mut state = ValidateProgressState::new(); let mut response = iroh.blobs.validate(repair).await?; @@ -804,7 +807,7 @@ pub enum TicketOption { Print, } -pub async fn add_with_opts>( +pub async fn add_with_opts>( client: &iroh::client::Iroh, source: BlobSource, opts: BlobAddOptions, @@ -837,7 +840,7 @@ pub async fn add_with_opts>( } /// Add data to iroh, either from a path or, if path is `None`, from STDIN. -pub async fn add>( +pub async fn add>( client: &iroh::client::Iroh, source: BlobSourceIroh, tag: SetTagOption, diff --git a/iroh-cli/src/commands/console.rs b/iroh-cli/src/commands/console.rs index b7f9178ce1..f087e6d490 100644 --- a/iroh-cli/src/commands/console.rs +++ b/iroh-cli/src/commands/console.rs @@ -2,7 +2,7 @@ use anyhow::Result; use clap::{Parser, Subcommand}; use colored::Colorize; use iroh::base::base32::fmt_short; -use iroh::{client::Iroh, rpc_protocol::ProviderService}; +use iroh::client::{Iroh, RpcService}; use quic_rpc::ServiceConnection; use rustyline::{error::ReadlineError, Config, DefaultEditor}; use tokio::sync::{mpsc, oneshot}; @@ -14,7 +14,7 @@ use crate::{ pub async fn run(iroh: &Iroh, env: &ConsoleEnv) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { println!("{}", "Welcome to the Iroh console!".purple().bold()); println!("Type `{}` for a list of commands.", "help".bold()); diff --git a/iroh-cli/src/commands/doc.rs b/iroh-cli/src/commands/doc.rs index 6320160d2a..fcdb3af5c0 100644 --- a/iroh-cli/src/commands/doc.rs +++ b/iroh-cli/src/commands/doc.rs @@ -13,20 +13,21 @@ use dialoguer::Confirm; use futures_buffered::BufferedStreamExt; use futures_lite::{Stream, StreamExt}; use indicatif::{HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressStyle}; -use iroh::base::{base32::fmt_short, node_addr::AddrInfoOptions}; use quic_rpc::ServiceConnection; -use serde::{Deserialize, Serialize}; use tokio::io::AsyncReadExt; -use iroh::bytes::{provider::AddProgress, Hash, Tag}; -use iroh::sync::{ - store::{DownloadPolicy, FilterKind, Query, SortDirection}, - AuthorId, NamespaceId, -}; use iroh::{ - client::{Doc, Entry, Iroh, LiveEvent}, - rpc_protocol::{DocTicket, ProviderService, SetTagOption, WrapOption}, - sync_engine::Origin, + base::{base32::fmt_short, node_addr::AddrInfoOptions}, + bytes::{provider::AddProgress, util::SetTagOption, Hash, Tag}, + client::{ + blobs::WrapOption, + docs::{Doc, Entry, LiveEvent, Origin, ShareMode}, + Iroh, RpcService, + }, + sync::{ + store::{DownloadPolicy, FilterKind, Query, SortDirection}, + AuthorId, DocTicket, NamespaceId, + }, util::fs::{path_content_info, path_to_key, PathContent}, }; @@ -112,6 +113,7 @@ pub enum DocCommands { /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] doc: Option, + /// The sharing mode. mode: ShareMode, /// Options to configure the address information in the generated ticket. /// @@ -282,24 +284,6 @@ pub enum DocCommands { }, } -/// Intended capability for document share tickets -#[derive(Serialize, Deserialize, Debug, Clone, clap::ValueEnum)] -pub enum ShareMode { - /// Read-only access - Read, - /// Write access - Write, -} - -impl From for iroh::rpc_protocol::ShareMode { - fn from(value: ShareMode) -> Self { - match value { - ShareMode::Read => iroh::rpc_protocol::ShareMode::Read, - ShareMode::Write => iroh::rpc_protocol::ShareMode::Write, - } - } -} - #[derive(clap::ValueEnum, Clone, Debug, Default, strum::Display)] #[strum(serialize_all = "kebab-case")] pub enum Sorting { @@ -321,7 +305,7 @@ impl From for iroh::sync::store::SortBy { impl DocCommands { pub async fn run(self, iroh: &Iroh, env: &ConsoleEnv) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { match self { Self::Switch { id: doc } => { @@ -366,7 +350,7 @@ impl DocCommands { addr_options, } => { let doc = get_doc(iroh, env, doc).await?; - let ticket = doc.share(mode.into(), addr_options).await?; + let ticket = doc.share(mode, addr_options).await?; println!("{}", ticket); } Self::Set { @@ -692,7 +676,7 @@ async fn get_doc( id: Option, ) -> anyhow::Result> where - C: ServiceConnection, + C: ServiceConnection, { iroh.docs .open(env.doc(id)?) @@ -707,7 +691,7 @@ async fn fmt_content( mode: DisplayContentMode, ) -> Result where - C: ServiceConnection, + C: ServiceConnection, { let read_failed = |err: anyhow::Error| format!(""); let encode_hex = |err: std::string::FromUtf8Error| format!("0x{}", hex::encode(err.as_bytes())); @@ -758,7 +742,7 @@ fn human_len(entry: &Entry) -> HumanBytes { #[must_use = "this won't be printed, you need to print it yourself"] async fn fmt_entry(doc: &Doc, entry: &Entry, mode: DisplayContentMode) -> String where - C: ServiceConnection, + C: ServiceConnection, { let key = std::str::from_utf8(entry.key()) .unwrap_or("") @@ -799,7 +783,7 @@ async fn import_coordinator( expected_entries: u64, ) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { let imp = ImportProgressBar::new( &root.display().to_string(), @@ -993,7 +977,7 @@ mod tests { // set up command, getting iroh node let cli = ConsoleEnv::for_console(data_dir.path()).context("ConsoleEnv")?; - let iroh = iroh::client::quic::Iroh::connect(data_dir.path()) + let iroh = iroh::client::QuicIroh::connect(data_dir.path()) .await .context("rpc connect")?; diff --git a/iroh-cli/src/commands/doctor.rs b/iroh-cli/src/commands/doctor.rs index f02aa1b622..93c491186f 100644 --- a/iroh-cli/src/commands/doctor.rs +++ b/iroh-cli/src/commands/doctor.rs @@ -19,7 +19,7 @@ use derive_more::Display; use futures_lite::StreamExt; use indicatif::{HumanBytes, MultiProgress, ProgressBar}; use iroh::{ - base::ticket::Ticket, + base::ticket::{BlobTicket, Ticket}, bytes::{ store::{ReadableStore, Store as _}, util::progress::{FlumeProgressSender, ProgressSender}, @@ -33,10 +33,11 @@ use iroh::{ key::{PublicKey, SecretKey}, magic_endpoint, netcheck, portmapper, relay::{RelayMap, RelayMode, RelayUrl}, + ticket::NodeTicket, util::AbortingJoinHandle, MagicEndpoint, NodeAddr, NodeId, }, - sync::Capability, + sync::{Capability, DocTicket}, util::{path::IrohPaths, progress::ProgressWriter}, }; use portable_atomic::AtomicU64; @@ -990,17 +991,15 @@ fn print_node_addr(prefix: &str, node_addr: &NodeAddr, zbase32: bool) { } fn inspect_ticket(ticket: &str, zbase32: bool) -> anyhow::Result<()> { - if ticket.starts_with(iroh::ticket::BlobTicket::KIND) { - let ticket = - iroh::ticket::BlobTicket::from_str(ticket).context("failed parsing blob ticket")?; + if ticket.starts_with(BlobTicket::KIND) { + let ticket = BlobTicket::from_str(ticket).context("failed parsing blob ticket")?; println!("BlobTicket"); println!(" hash: {}", bold(ticket.hash())); println!(" format: {}", bold(ticket.format())); println!(" NodeInfo"); print_node_addr(" ", ticket.node_addr(), zbase32); - } else if ticket.starts_with(iroh::ticket::DocTicket::KIND) { - let ticket = - iroh::ticket::DocTicket::from_str(ticket).context("failed parsing doc ticket")?; + } else if ticket.starts_with(DocTicket::KIND) { + let ticket = DocTicket::from_str(ticket).context("failed parsing doc ticket")?; println!("DocTicket:\n"); match ticket.capability { Capability::Read(namespace) => { @@ -1013,13 +1012,10 @@ fn inspect_ticket(ticket: &str, zbase32: bool) -> anyhow::Result<()> { for node in &ticket.nodes { print_node_addr(" ", node, zbase32); } - } else if ticket.starts_with(iroh::ticket::NodeTicket::KIND) { - let ticket = - iroh::ticket::NodeTicket::from_str(ticket).context("failed parsing node ticket")?; + } else if ticket.starts_with(NodeTicket::KIND) { + let ticket = NodeTicket::from_str(ticket).context("failed parsing node ticket")?; println!("NodeTicket"); print_node_addr(" ", ticket.node_addr(), zbase32); - } else { - println!("Unknown ticket type"); } Ok(()) diff --git a/iroh-cli/src/commands/node.rs b/iroh-cli/src/commands/node.rs index d8f6bd499f..cd4eac1712 100644 --- a/iroh-cli/src/commands/node.rs +++ b/iroh-cli/src/commands/node.rs @@ -8,11 +8,11 @@ use comfy_table::{presets::NOTHING, Cell}; use futures_lite::{Stream, StreamExt}; use human_time::ToHumanTimeString; use iroh::client::Iroh; +use iroh::client::RpcService; use iroh::net::{ key::PublicKey, magic_endpoint::{ConnectionInfo, DirectAddrInfo}, }; -use iroh::rpc_protocol::ProviderService; use quic_rpc::ServiceConnection; #[derive(Subcommand, Debug, Clone)] @@ -40,7 +40,7 @@ pub enum NodeCommands { impl NodeCommands { pub async fn run(self, iroh: &Iroh) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { match self { Self::Connections => { diff --git a/iroh-cli/src/commands/rpc.rs b/iroh-cli/src/commands/rpc.rs index 036ecf08c7..1d26121681 100644 --- a/iroh-cli/src/commands/rpc.rs +++ b/iroh-cli/src/commands/rpc.rs @@ -1,6 +1,6 @@ use anyhow::Result; use clap::Subcommand; -use iroh::{client::Iroh, rpc_protocol::ProviderService}; +use iroh::client::{Iroh, RpcService}; use quic_rpc::ServiceConnection; use crate::config::ConsoleEnv; @@ -60,7 +60,7 @@ pub enum RpcCommands { impl RpcCommands { pub async fn run(self, iroh: &Iroh, env: &ConsoleEnv) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { match self { Self::Node { command } => command.run(iroh).await, diff --git a/iroh-cli/src/commands/start.rs b/iroh-cli/src/commands/start.rs index 36a3db1480..19473ae7b0 100644 --- a/iroh-cli/src/commands/start.rs +++ b/iroh-cli/src/commands/start.rs @@ -33,7 +33,7 @@ pub async fn run_with_command( command: F, ) -> Result<()> where - F: FnOnce(iroh::client::mem::Iroh) -> T + Send + 'static, + F: FnOnce(iroh::client::MemIroh) -> T + Send + 'static, T: Future> + 'static, { let _guard = crate::logging::init_terminal_and_file_logging(&config.file_logs, iroh_data_root)?; @@ -68,7 +68,7 @@ async fn run_with_command_inner( command: F, ) -> Result<()> where - F: FnOnce(iroh::client::mem::Iroh) -> T + Send + 'static, + F: FnOnce(iroh::client::MemIroh) -> T + Send + 'static, T: Future> + 'static, { let relay_map = config.relay_map()?; diff --git a/iroh-cli/src/commands/tag.rs b/iroh-cli/src/commands/tag.rs index 69edf005c3..0992d9abad 100644 --- a/iroh-cli/src/commands/tag.rs +++ b/iroh-cli/src/commands/tag.rs @@ -3,7 +3,7 @@ use bytes::Bytes; use clap::Subcommand; use futures_lite::StreamExt; use iroh::bytes::Tag; -use iroh::{client::Iroh, rpc_protocol::ProviderService}; +use iroh::client::{Iroh, RpcService}; use quic_rpc::ServiceConnection; #[derive(Subcommand, Debug, Clone)] @@ -22,14 +22,14 @@ pub enum TagCommands { impl TagCommands { pub async fn run(self, iroh: &Iroh) -> Result<()> where - C: ServiceConnection, + C: ServiceConnection, { match self { Self::List => { let mut response = iroh.tags.list().await?; while let Some(res) = response.next().await { let res = res?; - println!("{}: {} ({:?})", res.name, res.hash, res.format,); + println!("{}: {} ({:?})", res.name, res.hash, res.format); } } Self::Delete { tag, hex } => { diff --git a/iroh-cli/tests/cli.rs b/iroh-cli/tests/cli.rs index 0663459a3e..e08e349abc 100644 --- a/iroh-cli/tests/cli.rs +++ b/iroh-cli/tests/cli.rs @@ -10,10 +10,11 @@ use std::str::FromStr; use anyhow::{Context, Result}; use bao_tree::blake3; use duct::{cmd, ReaderHandle}; -use iroh::bytes::Hash; -use iroh::bytes::HashAndFormat; -use iroh::ticket::BlobTicket; -use iroh::util::path::IrohPaths; +use iroh::{ + base::ticket::BlobTicket, + bytes::{Hash, HashAndFormat}, + util::path::IrohPaths, +}; use rand::distributions::{Alphanumeric, DistString}; use rand::SeedableRng; use regex::Regex; diff --git a/iroh-dns-server/Cargo.toml b/iroh-dns-server/Cargo.toml index 5abf67db80..5b0a23c6cc 100644 --- a/iroh-dns-server/Cargo.toml +++ b/iroh-dns-server/Cargo.toml @@ -17,7 +17,7 @@ axum-server = { version = "0.6.0", features = ["tls-rustls"] } base64-url = "2.0.2" bytes = "1.5.0" clap = { version = "4.5.1", features = ["derive"] } -derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "into", "from"] } +derive_more = { version = "1.0.0-beta.6", features = ["debug", "display", "into", "from"] } dirs-next = "2.0.0" futures-lite = "2.3.0" governor = "0.6.3" @@ -27,7 +27,7 @@ http = "1.0.0" iroh-metrics = { version = "0.15.0", path = "../iroh-metrics" } lru = "0.12.3" parking_lot = "0.12.1" -pkarr = { version = "1.1.4", features = [ "async", "relay", "dht"], default_features = false } +pkarr = { version = "1.1.4", features = [ "async", "relay", "dht"], default-features = false } rcgen = "0.12.1" redb = "2.0.0" regex = "1.10.3" @@ -55,3 +55,6 @@ hickory-resolver = "0.24.0" iroh-net = { version = "0.15.0", path = "../iroh-net" } iroh-test = { path = "../iroh-test" } mainline = "<1.5.0" + +[package.metadata.docs.rs] +all-features = true diff --git a/iroh-gossip/Cargo.toml b/iroh-gossip/Cargo.toml index cd83f63df5..8cb0ef47f3 100644 --- a/iroh-gossip/Cargo.toml +++ b/iroh-gossip/Cargo.toml @@ -19,7 +19,7 @@ workspace = true anyhow = { version = "1" } blake3 = { package = "iroh-blake3", version = "1.4.5"} bytes = { version = "1.4.0", features = ["serde"] } -derive_more = { version = "1.0.0-beta.1", features = ["add", "debug", "deref", "display", "from", "try_into", "into"] } +derive_more = { version = "1.0.0-beta.6", features = ["add", "debug", "deref", "display", "from", "try_into", "into"] } ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"] } indexmap = "2.0" postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } @@ -52,3 +52,6 @@ net = ["dep:futures-lite", "dep:iroh-net", "dep:quinn", "dep:tokio", "dep:tokio- [[example]] name = "chat" required-features = ["net"] + +[package.metadata.docs.rs] +all-features = true diff --git a/iroh-metrics/Cargo.toml b/iroh-metrics/Cargo.toml index b3bab3346e..3a7679d4c3 100644 --- a/iroh-metrics/Cargo.toml +++ b/iroh-metrics/Cargo.toml @@ -35,3 +35,6 @@ tokio = { version = "1", features = ["io-util", "sync", "rt", "net", "fs", "macr [features] default = ["metrics"] metrics = ["prometheus-client"] + +[package.metadata.docs.rs] +all-features = true diff --git a/iroh-net/Cargo.toml b/iroh-net/Cargo.toml index 0bc25b89a4..4e922c12f5 100644 --- a/iroh-net/Cargo.toml +++ b/iroh-net/Cargo.toml @@ -23,7 +23,7 @@ backoff = "0.4.0" bytes = "1" netdev = "0.25" der = { version = "0.7", features = ["alloc", "derive"] } -derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into", "deref"] } +derive_more = { version = "1.0.0-beta.6", features = ["debug", "display", "from", "try_into", "deref"] } flume = "0.11" futures-buffered = "0.2.4" futures-lite = "2.3" @@ -128,6 +128,9 @@ test-utils = ["axum"] name = "iroh-relay" required-features = ["iroh-relay"] +[package.metadata.docs.rs] +all-features = true + [[example]] name = "listen" diff --git a/iroh-net/src/net/interfaces.rs b/iroh-net/src/net/interfaces.rs index 9595774380..23964b7896 100644 --- a/iroh-net/src/net/interfaces.rs +++ b/iroh-net/src/net/interfaces.rs @@ -326,12 +326,6 @@ pub struct DefaultRouteDetails { /// The interface name. /// It's like "eth0" (Linux), "Ethernet 2" (Windows), "en0" (macOS). pub interface_name: String, - - /// Ppopulated on Windows at least. Longer description of the interface. - pub interface_description: Option, - - /// The index of the interface, `0` means not populated. - pub interface_index: u32, } impl DefaultRouteDetails { diff --git a/iroh-net/src/net/interfaces/bsd.rs b/iroh-net/src/net/interfaces/bsd.rs index d27912d514..51c3f67728 100644 --- a/iroh-net/src/net/interfaces/bsd.rs +++ b/iroh-net/src/net/interfaces/bsd.rs @@ -21,9 +21,7 @@ pub async fn default_route() -> Option { let iface = interfaces.into_iter().find(|i| i.index == idx)?; Some(DefaultRouteDetails { - interface_index: idx, interface_name: iface.name, - interface_description: None, }) } diff --git a/iroh-net/src/net/interfaces/linux.rs b/iroh-net/src/net/interfaces/linux.rs index de15895d15..b502daf9f4 100644 --- a/iroh-net/src/net/interfaces/linux.rs +++ b/iroh-net/src/net/interfaces/linux.rs @@ -57,8 +57,6 @@ async fn default_route_proc() -> Result> { if destination == ZERO_ADDR && mask == ZERO_ADDR { return Ok(Some(DefaultRouteDetails { interface_name: iface.to_string(), - interface_description: Default::default(), - interface_index: Default::default(), })); } } @@ -81,8 +79,6 @@ pub async fn default_route_android_ip_route() -> Result Result> { }; task.abort(); task.await.ok(); - Ok(default.map(|(name, index)| DefaultRouteDetails { + Ok(default.map(|(name, _index)| DefaultRouteDetails { interface_name: name, - interface_description: None, - interface_index: index, })) } @@ -180,8 +174,6 @@ mod tests { // assert!(route.is_some()); if let Some(route) = route { assert!(!route.interface_name.is_empty()); - assert!(route.interface_description.is_none()); - assert_eq!(route.interface_index, 0); } } @@ -199,7 +191,6 @@ mod tests { // assert!(route.is_some()); if let Some(route) = route { assert!(!route.interface_name.is_empty()); - assert!(route.interface_index > 0); } } } diff --git a/iroh-net/src/net/interfaces/windows.rs b/iroh-net/src/net/interfaces/windows.rs index cfbccf7f41..06f118c33b 100644 --- a/iroh-net/src/net/interfaces/windows.rs +++ b/iroh-net/src/net/interfaces/windows.rs @@ -11,8 +11,6 @@ use super::DefaultRouteDetails; #[allow(non_camel_case_types, non_snake_case)] struct Win32_IP4RouteTable { Name: String, - InterfaceIndex: i64, - Description: String, } fn get_default_route() -> anyhow::Result { @@ -26,12 +24,8 @@ fn get_default_route() -> anyhow::Result { .next() .ok_or_else(|| anyhow::anyhow!("no route found"))?; - let idx = route.InterfaceIndex.try_into()?; - Ok(DefaultRouteDetails { - interface_index: idx, interface_name: route.Name, - interface_description: Some(route.Description), }) } diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index 762f521eb0..01279cf3e2 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -17,7 +17,7 @@ workspace = true [dependencies] anyhow = "1" blake3 = { package = "iroh-blake3", version = "1.4.5"} -derive_more = { version = "1.0.0-beta.1", features = ["debug", "deref", "display", "from", "try_into", "into", "as_ref"] } +derive_more = { version = "1.0.0-beta.6", features = ["debug", "deref", "display", "from", "try_into", "into", "as_ref"] } ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"] } flume = "0.11" iroh-base = { version = "0.15.0", path = "../iroh-base" } @@ -60,3 +60,6 @@ test-strategy = "0.3.1" default = ["net", "metrics"] net = ["dep:iroh-net", "tokio/io-util", "dep:tokio-stream", "dep:tokio-util", "dep:quinn", "dep:futures-util"] metrics = ["dep:iroh-metrics"] + +[package.metadata.docs.rs] +all-features = true diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index edca63a39b..7bee08d712 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -33,17 +33,23 @@ //! [paper]: https://arxiv.org/abs/2212.13567 #![deny(missing_docs, rustdoc::broken_intra_doc_links)] -pub mod actor; -mod heads; -mod keys; #[cfg(feature = "metrics")] pub mod metrics; #[cfg(feature = "net")] pub mod net; -mod ranger; +#[cfg(feature = "net")] +mod ticket; + +pub mod actor; pub mod store; pub mod sync; +mod heads; +mod keys; +mod ranger; + pub use self::heads::*; pub use self::keys::*; pub use self::sync::*; +#[cfg(feature = "net")] +pub use self::ticket::DocTicket; diff --git a/iroh/src/ticket/doc.rs b/iroh-sync/src/ticket.rs similarity index 98% rename from iroh/src/ticket/doc.rs rename to iroh-sync/src/ticket.rs index f0f6667918..b9cdad2c7c 100644 --- a/iroh/src/ticket/doc.rs +++ b/iroh-sync/src/ticket.rs @@ -2,9 +2,10 @@ use iroh_base::ticket; use iroh_net::NodeAddr; -use iroh_sync::Capability; use serde::{Deserialize, Serialize}; +use crate::Capability; + /// Contains both a key (either secret or public) to a document, and a list of peers to join. #[derive(Serialize, Deserialize, Clone, Debug, derive_more::Display)] #[display("{}", ticket::Ticket::serialize(self))] @@ -64,10 +65,11 @@ impl std::str::FromStr for DocTicket { mod tests { use std::str::FromStr; + use crate::NamespaceId; + use super::*; use iroh_base::base32; use iroh_net::key::PublicKey; - use iroh_sync::NamespaceId; use iroh_test::{assert_eq_hex, hexdump::parse_hexdump}; #[test] diff --git a/iroh-test/Cargo.toml b/iroh-test/Cargo.toml index 923f5d6ad2..1a60654b6c 100644 --- a/iroh-test/Cargo.toml +++ b/iroh-test/Cargo.toml @@ -20,3 +20,6 @@ anyhow = "1" tokio = { version = "1", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[package.metadata.docs.rs] +all-features = true diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index fc83b1e3c0..03c0f71156 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -19,7 +19,7 @@ workspace = true anyhow = { version = "1" } bao-tree = { version = "0.13", features = ["tokio_fsm"], default-features = false } bytes = "1" -derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into", "from_str"] } +derive_more = { version = "1.0.0-beta.6", features = ["debug", "display", "from", "try_into", "from_str"] } flume = "0.11" futures-buffered = "0.2.4" futures-lite = "2.3" @@ -76,6 +76,12 @@ testdir = "0.9.1" tokio = { version = "1", features = ["macros", "io-util", "rt"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } +[package.metadata.docs.rs] +all-features = true +# enable unstable features in the documentation +rustdoc-args = ["--cfg", "docsrs"] + + [[example]] name = "hello-world-provide" diff --git a/iroh/examples/client.rs b/iroh/examples/client.rs index 7688ac0bdc..0d488c42c2 100644 --- a/iroh/examples/client.rs +++ b/iroh/examples/client.rs @@ -6,9 +6,7 @@ //! run this example from the project root: //! $ cargo run --example client use indicatif::HumanBytes; -use iroh::{client::Entry, node::Node}; -use iroh_base::base32; -use iroh_sync::store::Query; +use iroh::{base::base32, client::docs::Entry, node::Node, sync::store::Query}; use tokio_stream::StreamExt; #[tokio::main] diff --git a/iroh/examples/collection-fetch.rs b/iroh/examples/collection-fetch.rs index 53fac68f52..88a8598c47 100644 --- a/iroh/examples/collection-fetch.rs +++ b/iroh/examples/collection-fetch.rs @@ -3,11 +3,10 @@ //! //! This is using an in memory database and a random node id. //! Run the `collection-provide` example, which will give you instructions on how to run this example. +use std::{env, str::FromStr}; + use anyhow::{bail, ensure, Context, Result}; -use iroh::rpc_protocol::{BlobDownloadRequest, DownloadMode}; -use iroh_bytes::BlobFormat; -use std::env; -use std::str::FromStr; +use iroh::{base::ticket::BlobTicket, bytes::BlobFormat}; use tracing_subscriber::{prelude::*, EnvFilter}; // set the RUST_LOG env var to one of {debug,info,warn} to see logging info @@ -32,7 +31,7 @@ async fn main() -> Result<()> { // deserialize ticket string into a ticket let ticket = - iroh::ticket::BlobTicket::from_str(&args[1]).context("failed parsing blob ticket\n\nGet a ticket by running the follow command in a separate terminal:\n\n`cargo run --example collection-provide`")?; + BlobTicket::from_str(&args[1]).context("failed parsing blob ticket\n\nGet a ticket by running the follow command in a separate terminal:\n\n`cargo run --example collection-provide`")?; // create a new node let node = iroh::node::Node::memory().spawn().await?; @@ -50,32 +49,19 @@ async fn main() -> Result<()> { .expect("a default relay url should be provided") .to_string() ); - let req = BlobDownloadRequest { - // The hash of the content we are trying to download. Provided in the ticket. - hash: ticket.hash(), - - // The format here is referring to the `BlobFormat`. We can request to download a single blob (which you can think of as a single file) or a `HashSeq` ("hash sequence"), which is a list of blobs you want to download. - // Iroh has a special kind of `HashSeq` called a "collection". A collection is just a `HashSeq` that reserves the first blob in the sequence for metadata about the `HashSeq` - // The metadata primarily contains the names of the blobs, which allows us, for example, to preserve filenames. - // When interacting with the iroh API, you will most likely be using blobs and collections. - format: ticket.format(), - - // The `nodes` field is a list of `NodeAddr`, where each combines all of the known address information we have for the remote node. - // This includes the `node_id` (or `PublicKey` of the node), any direct UDP addresses we know about for that node, as well as the relay url of that node. The relay url is the url of the relay server that that node is connected to. - // If the direct UDP addresses to that node do not work, than we can use the relay node to attempt to holepunch between your current node and the remote node. - // If holepunching fails, iroh will use the relay node to proxy a connection to the remote node over HTTPS. - // Thankfully, the ticket contains all of this information - nodes: vec![ticket.node_addr().clone()], - - // You can create a special tag name (`SetTagOption::Named`), or create an automatic tag that is derived from the timestamp. - tag: iroh::rpc_protocol::SetTagOption::Auto, - // Whether to use the download queue, or do a direct download. - mode: DownloadMode::Direct, - }; + // Get the content we have just fetched from the iroh database. + ensure!( + ticket.format() == BlobFormat::HashSeq, + "'collection' example expects to fetch a collection, but the ticket indicates a single blob." + ); - // `download` returns a stream of `DownloadProgress` events. You can iterate through these updates to get progress on the state of your download. - let download_stream = node.blobs.download(req).await?; + // `download` returns a stream of `DownloadProgress` events. You can iterate through these updates to get progress + // on the state of your download. + let download_stream = node + .blobs + .download_hash_seq(ticket.hash(), ticket.node_addr().clone()) + .await?; // You can also just `await` the stream, which poll the `DownloadProgress` stream for you. let outcome = download_stream.await.context("unable to download hash")?; @@ -86,12 +72,6 @@ async fn main() -> Result<()> { ticket.node_addr().node_id ); - // Get the content we have just fetched from the iroh database. - ensure!( - ticket.format() == BlobFormat::HashSeq, - "'collection' example expects to fetch a collection, but the ticket indicates a single blob." - ); - // If the `BlobFormat` is `HashSeq`, then we can assume for the example (and for any `HashSeq` that is derived from any iroh API), that it can be parsed as a `Collection` // A `Collection` is a special `HashSeq`, where we preserve the names of any blobs added to the collection. (We do this by designating the first entry in the `Collection` as meta data.) // To get the content of the collection, we first get the collection from the database using the `blobs` API diff --git a/iroh/examples/collection-provide.rs b/iroh/examples/collection-provide.rs index 3269ced626..5f12a7b6c9 100644 --- a/iroh/examples/collection-provide.rs +++ b/iroh/examples/collection-provide.rs @@ -6,8 +6,7 @@ //! This is using an in memory database and a random node id. //! run this example from the project root: //! $ cargo run --example collection-provide -use iroh::rpc_protocol::SetTagOption; -use iroh_bytes::{format::collection::Collection, BlobFormat}; +use iroh::bytes::{format::collection::Collection, util::SetTagOption, BlobFormat}; use tracing_subscriber::{prelude::*, EnvFilter}; // set the RUST_LOG env var to one of {debug,info,warn} to see logging info diff --git a/iroh/examples/hello-world-fetch.rs b/iroh/examples/hello-world-fetch.rs index fcc75cdbca..9470d39756 100644 --- a/iroh/examples/hello-world-fetch.rs +++ b/iroh/examples/hello-world-fetch.rs @@ -3,11 +3,10 @@ //! //! This is using an in memory database and a random node id. //! Run the `provide` example, which will give you instructions on how to run this example. +use std::{env, str::FromStr}; + use anyhow::{bail, ensure, Context, Result}; -use iroh::rpc_protocol::{BlobDownloadRequest, DownloadMode}; -use iroh_bytes::BlobFormat; -use std::env; -use std::str::FromStr; +use iroh::{base::ticket::BlobTicket, bytes::BlobFormat}; use tracing_subscriber::{prelude::*, EnvFilter}; // set the RUST_LOG env var to one of {debug,info,warn} to see logging info @@ -32,7 +31,7 @@ async fn main() -> Result<()> { // deserialize ticket string into a ticket let ticket = - iroh::ticket::BlobTicket::from_str(&args[1]).context("failed parsing blob ticket\n\nGet a ticket by running the follow command in a separate terminal:\n\n`cargo run --example hello-world-provide`")?; + BlobTicket::from_str(&args[1]).context("failed parsing blob ticket\n\nGet a ticket by running the follow command in a separate terminal:\n\n`cargo run --example hello-world-provide`")?; // create a new node let node = iroh::node::Node::memory().spawn().await?; @@ -50,32 +49,19 @@ async fn main() -> Result<()> { .expect("a default relay url should be provided") .to_string() ); - let req = BlobDownloadRequest { - // The hash of the content we are trying to download. Provided in the ticket. - hash: ticket.hash(), - - // The format here is referring to the `BlobFormat`. We can request to download a single blob (which you can think of as a single file) or a `HashSeq` ("hash sequence"), which is a list of blobs you want to download. - // Iroh has a special kind of `HashSeq` called a "collection". A collection is just a `HashSeq` that reserves the first blob in the sequence for metadata about the `HashSeq` - // The metadata primarily contains the names of the blobs, which allows us, for example, to preserve filenames. - // When interacting with the iroh API, you will most likely be using blobs and collections. - format: ticket.format(), - - // The `nodes` field is a list of `NodeAddr`, where each combines all of the known address information we have for the remote node. - // This includes the `node_id` (or `PublicKey` of the node), any direct UDP addresses we know about for that node, as well as the relay url of that node. The relay url is the url of the relay server that that node is connected to. - // If the direct UDP addresses to that node do not work, than we can use the relay node to attempt to holepunch between your current node and the remote node. - // If holepunching fails, iroh will use the relay node to proxy a connection to the remote node over HTTPS. - // Thankfully, the ticket contains all of this information - nodes: vec![ticket.node_addr().clone()], - - // You can create a special tag name (`SetTagOption::Named`), or create an automatic tag that is derived from the timestamp. - tag: iroh::rpc_protocol::SetTagOption::Auto, - // Whether to use the download queue, or do a direct download. - mode: DownloadMode::Direct, - }; + // If the `BlobFormat` is `Raw`, we have the hash for a single blob, and simply need to read the blob using the `blobs` API on the client to get the content. + ensure!( + ticket.format() == BlobFormat::Raw, + "'Hello World' example expects to fetch a single blob, but the ticket indicates a collection.", + ); - // `download` returns a stream of `DownloadProgress` events. You can iterate through these updates to get progress on the state of your download. - let download_stream = node.blobs.download(req).await?; + // `download` returns a stream of `DownloadProgress` events. You can iterate through these updates to get progress + // on the state of your download. + let download_stream = node + .blobs + .download(ticket.hash(), ticket.node_addr().clone()) + .await?; // You can also just `await` the stream, which will poll the `DownloadProgress` stream for you. let outcome = download_stream.await.context("unable to download hash")?; @@ -87,11 +73,6 @@ async fn main() -> Result<()> { ); // Get the content we have just fetched from the iroh database. - // If the `BlobFormat` is `Raw`, we have the hash for a single blob, and simply need to read the blob using the `blobs` API on the client to get the content. - ensure!( - ticket.format() == BlobFormat::Raw, - "'Hello World' example expects to fetch a single blob, but the ticket indicates a collection.", - ); let bytes = node.blobs.read_to_bytes(ticket.hash()).await?; let s = std::str::from_utf8(&bytes).context("unable to parse blob as as utf-8 string")?; diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 840cb103b2..3e8ad63c5c 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -1,57 +1,52 @@ -//! Client to an iroh node. Is generic over the connection (in-memory or RPC). -//! -//! TODO: Contains only iroh sync related methods. Add other methods. +//! Client to an Iroh node. use futures_lite::{Stream, StreamExt}; use quic_rpc::{RpcClient, ServiceConnection}; -use crate::rpc_protocol::ProviderService; +#[doc(inline)] +pub use crate::rpc_protocol::RpcService; -pub mod mem; -pub mod quic; +mod mem; +mod quic; -mod authors; -mod blobs; -mod docs; -mod node; -mod tags; +pub use self::mem::{Doc as MemDoc, Iroh as MemIroh, RpcClient as MemRpcClient}; +pub use self::quic::{Doc as QuicDoc, Iroh as QuicIroh, RpcClient as QuicRpcClient}; -pub use self::authors::Client as AuthorsClient; -pub use self::blobs::{ - BlobAddOutcome, BlobAddProgress, BlobDownloadOutcome, BlobDownloadProgress, BlobReader, - BlobStatus, Client as BlobsClient, -}; -pub use self::docs::{Client as DocsClient, Doc, Entry, LiveEvent}; -pub use self::node::Client as NodeClient; -pub use self::tags::Client as TagsClient; +pub(crate) use self::quic::{connect_raw as quic_connect_raw, RPC_ALPN}; -/// Iroh client +pub mod authors; +pub mod blobs; +pub mod docs; +pub mod node; +pub mod tags; + +/// Iroh client. #[derive(Debug, Clone)] pub struct Iroh { /// Client for node operations. - pub node: NodeClient, + pub node: node::Client, /// Client for blobs operations. - pub blobs: BlobsClient, + pub blobs: blobs::Client, /// Client for docs operations. - pub docs: DocsClient, + pub docs: docs::Client, /// Client for author operations. - pub authors: AuthorsClient, + pub authors: authors::Client, /// Client for tags operations. - pub tags: TagsClient, + pub tags: tags::Client, } impl Iroh where - C: ServiceConnection, + C: ServiceConnection, { /// Create a new high-level client to a Iroh node from the low-level RPC client. - pub fn new(rpc: RpcClient) -> Self { + pub fn new(rpc: RpcClient) -> Self { Self { - node: NodeClient { rpc: rpc.clone() }, - blobs: BlobsClient { rpc: rpc.clone() }, - docs: DocsClient { rpc: rpc.clone() }, - authors: AuthorsClient { rpc: rpc.clone() }, - tags: TagsClient { rpc }, + node: node::Client { rpc: rpc.clone() }, + blobs: blobs::Client { rpc: rpc.clone() }, + docs: docs::Client { rpc: rpc.clone() }, + authors: authors::Client { rpc: rpc.clone() }, + tags: tags::Client { rpc }, } } } diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index 8f74d0d107..8d0b5b69a2 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -1,3 +1,5 @@ +//! API for author management. + use anyhow::Result; use futures_lite::{stream::StreamExt, Stream}; use iroh_sync::{Author, AuthorId}; @@ -5,7 +7,7 @@ use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ AuthorCreateRequest, AuthorDeleteRequest, AuthorExportRequest, AuthorImportRequest, - AuthorListRequest, ProviderService, + AuthorListRequest, RpcService, }; use super::flatten; @@ -13,12 +15,12 @@ use super::flatten; /// Iroh authors client. #[derive(Debug, Clone)] pub struct Client { - pub(super) rpc: RpcClient, + pub(super) rpc: RpcClient, } impl Client where - C: ServiceConnection, + C: ServiceConnection, { /// Create a new document author. pub async fn create(&self) -> Result { diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs index 5f7fe5561b..9130a701e9 100644 --- a/iroh/src/client/blobs.rs +++ b/iroh/src/client/blobs.rs @@ -1,3 +1,5 @@ +//! API for blobs management. + use std::{ future::Future, io, @@ -13,15 +15,16 @@ use futures_lite::{Stream, StreamExt}; use futures_util::SinkExt; use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; use iroh_bytes::{ - export::ExportProgress, + export::ExportProgress as BytesExportProgress, format::collection::Collection, - get::db::DownloadProgress, - provider::AddProgress, + get::db::DownloadProgress as BytesDownloadProgress, store::{ConsistencyCheckProgress, ExportFormat, ExportMode, ValidateProgress}, BlobFormat, Hash, Tag, }; +use iroh_net::NodeAddr; use portable_atomic::{AtomicU64, Ordering}; use quic_rpc::{client::BoxStreamSync, RpcClient, ServiceConnection}; +use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; use tokio_util::io::{ReaderStream, StreamReader}; use tracing::warn; @@ -29,11 +32,9 @@ use tracing::warn; use crate::rpc_protocol::{ BlobAddPathRequest, BlobAddStreamRequest, BlobAddStreamUpdate, BlobConsistencyCheckRequest, BlobDeleteBlobRequest, BlobDownloadRequest, BlobExportRequest, BlobGetCollectionRequest, - BlobGetCollectionResponse, BlobListCollectionsRequest, BlobListCollectionsResponse, - BlobListIncompleteRequest, BlobListIncompleteResponse, BlobListRequest, BlobListResponse, - BlobReadAtRequest, BlobReadAtResponse, BlobValidateRequest, CreateCollectionRequest, - CreateCollectionResponse, NodeStatusRequest, NodeStatusResponse, ProviderService, SetTagOption, - WrapOption, + BlobGetCollectionResponse, BlobListCollectionsRequest, BlobListIncompleteRequest, + BlobListRequest, BlobReadAtRequest, BlobReadAtResponse, BlobValidateRequest, + CreateCollectionRequest, CreateCollectionResponse, NodeStatusRequest, RpcService, SetTagOption, }; use super::{flatten, Iroh}; @@ -41,42 +42,40 @@ use super::{flatten, Iroh}; /// Iroh blobs client. #[derive(Debug, Clone)] pub struct Client { - pub(super) rpc: RpcClient, + pub(super) rpc: RpcClient, } -impl<'a, C: ServiceConnection> From<&'a Iroh> - for &'a RpcClient -{ - fn from(client: &'a Iroh) -> &'a RpcClient { +impl<'a, C: ServiceConnection> From<&'a Iroh> for &'a RpcClient { + fn from(client: &'a Iroh) -> &'a RpcClient { &client.blobs.rpc } } impl Client where - C: ServiceConnection, + C: ServiceConnection, { /// Stream the contents of a a single blob. /// - /// Returns a [`BlobReader`], which can report the size of the blob before reading it. - pub async fn read(&self, hash: Hash) -> Result { - BlobReader::from_rpc_read(&self.rpc, hash).await + /// Returns a [`Reader`], which can report the size of the blob before reading it. + pub async fn read(&self, hash: Hash) -> Result { + Reader::from_rpc_read(&self.rpc, hash).await } /// Read offset + len from a single blob. /// /// If `len` is `None` it will read the full blob. - pub async fn read_at(&self, hash: Hash, offset: u64, len: Option) -> Result { - BlobReader::from_rpc_read_at(&self.rpc, hash, offset, len).await + pub async fn read_at(&self, hash: Hash, offset: u64, len: Option) -> Result { + Reader::from_rpc_read_at(&self.rpc, hash, offset, len).await } /// Read all bytes of single blob. /// /// This allocates a buffer for the full blob. Use only if you know that the blob you're /// reading is small. If not sure, use [`Self::read`] and check the size with - /// [`BlobReader::size`] before calling [`BlobReader::read_to_bytes`]. + /// [`Reader::size`] before calling [`Reader::read_to_bytes`]. pub async fn read_to_bytes(&self, hash: Hash) -> Result { - BlobReader::from_rpc_read(&self.rpc, hash) + Reader::from_rpc_read(&self.rpc, hash) .await? .read_to_bytes() .await @@ -91,7 +90,7 @@ where offset: u64, len: Option, ) -> Result { - BlobReader::from_rpc_read_at(&self.rpc, hash, offset, len) + Reader::from_rpc_read_at(&self.rpc, hash, offset, len) .await? .read_to_bytes() .await @@ -109,7 +108,7 @@ where in_place: bool, tag: SetTagOption, wrap: WrapOption, - ) -> Result { + ) -> Result { let stream = self .rpc .server_streaming(BlobAddPathRequest { @@ -119,7 +118,7 @@ where wrap, }) .await?; - Ok(BlobAddProgress::new(stream)) + Ok(AddProgress::new(stream)) } /// Create a collection from already existing blobs. @@ -148,7 +147,7 @@ where &self, reader: impl AsyncRead + Unpin + Send + 'static, tag: SetTagOption, - ) -> anyhow::Result { + ) -> anyhow::Result { const CAP: usize = 1024 * 64; // send 64KB per request by default let input = ReaderStream::with_capacity(reader, CAP); self.add_stream(input, tag).await @@ -159,7 +158,7 @@ where &self, input: impl Stream> + Send + Unpin + 'static, tag: SetTagOption, - ) -> anyhow::Result { + ) -> anyhow::Result { let (mut sink, progress) = self.rpc.bidi(BlobAddStreamRequest { tag }).await?; let mut input = input.map(|chunk| match chunk { Ok(chunk) => Ok(BlobAddStreamUpdate::Chunk(chunk)), @@ -177,11 +176,11 @@ where } }); - Ok(BlobAddProgress::new(progress)) + Ok(AddProgress::new(progress)) } /// Write a blob by passing bytes. - pub async fn add_bytes(&self, bytes: impl Into) -> anyhow::Result { + pub async fn add_bytes(&self, bytes: impl Into) -> anyhow::Result { let input = futures_lite::stream::once(Ok(bytes.into())); self.add_stream(input, SetTagOption::Auto).await?.await } @@ -191,7 +190,7 @@ where &self, bytes: impl Into, name: impl Into, - ) -> anyhow::Result { + ) -> anyhow::Result { let input = futures_lite::stream::once(Ok(bytes.into())); self.add_stream(input, SetTagOption::Named(name.into())) .await? @@ -227,9 +226,56 @@ where } /// Download a blob from another node and add it to the local database. - pub async fn download(&self, req: BlobDownloadRequest) -> Result { - let stream = self.rpc.server_streaming(req).await?; - Ok(BlobDownloadProgress::new( + pub async fn download(&self, hash: Hash, node: NodeAddr) -> Result { + self.download_with_opts( + hash, + DownloadOptions { + format: BlobFormat::Raw, + nodes: vec![node], + tag: SetTagOption::Auto, + mode: DownloadMode::Queued, + }, + ) + .await + } + + /// Download a hash sequence from another node and add it to the local database. + pub async fn download_hash_seq(&self, hash: Hash, node: NodeAddr) -> Result { + self.download_with_opts( + hash, + DownloadOptions { + format: BlobFormat::HashSeq, + nodes: vec![node], + tag: SetTagOption::Auto, + mode: DownloadMode::Queued, + }, + ) + .await + } + + /// Download a blob, with additional options. + pub async fn download_with_opts( + &self, + hash: Hash, + opts: DownloadOptions, + ) -> Result { + let DownloadOptions { + format, + nodes, + tag, + mode, + } = opts; + let stream = self + .rpc + .server_streaming(BlobDownloadRequest { + hash, + format, + nodes, + tag, + mode, + }) + .await?; + Ok(DownloadProgress::new( stream.map(|res| res.map_err(anyhow::Error::from)), )) } @@ -249,7 +295,7 @@ where destination: PathBuf, format: ExportFormat, mode: ExportMode, - ) -> Result { + ) -> Result { let req = BlobExportRequest { hash, path: destination, @@ -257,21 +303,19 @@ where mode, }; let stream = self.rpc.server_streaming(req).await?; - Ok(BlobExportProgress::new( + Ok(ExportProgress::new( stream.map(|r| r.map_err(anyhow::Error::from)), )) } /// List all complete blobs. - pub async fn list(&self) -> Result>> { + pub async fn list(&self) -> Result>> { let stream = self.rpc.server_streaming(BlobListRequest).await?; Ok(flatten(stream)) } /// List all incomplete (partial) blobs. - pub async fn list_incomplete( - &self, - ) -> Result>> { + pub async fn list_incomplete(&self) -> Result>> { let stream = self.rpc.server_streaming(BlobListIncompleteRequest).await?; Ok(flatten(stream)) } @@ -284,9 +328,7 @@ where } /// List all collections. - pub async fn list_collections( - &self, - ) -> Result>> { + pub async fn list_collections(&self) -> Result>> { let stream = self .rpc .server_streaming(BlobListCollectionsRequest) @@ -307,7 +349,7 @@ where blob_format: BlobFormat, addr_options: AddrInfoOptions, ) -> Result { - let NodeStatusResponse { mut addr, .. } = self.rpc.rpc(NodeStatusRequest).await??; + let mut addr = self.rpc.rpc(NodeStatusRequest).await??.addr; addr.apply_options(addr_options); let ticket = BlobTicket::new(addr, hash, blob_format).expect("correct ticket"); @@ -326,6 +368,18 @@ where } } +/// Whether to wrap the added data in a collection. +#[derive(Debug, Serialize, Deserialize)] +pub enum WrapOption { + /// Do not wrap the file or directory. + NoWrap, + /// Wrap the file or directory in a collection. + Wrap { + /// Override the filename in the wrapping collection. + name: Option, + }, +} + /// Status information about a blob. #[derive(Debug, Clone, PartialEq, Eq)] pub enum BlobStatus { @@ -343,7 +397,7 @@ pub enum BlobStatus { /// Outcome of a blob add operation. #[derive(Debug, Clone)] -pub struct BlobAddOutcome { +pub struct AddOutcome { /// The hash of the blob pub hash: Hash, /// The format the blob @@ -354,18 +408,61 @@ pub struct BlobAddOutcome { pub tag: Tag, } +/// Information about a stored collection. +#[derive(Debug, Serialize, Deserialize)] +pub struct CollectionInfo { + /// Tag of the collection + pub tag: Tag, + + /// Hash of the collection + pub hash: Hash, + /// Number of children in the collection + /// + /// This is an optional field, because the data is not always available. + pub total_blobs_count: Option, + /// Total size of the raw data referred to by all links + /// + /// This is an optional field, because the data is not always available. + pub total_blobs_size: Option, +} + +/// Information about a complete blob. +#[derive(Debug, Serialize, Deserialize)] +pub struct BlobInfo { + /// Location of the blob + pub path: String, + /// The hash of the blob + pub hash: Hash, + /// The size of the blob + pub size: u64, +} + +/// Information about an incomplete blob. +#[derive(Debug, Serialize, Deserialize)] +pub struct IncompleteBlobInfo { + /// The size we got + pub size: u64, + /// The size we expect + pub expected_size: u64, + /// The hash of the blob + pub hash: Hash, +} + /// Progress stream for blob add operations. #[derive(derive_more::Debug)] -pub struct BlobAddProgress { +pub struct AddProgress { #[debug(skip)] - stream: Pin> + Send + Unpin + 'static>>, + stream: Pin< + Box> + Send + Unpin + 'static>, + >, current_total_size: Arc, } -impl BlobAddProgress { +impl AddProgress { fn new( - stream: (impl Stream, impl Into>> - + Send + stream: (impl Stream< + Item = Result, impl Into>, + > + Send + Unpin + 'static), ) -> Self { @@ -374,7 +471,7 @@ impl BlobAddProgress { let stream = stream.map(move |item| match item { Ok(item) => { let item = item.into(); - if let AddProgress::Found { size, .. } = &item { + if let iroh_bytes::provider::AddProgress::Found { size, .. } = &item { total_size.fetch_add(*size, Ordering::Relaxed); } Ok(item) @@ -388,25 +485,25 @@ impl BlobAddProgress { } /// Finish writing the stream, ignoring all intermediate progress events. /// - /// Returns a [`BlobAddOutcome`] which contains a tag, format, hash and a size. + /// Returns a [`AddOutcome`] which contains a tag, format, hash and a size. /// When importing a single blob, this is the hash and size of that blob. /// When importing a collection, the hash is the hash of the collection and the size /// is the total size of all imported blobs (but excluding the size of the collection blob /// itself). - pub async fn finish(self) -> Result { + pub async fn finish(self) -> Result { self.await } } -impl Stream for BlobAddProgress { - type Item = Result; +impl Stream for AddProgress { + type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.stream).poll_next(cx) } } -impl Future for BlobAddProgress { - type Output = Result; +impl Future for AddProgress { + type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { loop { @@ -417,8 +514,8 @@ impl Future for BlobAddProgress { } Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), Poll::Ready(Some(Ok(msg))) => match msg { - AddProgress::AllDone { hash, format, tag } => { - let outcome = BlobAddOutcome { + iroh_bytes::provider::AddProgress::AllDone { hash, format, tag } => { + let outcome = AddOutcome { hash, format, tag, @@ -426,7 +523,7 @@ impl Future for BlobAddProgress { }; return Poll::Ready(Ok(outcome)); } - AddProgress::Abort(err) => { + iroh_bytes::provider::AddProgress::Abort(err) => { return Poll::Ready(Err(err.into())); } _ => {} @@ -438,7 +535,7 @@ impl Future for BlobAddProgress { /// Outcome of a blob download operation. #[derive(Debug, Clone)] -pub struct BlobDownloadOutcome { +pub struct DownloadOutcome { /// The size of the data we already had locally pub local_size: u64, /// The size of the data we downloaded from the network @@ -449,17 +546,17 @@ pub struct BlobDownloadOutcome { /// Progress stream for blob download operations. #[derive(derive_more::Debug)] -pub struct BlobDownloadProgress { +pub struct DownloadProgress { #[debug(skip)] - stream: Pin> + Send + Unpin + 'static>>, + stream: Pin> + Send + Unpin + 'static>>, current_local_size: Arc, current_network_size: Arc, } -impl BlobDownloadProgress { - /// Create a `BlobDownloadProgress` that can help you easily poll the `DownloadProgress` stream from your download until it is finished or errors. +impl DownloadProgress { + /// Create a [`DownloadProgress`] that can help you easily poll the [`BytesDownloadProgress`] stream from your download until it is finished or errors. pub fn new( - stream: (impl Stream, impl Into>> + stream: (impl Stream, impl Into>> + Send + Unpin + 'static), @@ -474,10 +571,10 @@ impl BlobDownloadProgress { Ok(item) => { let item = item.into(); match &item { - DownloadProgress::FoundLocal { size, .. } => { + BytesDownloadProgress::FoundLocal { size, .. } => { local_size.fetch_add(size.value(), Ordering::Relaxed); } - DownloadProgress::Found { size, .. } => { + BytesDownloadProgress::Found { size, .. } => { network_size.fetch_add(*size, Ordering::Relaxed); } _ => {} @@ -493,25 +590,26 @@ impl BlobDownloadProgress { current_network_size, } } + /// Finish writing the stream, ignoring all intermediate progress events. /// - /// Returns a [`BlobDownloadOutcome`] which contains the size of the content we downloaded and the size of the content we already had locally. + /// Returns a [`DownloadOutcome`] which contains the size of the content we downloaded and the size of the content we already had locally. /// When importing a single blob, this is the size of that blob. /// When importing a collection, this is the total size of all imported blobs (but excluding the size of the collection blob itself). - pub async fn finish(self) -> Result { + pub async fn finish(self) -> Result { self.await } } -impl Stream for BlobDownloadProgress { - type Item = Result; +impl Stream for DownloadProgress { + type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.stream).poll_next(cx) } } -impl Future for BlobDownloadProgress { - type Output = Result; +impl Future for DownloadProgress { + type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { loop { @@ -522,15 +620,15 @@ impl Future for BlobDownloadProgress { } Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), Poll::Ready(Some(Ok(msg))) => match msg { - DownloadProgress::AllDone(stats) => { - let outcome = BlobDownloadOutcome { + BytesDownloadProgress::AllDone(stats) => { + let outcome = DownloadOutcome { local_size: self.current_local_size.load(Ordering::Relaxed), downloaded_size: self.current_network_size.load(Ordering::Relaxed), stats, }; return Poll::Ready(Ok(outcome)); } - DownloadProgress::Abort(err) => { + BytesDownloadProgress::Abort(err) => { return Poll::Ready(Err(err.into())); } _ => {} @@ -542,23 +640,24 @@ impl Future for BlobDownloadProgress { /// Outcome of a blob export operation. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct BlobExportOutcome { +pub struct ExportOutcome { /// The total size of the exported data. total_size: u64, } /// Progress stream for blob export operations. #[derive(derive_more::Debug)] -pub struct BlobExportProgress { +pub struct ExportProgress { #[debug(skip)] - stream: Pin> + Send + Unpin + 'static>>, + stream: Pin> + Send + Unpin + 'static>>, current_total_size: Arc, } -impl BlobExportProgress { - /// Create a `BlobExportProgress` that can help you easily poll the `ExportProgress` stream from your download until it is finished or errors. +impl ExportProgress { + /// Create a [`ExportProgress`] that can help you easily poll the [`BytesExportProgress`] stream from your + /// download until it is finished or errors. pub fn new( - stream: (impl Stream, impl Into>> + stream: (impl Stream, impl Into>> + Send + Unpin + 'static), @@ -568,7 +667,7 @@ impl BlobExportProgress { let stream = stream.map(move |item| match item { Ok(item) => { let item = item.into(); - if let ExportProgress::Found { size, .. } = &item { + if let BytesExportProgress::Found { size, .. } = &item { let size = size.value(); total_size.fetch_add(size, Ordering::Relaxed); } @@ -585,21 +684,21 @@ impl BlobExportProgress { /// Finish writing the stream, ignoring all intermediate progress events. /// - /// Returns a [`BlobExportOutcome`] which contains the size of the content we exported. - pub async fn finish(self) -> Result { + /// Returns a [`ExportOutcome`] which contains the size of the content we exported. + pub async fn finish(self) -> Result { self.await } } -impl Stream for BlobExportProgress { - type Item = Result; +impl Stream for ExportProgress { + type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.stream).poll_next(cx) } } -impl Future for BlobExportProgress { - type Output = Result; +impl Future for ExportProgress { + type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { loop { @@ -610,13 +709,13 @@ impl Future for BlobExportProgress { } Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), Poll::Ready(Some(Ok(msg))) => match msg { - ExportProgress::AllDone => { - let outcome = BlobExportOutcome { + BytesExportProgress::AllDone => { + let outcome = ExportOutcome { total_size: self.current_total_size.load(Ordering::Relaxed), }; return Poll::Ready(Ok(outcome)); } - ExportProgress::Abort(err) => { + BytesExportProgress::Abort(err) => { return Poll::Ready(Err(err.into())); } _ => {} @@ -630,7 +729,7 @@ impl Future for BlobExportProgress { /// /// Implements [`AsyncRead`]. #[derive(derive_more::Debug)] -pub struct BlobReader { +pub struct Reader { size: u64, response_size: u64, is_complete: bool, @@ -638,7 +737,7 @@ pub struct BlobReader { stream: tokio_util::io::StreamReader>, Bytes>, } -impl BlobReader { +impl Reader { fn new( size: u64, response_size: u64, @@ -653,15 +752,15 @@ impl BlobReader { } } - pub(crate) async fn from_rpc_read>( - rpc: &RpcClient, + pub(crate) async fn from_rpc_read>( + rpc: &RpcClient, hash: Hash, ) -> anyhow::Result { Self::from_rpc_read_at(rpc, hash, 0, None).await } - async fn from_rpc_read_at>( - rpc: &RpcClient, + async fn from_rpc_read_at>( + rpc: &RpcClient, hash: Hash, offset: u64, len: Option, @@ -708,7 +807,7 @@ impl BlobReader { } } -impl AsyncRead for BlobReader { +impl AsyncRead for Reader { fn poll_read( mut self: Pin<&mut Self>, cx: &mut Context<'_>, @@ -718,7 +817,7 @@ impl AsyncRead for BlobReader { } } -impl Stream for BlobReader { +impl Stream for Reader { type Item = io::Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -730,6 +829,38 @@ impl Stream for BlobReader { } } +/// Options to configure a download request. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DownloadOptions { + /// The format of the data to download. + pub format: BlobFormat, + /// Source nodes to download from. + /// + /// If set to more than a single node, they will all be tried. If `mode` is set to + /// [`DownloadMode::Direct`], they will be tried sequentially until a download succeeds. + /// If `mode` is set to [`DownloadMode::Queued`], the nodes may be dialed in parallel, + /// if the concurrency limits permit. + pub nodes: Vec, + /// Optional tag to tag the data with. + pub tag: SetTagOption, + /// Whether to directly start the download or add it to the download queue. + pub mode: DownloadMode, +} + +/// Set the mode for whether to directly start the download or add it to the download queue. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum DownloadMode { + /// Start the download right away. + /// + /// No concurrency limits or queuing will be applied. It is up to the user to manage download + /// concurrency. + Direct, + /// Queue the download. + /// + /// The download queue will be processed in-order, while respecting the downloader concurrency limits. + Queued, +} + #[cfg(test)] mod tests { use super::*; @@ -802,7 +933,7 @@ mod tests { assert_eq!(collections.len(), 1); { - let BlobListCollectionsResponse { + let CollectionInfo { tag, hash, total_blobs_count, diff --git a/iroh/src/client/docs.rs b/iroh/src/client/docs.rs index 83bb9d002d..9c4229325d 100644 --- a/iroh/src/client/docs.rs +++ b/iroh/src/client/docs.rs @@ -1,3 +1,5 @@ +//! API for document management. + use std::{ path::{Path, PathBuf}, pin::Pin, @@ -7,6 +9,7 @@ use std::{ use anyhow::{anyhow, Context as _, Result}; use bytes::Bytes; +use derive_more::{Display, FromStr}; use futures_lite::{Stream, StreamExt}; use iroh_base::{key::PublicKey, node_addr::AddrInfoOptions}; use iroh_bytes::{export::ExportProgress, store::ExportMode, Hash}; @@ -14,36 +17,35 @@ use iroh_net::NodeAddr; use iroh_sync::{ actor::OpenState, store::{DownloadPolicy, Query}, - AuthorId, CapabilityKind, ContentStatus, NamespaceId, PeerIdBytes, RecordIdentifier, + AuthorId, CapabilityKind, ContentStatus, DocTicket, NamespaceId, PeerIdBytes, RecordIdentifier, }; use portable_atomic::{AtomicBool, Ordering}; use quic_rpc::{message::RpcMsg, RpcClient, ServiceConnection}; use serde::{Deserialize, Serialize}; -use crate::{ - rpc_protocol::{ - DocCloseRequest, DocCreateRequest, DocDelRequest, DocDelResponse, DocDropRequest, - DocExportFileRequest, DocGetDownloadPolicyRequest, DocGetExactRequest, DocGetManyRequest, - DocGetSyncPeersRequest, DocImportFileRequest, DocImportProgress, DocImportRequest, - DocLeaveRequest, DocListRequest, DocOpenRequest, DocSetDownloadPolicyRequest, - DocSetHashRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocStatusRequest, - DocSubscribeRequest, ProviderService, ShareMode, - }, - sync_engine::SyncEvent, - ticket::DocTicket, +use crate::rpc_protocol::{ + DocCloseRequest, DocCreateRequest, DocDelRequest, DocDelResponse, DocDropRequest, + DocExportFileRequest, DocGetDownloadPolicyRequest, DocGetExactRequest, DocGetManyRequest, + DocGetSyncPeersRequest, DocImportFileRequest, DocImportProgress, DocImportRequest, + DocLeaveRequest, DocListRequest, DocOpenRequest, DocSetDownloadPolicyRequest, + DocSetHashRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocStatusRequest, + DocSubscribeRequest, RpcService, }; -use super::{blobs::BlobReader, flatten}; +#[doc(inline)] +pub use crate::sync_engine::{Origin, SyncEvent, SyncReason}; + +use super::{blobs, flatten}; /// Iroh docs client. #[derive(Debug, Clone)] pub struct Client { - pub(super) rpc: RpcClient, + pub(super) rpc: RpcClient, } impl Client where - C: ServiceConnection, + C: ServiceConnection, { /// Create a new document. pub async fn create(&self) -> Result> { @@ -85,27 +87,27 @@ where /// Document handle #[derive(Debug, Clone)] -pub struct Doc>(Arc>); +pub struct Doc>(Arc>); -impl> PartialEq for Doc { +impl> PartialEq for Doc { fn eq(&self, other: &Self) -> bool { self.0.id == other.0.id } } -impl> Eq for Doc {} +impl> Eq for Doc {} #[derive(Debug)] -struct DocInner> { +struct DocInner> { id: NamespaceId, - rpc: RpcClient, + rpc: RpcClient, closed: AtomicBool, rt: tokio::runtime::Handle, } impl Drop for DocInner where - C: ServiceConnection, + C: ServiceConnection, { fn drop(&mut self) { let doc_id = self.id; @@ -118,9 +120,9 @@ where impl Doc where - C: ServiceConnection, + C: ServiceConnection, { - fn new(rpc: RpcClient, id: NamespaceId) -> Self { + fn new(rpc: RpcClient, id: NamespaceId) -> Self { Self(Arc::new(DocInner { rpc, id, @@ -131,7 +133,7 @@ where async fn rpc(&self, msg: M) -> Result where - M: RpcMsg, + M: RpcMsg, { let res = self.0.rpc.rpc(msg).await?; Ok(res) @@ -203,7 +205,7 @@ where key: Bytes, path: impl AsRef, in_place: bool, - ) -> Result { + ) -> Result { self.ensure_open()?; let stream = self .0 @@ -216,7 +218,7 @@ where in_place, }) .await?; - Ok(DocImportFileProgress::new(stream)) + Ok(ImportFileProgress::new(stream)) } /// Export an entry as a file to a given absolute path. @@ -225,7 +227,7 @@ where entry: Entry, path: impl AsRef, mode: ExportMode, - ) -> Result { + ) -> Result { self.ensure_open()?; let stream = self .0 @@ -236,7 +238,7 @@ where mode, }) .await?; - Ok(DocExportFileProgress::new(stream)) + Ok(ExportFileProgress::new(stream)) } /// Delete entries that match the given `author` and key `prefix`. @@ -385,10 +387,8 @@ where } } -impl<'a, C: ServiceConnection> From<&'a Doc> - for &'a RpcClient -{ - fn from(doc: &'a Doc) -> &'a RpcClient { +impl<'a, C: ServiceConnection> From<&'a Doc> for &'a RpcClient { + fn from(doc: &'a Doc) -> &'a RpcClient { &doc.0.rpc } } @@ -440,17 +440,17 @@ impl Entry { self.0.timestamp() } - /// Read the content of an [`Entry`] as a streaming [`BlobReader`]. + /// Read the content of an [`Entry`] as a streaming [`blobs::Reader`]. /// /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. pub async fn content_reader( &self, - client: impl Into<&RpcClient>, - ) -> Result + client: impl Into<&RpcClient>, + ) -> Result where - C: ServiceConnection, + C: ServiceConnection, { - BlobReader::from_rpc_read(client.into(), self.content_hash()).await + blobs::Reader::from_rpc_read(client.into(), self.content_hash()).await } /// Read all content of an [`Entry`] into a buffer. @@ -458,18 +458,27 @@ impl Entry { /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. pub async fn content_bytes( &self, - client: impl Into<&RpcClient>, + client: impl Into<&RpcClient>, ) -> Result where - C: ServiceConnection, + C: ServiceConnection, { - BlobReader::from_rpc_read(client.into(), self.content_hash()) + blobs::Reader::from_rpc_read(client.into(), self.content_hash()) .await? .read_to_bytes() .await } } +/// Intended capability for document share tickets +#[derive(Serialize, Deserialize, Debug, Clone, Display, FromStr)] +pub enum ShareMode { + /// Read-only access + Read, + /// Write access + Write, +} + /// Events informing about actions of the live sync progress. #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, strum::Display)] pub enum LiveEvent { @@ -523,15 +532,15 @@ impl From for LiveEvent { } } -/// Progress stream for doc import operations. +/// Progress stream for [`Doc::import_file`]. #[derive(derive_more::Debug)] #[must_use = "streams do nothing unless polled"] -pub struct DocImportFileProgress { +pub struct ImportFileProgress { #[debug(skip)] stream: Pin> + Send + Unpin + 'static>>, } -impl DocImportFileProgress { +impl ImportFileProgress { fn new( stream: (impl Stream, impl Into>> + Send @@ -549,9 +558,9 @@ impl DocImportFileProgress { /// Finish writing the stream, ignoring all intermediate progress events. /// - /// Returns a [`DocImportFileOutcome`] which contains a tag, key, and hash and the size of the + /// Returns a [`ImportFileOutcome`] which contains a tag, key, and hash and the size of the /// content. - pub async fn finish(mut self) -> Result { + pub async fn finish(mut self) -> Result { let mut entry_size = 0; let mut entry_hash = None; while let Some(msg) = self.next().await { @@ -562,7 +571,7 @@ impl DocImportFileProgress { DocImportProgress::AllDone { key } => { let hash = entry_hash .context("expected DocImportProgress::IngestDone event to occur")?; - let outcome = DocImportFileOutcome { + let outcome = ImportFileOutcome { hash, key, size: entry_size, @@ -582,7 +591,7 @@ impl DocImportFileProgress { /// Outcome of a [`Doc::import_file`] operation #[derive(Debug, Clone, PartialEq, Eq)] -pub struct DocImportFileOutcome { +pub struct ImportFileOutcome { /// The hash of the entry's content pub hash: Hash, /// The size of the entry @@ -591,20 +600,20 @@ pub struct DocImportFileOutcome { pub key: Bytes, } -impl Stream for DocImportFileProgress { +impl Stream for ImportFileProgress { type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.stream).poll_next(cx) } } -/// Progress stream for doc export operations. +/// Progress stream for [`Doc::export_file`]. #[derive(derive_more::Debug)] -pub struct DocExportFileProgress { +pub struct ExportFileProgress { #[debug(skip)] stream: Pin> + Send + Unpin + 'static>>, } -impl DocExportFileProgress { +impl ExportFileProgress { fn new( stream: (impl Stream, impl Into>> + Send @@ -621,8 +630,8 @@ impl DocExportFileProgress { } /// Iterate through the export progress stream, returning when the stream has completed. - /// Returns a [`DocExportFileOutcome`] which contains a file path the data was written to and the size of the content. - pub async fn finish(mut self) -> Result { + /// Returns a [`ExportFileOutcome`] which contains a file path the data was written to and the size of the content. + pub async fn finish(mut self) -> Result { let mut total_size = 0; let mut path = None; while let Some(msg) = self.next().await { @@ -633,7 +642,7 @@ impl DocExportFileProgress { } ExportProgress::AllDone => { let path = path.context("expected ExportProgress::Found event to occur")?; - let outcome = DocExportFileOutcome { + let outcome = ExportFileOutcome { size: total_size, path, }; @@ -650,14 +659,14 @@ impl DocExportFileProgress { /// Outcome of a [`Doc::export_file`] operation #[derive(Debug, Clone, PartialEq, Eq)] -pub struct DocExportFileOutcome { +pub struct ExportFileOutcome { /// The size of the entry size: u64, /// The path to which the entry was saved path: PathBuf, } -impl Stream for DocExportFileProgress { +impl Stream for ExportFileProgress { type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { diff --git a/iroh/src/client/mem.rs b/iroh/src/client/mem.rs index 1d72f7368a..d34cdca5ea 100644 --- a/iroh/src/client/mem.rs +++ b/iroh/src/client/mem.rs @@ -5,16 +5,15 @@ use quic_rpc::transport::flume::FlumeConnection; -use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService}; +use crate::rpc_protocol::{Request, Response, RpcService}; /// RPC client to an iroh node running in the same process. -pub type RpcClient = - quic_rpc::RpcClient>; +pub type RpcClient = quic_rpc::RpcClient>; /// In-memory client to an iroh node running in the same process. /// /// This is obtained from [`crate::node::Node::client`]. -pub type Iroh = super::Iroh>; +pub type Iroh = super::Iroh>; /// In-memory document client to an iroh node running in the same process. -pub type Doc = super::Doc>; +pub type Doc = super::docs::Doc>; diff --git a/iroh/src/client/node.rs b/iroh/src/client/node.rs index 6a4d15ff92..eb1de5703a 100644 --- a/iroh/src/client/node.rs +++ b/iroh/src/client/node.rs @@ -1,14 +1,17 @@ -use std::collections::BTreeMap; +//! API to manage the iroh node itself. + +use std::{collections::BTreeMap, net::SocketAddr}; use anyhow::Result; use futures_lite::{Stream, StreamExt}; use iroh_base::key::PublicKey; -use iroh_net::magic_endpoint::ConnectionInfo; +use iroh_net::{magic_endpoint::ConnectionInfo, NodeAddr, NodeId}; use quic_rpc::{RpcClient, ServiceConnection}; +use serde::{Deserialize, Serialize}; use crate::rpc_protocol::{ CounterStats, NodeConnectionInfoRequest, NodeConnectionInfoResponse, NodeConnectionsRequest, - NodeShutdownRequest, NodeStatsRequest, NodeStatusRequest, NodeStatusResponse, ProviderService, + NodeIdRequest, NodeShutdownRequest, NodeStatsRequest, NodeStatusRequest, RpcService, }; use super::flatten; @@ -16,12 +19,12 @@ use super::flatten; /// Iroh node client. #[derive(Debug, Clone)] pub struct Client { - pub(super) rpc: RpcClient, + pub(super) rpc: RpcClient, } impl Client where - C: ServiceConnection, + C: ServiceConnection, { /// Get statistics of the running node. pub async fn stats(&self) -> Result> { @@ -44,12 +47,18 @@ where Ok(conn_info) } - /// Get status information about a node - pub async fn status(&self) -> Result { + /// Get status information about a node. + pub async fn status(&self) -> Result { let response = self.rpc.rpc(NodeStatusRequest).await??; Ok(response) } + /// Get the id of this node. + pub async fn id(&self) -> Result { + let id = self.rpc.rpc(NodeIdRequest).await??; + Ok(id) + } + /// Shutdown the node. /// /// If `force` is true, the node will be killed instantly without waiting for things to @@ -59,3 +68,14 @@ where Ok(()) } } + +/// The response to a version request +#[derive(Debug, Serialize, Deserialize)] +pub struct NodeStatus { + /// The node id and socket addresses of this node. + pub addr: NodeAddr, + /// The bound listening addresses of the node + pub listen_addrs: Vec, + /// The version of the node + pub version: String, +} diff --git a/iroh/src/client/quic.rs b/iroh/src/client/quic.rs index 066a858f59..e403e4ee0b 100644 --- a/iroh/src/client/quic.rs +++ b/iroh/src/client/quic.rs @@ -12,23 +12,23 @@ use quic_rpc::transport::quinn::QuinnConnection; use crate::{ node::RpcStatus, - rpc_protocol::{NodeStatusRequest, ProviderRequest, ProviderResponse, ProviderService}, + rpc_protocol::{NodeStatusRequest, Request, Response, RpcService}, }; -/// TODO: Change to "/iroh-rpc/1" -pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; +/// ALPN used by irohs RPC mechanism. +// TODO: Change to "/iroh-rpc/1" +pub(crate) const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; /// RPC client to an iroh node running in a separate process. -pub type RpcClient = - quic_rpc::RpcClient>; +pub type RpcClient = quic_rpc::RpcClient>; /// Client to an iroh node running in a separate process. /// /// This is obtained from [`Iroh::connect`]. -pub type Iroh = super::Iroh>; +pub type Iroh = super::Iroh>; /// RPC document client to an iroh node running in a separate process. -pub type Doc = super::Doc>; +pub type Doc = super::docs::Doc>; impl Iroh { /// Connect to an iroh node running on the same computer, but in a different process. @@ -45,7 +45,7 @@ impl Iroh { /// Create a raw RPC client to an iroh node running on the same computer, but in a different /// process. -pub async fn connect_raw(rpc_port: u16) -> anyhow::Result { +pub(crate) async fn connect_raw(rpc_port: u16) -> anyhow::Result { let bind_addr = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into(); let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), rpc_port); diff --git a/iroh/src/client/tags.rs b/iroh/src/client/tags.rs index 0d5a402a3c..b16a9c187c 100644 --- a/iroh/src/client/tags.rs +++ b/iroh/src/client/tags.rs @@ -1,22 +1,25 @@ +//! API for tag management. + use anyhow::Result; use futures_lite::{Stream, StreamExt}; -use iroh_bytes::Tag; +use iroh_bytes::{BlobFormat, Hash, Tag}; use quic_rpc::{RpcClient, ServiceConnection}; +use serde::{Deserialize, Serialize}; -use crate::rpc_protocol::{DeleteTagRequest, ListTagsRequest, ListTagsResponse, ProviderService}; +use crate::rpc_protocol::{DeleteTagRequest, ListTagsRequest, RpcService}; /// Iroh tags client. #[derive(Debug, Clone)] pub struct Client { - pub(super) rpc: RpcClient, + pub(super) rpc: RpcClient, } impl Client where - C: ServiceConnection, + C: ServiceConnection, { /// List all tags. - pub async fn list(&self) -> Result>> { + pub async fn list(&self) -> Result>> { let stream = self.rpc.server_streaming(ListTagsRequest).await?; Ok(stream.map(|res| res.map_err(anyhow::Error::from))) } @@ -27,3 +30,14 @@ where Ok(()) } } + +/// Information about a tag. +#[derive(Debug, Serialize, Deserialize)] +pub struct TagInfo { + /// Name of the tag + pub name: Tag, + /// Format of the data + pub format: BlobFormat, + /// Hash of the data + pub hash: Hash, +} diff --git a/iroh/src/dial.rs b/iroh/src/dial.rs deleted file mode 100644 index 4723ffa303..0000000000 --- a/iroh/src/dial.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Utilities to dial a node. - -use anyhow::Context; -use iroh_net::key::SecretKey; -use iroh_net::relay::{RelayMap, RelayMode}; -use iroh_net::NodeAddr; - -/// Options for the client -#[derive(Clone, Debug)] -pub struct Options { - /// The secret key of the node - pub secret_key: SecretKey, - /// The peer to connect to. - pub peer: NodeAddr, - /// Whether to log the SSL keys when `SSLKEYLOGFILE` environment variable is set - pub keylog: bool, - /// The configuration of the relay services - pub relay_map: Option, -} - -/// Create a new endpoint and dial a peer, returning the connection -/// -/// Note that this will create an entirely new endpoint, so it should be only -/// used for short lived connections. If you want to connect to multiple peers, -/// it is preferable to create an endpoint and use `connect` on the endpoint. -pub async fn dial(opts: Options) -> anyhow::Result { - let endpoint = iroh_net::MagicEndpoint::builder() - .secret_key(opts.secret_key) - .keylog(opts.keylog); - let relay_mode = match opts.relay_map { - Some(relay_map) => RelayMode::Custom(relay_map), - None => RelayMode::Default, - }; - let endpoint = endpoint.relay_mode(relay_mode); - let endpoint = endpoint.bind(0).await?; - endpoint - .connect(opts.peer, iroh_bytes::protocol::ALPN) - .await - .context("failed to connect to provider") -} diff --git a/iroh/src/lib.rs b/iroh/src/lib.rs index 69ce9b98bb..6bf6ba4bfa 100644 --- a/iroh/src/lib.rs +++ b/iroh/src/lib.rs @@ -1,25 +1,31 @@ //! Send data over the internet. +//! +//! ## Feature Flags +//! +//! - `metrics`: Enable metrics collection. Enabled by default. +//! - `fs-store`: Enables the disk based storage backend for `iroh-bytes`. Enabled by default. +//! +#![cfg_attr(docsrs, feature(doc_cfg))] #![deny(missing_docs, rustdoc::broken_intra_doc_links)] // re-export the iroh crates +#[doc(inline)] pub use iroh_base as base; +#[doc(inline)] pub use iroh_bytes as bytes; +#[doc(inline)] pub use iroh_net as net; +#[doc(inline)] pub use iroh_sync as sync; -// reexport types from the iroh_base crate -// iroh_base::hash::* is exported from iroh_bytes as bytes -// iroh_base::rpc::* is exported from mod rpc_protocol -pub use iroh_base::base32; - pub mod client; -pub mod dial; pub mod node; -pub mod rpc_protocol; -pub mod sync_engine; -pub mod ticket; pub mod util; +mod rpc_protocol; +mod sync_engine; + /// Expose metrics module #[cfg(feature = "metrics")] +#[cfg_attr(all(docsrs, feature = "metrics"), doc(cfg(feature = "metrics")))] pub mod metrics; diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 2299691e61..cdd107eb51 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use futures_lite::{future::Boxed as BoxFuture, FutureExt, StreamExt}; +use iroh_base::ticket::BlobTicket; use iroh_bytes::downloader::Downloader; use iroh_bytes::store::Store as BaoStore; use iroh_bytes::BlobFormat; @@ -31,16 +32,15 @@ use tokio_util::sync::CancellationToken; use tokio_util::task::LocalPoolHandle; use tracing::debug; -use crate::rpc_protocol::{ProviderRequest, ProviderResponse}; +use crate::rpc_protocol::{Request, Response}; use crate::sync_engine::SyncEngine; -use crate::ticket::BlobTicket; mod builder; mod rpc; mod rpc_status; -pub use builder::{Builder, GcPolicy, NodeDiscoveryConfig, StorageConfig}; -pub use rpc_status::RpcStatus; +pub use self::builder::{Builder, DiscoveryConfig, GcPolicy, StorageConfig}; +pub use self::rpc_status::RpcStatus; type EventCallback = Box BoxFuture<()> + 'static + Sync + Send>; @@ -88,7 +88,7 @@ impl iroh_bytes::provider::EventSender for Callbacks { pub struct Node { inner: Arc>, task: Arc>, - client: crate::client::mem::Iroh, + client: crate::client::MemIroh, } #[derive(derive_more::Debug)] @@ -97,7 +97,7 @@ struct NodeInner { endpoint: MagicEndpoint, secret_key: SecretKey, cancel_token: CancellationToken, - controller: FlumeConnection, + controller: FlumeConnection, #[debug("callbacks: Sender>")] cb_sender: mpsc::Sender BoxFuture<()> + Send + Sync + 'static>>, callbacks: Callbacks, @@ -197,12 +197,12 @@ impl Node { } /// Returns a handle that can be used to do RPC calls to the node internally. - pub fn controller(&self) -> crate::client::mem::RpcClient { + pub fn controller(&self) -> crate::client::MemRpcClient { RpcClient::new(self.inner.controller.clone()) } /// Return a client to control this node over an in-memory channel. - pub fn client(&self) -> &crate::client::mem::Iroh { + pub fn client(&self) -> &crate::client::MemIroh { &self.client } @@ -254,7 +254,7 @@ impl Node { } impl std::ops::Deref for Node { - type Target = crate::client::mem::Iroh; + type Target = crate::client::MemIroh; fn deref(&self) -> &Self::Target { &self.client @@ -283,11 +283,8 @@ mod tests { use iroh_net::{relay::RelayMode, test_utils::DnsPkarrServer}; use crate::{ - client::BlobAddOutcome, - rpc_protocol::{ - BlobAddPathRequest, BlobAddPathResponse, BlobDownloadRequest, DownloadMode, - SetTagOption, WrapOption, - }, + client::blobs::{AddOutcome, WrapOption}, + rpc_protocol::{BlobAddPathRequest, BlobAddPathResponse, SetTagOption}, }; use super::*; @@ -425,18 +422,11 @@ mod tests { .insecure_skip_relay_cert_verify(true) .spawn() .await?; - let BlobAddOutcome { hash, .. } = node1.blobs.add_bytes(b"foo".to_vec()).await?; + let AddOutcome { hash, .. } = node1.blobs.add_bytes(b"foo".to_vec()).await?; // create a node addr with only a relay URL, no direct addresses let addr = NodeAddr::new(node1.node_id()).with_relay_url(relay_url); - let req = BlobDownloadRequest { - hash, - tag: SetTagOption::Auto, - format: BlobFormat::Raw, - mode: DownloadMode::Direct, - nodes: vec![addr], - }; - node2.blobs.download(req).await?.await?; + node2.blobs.download(hash, addr).await?.await?; assert_eq!( node2 .blobs @@ -479,14 +469,7 @@ mod tests { // create a node addr with node id only let addr = NodeAddr::new(node1.node_id()); - let req = BlobDownloadRequest { - hash, - tag: SetTagOption::Auto, - format: BlobFormat::Raw, - mode: DownloadMode::Direct, - nodes: vec![addr], - }; - node2.blobs.download(req).await?.await?; + node2.blobs.download(hash, addr).await?.await?; assert_eq!( node2 .blobs diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index 993ee78778..f60f03bd79 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -33,14 +33,14 @@ use tokio_util::{sync::CancellationToken, task::LocalPoolHandle}; use tracing::{debug, error, error_span, info, trace, warn, Instrument}; use crate::{ - client::quic::RPC_ALPN, + client::RPC_ALPN, node::{Event, NodeInner}, - rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService}, + rpc_protocol::{Request, Response, RpcService}, sync_engine::SyncEngine, util::{fs::load_secret_key, path::IrohPaths}, }; -use super::{rpc, Callbacks, EventCallback, Node, RpcStatus}; +use super::{rpc, rpc_status::RpcStatus, Callbacks, EventCallback, Node}; pub const PROTOCOLS: [&[u8]; 3] = [iroh_bytes::protocol::ALPN, GOSSIP_ALPN, SYNC_ALPN]; @@ -74,7 +74,7 @@ const MAX_STREAMS: u64 = 10; pub struct Builder where D: Map, - E: ServiceEndpoint, + E: ServiceEndpoint, { storage: StorageConfig, bind_port: Option, @@ -84,8 +84,8 @@ where keylog: bool, relay_mode: RelayMode, gc_policy: GcPolicy, - node_discovery: NodeDiscoveryConfig, dns_resolver: Option, + node_discovery: DiscoveryConfig, docs_store: iroh_sync::store::fs::Store, #[cfg(any(test, feature = "test-utils"))] insecure_skip_relay_cert_verify: bool, @@ -102,7 +102,7 @@ pub enum StorageConfig { /// Configuration for node discovery. #[derive(Debug, Default)] -pub enum NodeDiscoveryConfig { +pub enum DiscoveryConfig { /// Use no node discovery mechanism. None, /// Use the default discovery mechanism. @@ -114,7 +114,7 @@ pub enum NodeDiscoveryConfig { Custom(Box), } -impl From> for NodeDiscoveryConfig { +impl From> for DiscoveryConfig { fn from(value: Box) -> Self { Self::Custom(value) } @@ -168,7 +168,7 @@ impl Builder { impl Builder where D: BaoStore, - E: ServiceEndpoint, + E: ServiceEndpoint, { /// Persist all node data in the provided directory. pub async fn persist( @@ -227,7 +227,7 @@ where } /// Configure rpc endpoint, changing the type of the builder to the new endpoint type. - pub fn rpc_endpoint>(self, value: E2) -> Builder { + pub fn rpc_endpoint>(self, value: E2) -> Builder { // we can't use ..self here because the return type is different Builder { storage: self.storage, @@ -247,9 +247,7 @@ where } /// Configure the default iroh rpc endpoint. - pub async fn enable_rpc( - self, - ) -> Result>> { + pub async fn enable_rpc(self) -> Result>> { let (ep, actual_rpc_port) = make_rpc_endpoint(&self.secret_key, DEFAULT_RPC_PORT)?; if let StorageConfig::Persistent(ref root) = self.storage { // store rpc endpoint @@ -297,9 +295,9 @@ where /// Sets the node discovery mechanism. /// - /// The default is [`NodeDiscoveryConfig::Default`]. Use [`NodeDiscoveryConfig::Custom`] to pass a + /// The default is [`DiscoveryConfig::Default`]. Use [`DiscoveryConfig::Custom`] to pass a /// custom [`Discovery`]. - pub fn node_discovery(mut self, config: NodeDiscoveryConfig) -> Self { + pub fn node_discovery(mut self, config: DiscoveryConfig) -> Self { self.node_discovery = config; self } @@ -365,9 +363,9 @@ where .max_concurrent_uni_streams(0u32.into()); let discovery: Option> = match self.node_discovery { - NodeDiscoveryConfig::None => None, - NodeDiscoveryConfig::Custom(discovery) => Some(discovery), - NodeDiscoveryConfig::Default => { + DiscoveryConfig::None => None, + DiscoveryConfig::Custom(discovery) => Some(discovery), + DiscoveryConfig::Default => { let discovery = ConcurrentDiscovery::from_services(vec![ // Enable DNS discovery by default Box::new(DnsDiscovery::n0_dns()), @@ -515,7 +513,7 @@ where mut cb_receiver: mpsc::Receiver, handler: rpc::Handler, rpc: E, - internal_rpc: impl ServiceEndpoint, + internal_rpc: impl ServiceEndpoint, gossip: Gossip, ) { let rpc = RpcServer::new(rpc); @@ -739,7 +737,7 @@ const MAX_RPC_STREAMS: u32 = 1024; fn make_rpc_endpoint( secret_key: &SecretKey, rpc_port: u16, -) -> Result<(QuinnServerEndpoint, u16)> { +) -> Result<(QuinnServerEndpoint, u16)> { let rpc_addr = SocketAddrV4::new(Ipv4Addr::LOCALHOST, rpc_port); let mut transport_config = quinn::TransportConfig::default(); transport_config @@ -774,8 +772,7 @@ fn make_rpc_endpoint( }; let actual_rpc_port = rpc_quinn_endpoint.local_addr()?.port(); - let rpc_endpoint = - QuinnServerEndpoint::::new(rpc_quinn_endpoint)?; + let rpc_endpoint = QuinnServerEndpoint::::new(rpc_quinn_endpoint)?; Ok((rpc_endpoint, actual_rpc_port)) } diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index 29941fc680..9cdb9aba8b 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -24,7 +24,7 @@ use iroh_bytes::{ HashAndFormat, }; use iroh_io::AsyncSliceReader; -use iroh_net::{MagicEndpoint, NodeAddr}; +use iroh_net::{MagicEndpoint, NodeAddr, NodeId}; use quic_rpc::{ server::{RpcChannel, RpcServerError}, ServiceEndpoint, @@ -32,20 +32,23 @@ use quic_rpc::{ use tokio_util::task::LocalPoolHandle; use tracing::{debug, info}; +use crate::client::blobs::{ + BlobInfo, CollectionInfo, DownloadMode, IncompleteBlobInfo, WrapOption, +}; +use crate::client::node::NodeStatus; +use crate::client::tags::TagInfo; use crate::rpc_protocol::{ BlobAddPathRequest, BlobAddPathResponse, BlobAddStreamRequest, BlobAddStreamResponse, BlobAddStreamUpdate, BlobConsistencyCheckRequest, BlobDeleteBlobRequest, BlobDownloadRequest, BlobDownloadResponse, BlobExportRequest, BlobExportResponse, BlobGetCollectionRequest, - BlobGetCollectionResponse, BlobListCollectionsRequest, BlobListCollectionsResponse, - BlobListIncompleteRequest, BlobListIncompleteResponse, BlobListRequest, BlobListResponse, - BlobReadAtRequest, BlobReadAtResponse, BlobValidateRequest, CreateCollectionRequest, - CreateCollectionResponse, DeleteTagRequest, DocExportFileRequest, DocExportFileResponse, - DocImportFileRequest, DocImportFileResponse, DocImportProgress, DocSetHashRequest, - DownloadMode, ListTagsRequest, ListTagsResponse, NodeConnectionInfoRequest, - NodeConnectionInfoResponse, NodeConnectionsRequest, NodeConnectionsResponse, - NodeShutdownRequest, NodeStatsRequest, NodeStatsResponse, NodeStatusRequest, - NodeStatusResponse, NodeWatchRequest, NodeWatchResponse, ProviderRequest, ProviderService, - SetTagOption, + BlobGetCollectionResponse, BlobListCollectionsRequest, BlobListIncompleteRequest, + BlobListRequest, BlobReadAtRequest, BlobReadAtResponse, BlobValidateRequest, + CreateCollectionRequest, CreateCollectionResponse, DeleteTagRequest, DocExportFileRequest, + DocExportFileResponse, DocImportFileRequest, DocImportFileResponse, DocImportProgress, + DocSetHashRequest, ListTagsRequest, NodeConnectionInfoRequest, NodeConnectionInfoResponse, + NodeConnectionsRequest, NodeConnectionsResponse, NodeIdRequest, NodeShutdownRequest, + NodeStatsRequest, NodeStatsResponse, NodeStatusRequest, NodeWatchRequest, NodeWatchResponse, + Request, RpcService, SetTagOption, }; use super::{Event, NodeInner}; @@ -62,18 +65,19 @@ pub(crate) struct Handler { } impl Handler { - pub(crate) fn handle_rpc_request>( + pub(crate) fn handle_rpc_request>( &self, - msg: ProviderRequest, - chan: RpcChannel, + msg: Request, + chan: RpcChannel, ) { let handler = self.clone(); tokio::task::spawn(async move { - use ProviderRequest::*; + use Request::*; debug!("handling rpc request: {msg}"); match msg { NodeWatch(msg) => chan.server_streaming(msg, handler, Self::node_watch).await, NodeStatus(msg) => chan.rpc(msg, handler, Self::node_status).await, + NodeId(msg) => chan.rpc(msg, handler, Self::node_id).await, NodeShutdown(msg) => chan.rpc(msg, handler, Self::node_shutdown).await, NodeStats(msg) => chan.rpc(msg, handler, Self::node_stats).await, NodeConnections(msg) => { @@ -285,7 +289,7 @@ impl Handler { self.inner.rt.clone() } - async fn blob_list_impl(self, co: &Co>) -> io::Result<()> { + async fn blob_list_impl(self, co: &Co>) -> io::Result<()> { use bao_tree::io::fsm::Outboard; let db = self.inner.db.clone(); @@ -297,14 +301,14 @@ impl Handler { let hash = entry.hash(); let size = entry.outboard().await?.tree().size(); let path = "".to_owned(); - co.yield_(Ok(BlobListResponse { hash, size, path })).await; + co.yield_(Ok(BlobInfo { hash, size, path })).await; } Ok(()) } async fn blob_list_incomplete_impl( self, - co: &Co>, + co: &Co>, ) -> io::Result<()> { let db = self.inner.db.clone(); for hash in db.partial_blobs().await? { @@ -317,7 +321,7 @@ impl Handler { } let size = 0; let expected_size = entry.size().value(); - co.yield_(Ok(BlobListIncompleteResponse { + co.yield_(Ok(IncompleteBlobInfo { hash, size, expected_size, @@ -329,7 +333,7 @@ impl Handler { async fn blob_list_collections_impl( self, - co: &Co>, + co: &Co>, ) -> anyhow::Result<()> { let db = self.inner.db.clone(); let local = self.inner.rt.clone(); @@ -349,7 +353,7 @@ impl Handler { anyhow::Ok(count) }) .await??; - co.yield_(Ok(BlobListCollectionsResponse { + co.yield_(Ok(CollectionInfo { tag: name, hash, total_blobs_count: Some(count), @@ -363,7 +367,7 @@ impl Handler { fn blob_list( self, _msg: BlobListRequest, - ) -> impl Stream> + Send + 'static { + ) -> impl Stream> + Send + 'static { Gen::new(|co| async move { if let Err(e) = self.blob_list_impl(&co).await { co.yield_(Err(e.into())).await; @@ -374,7 +378,7 @@ impl Handler { fn blob_list_incomplete( self, _msg: BlobListIncompleteRequest, - ) -> impl Stream> + Send + 'static { + ) -> impl Stream> + Send + 'static { Gen::new(move |co| async move { if let Err(e) = self.blob_list_incomplete_impl(&co).await { co.yield_(Err(e.into())).await; @@ -385,7 +389,7 @@ impl Handler { fn blob_list_collections( self, _msg: BlobListCollectionsRequest, - ) -> impl Stream> + Send + 'static { + ) -> impl Stream> + Send + 'static { Gen::new(move |co| async move { if let Err(e) = self.blob_list_collections_impl(&co).await { co.yield_(Err(e.into())).await; @@ -403,10 +407,7 @@ impl Handler { Ok(()) } - fn blob_list_tags( - self, - _msg: ListTagsRequest, - ) -> impl Stream + Send + 'static { + fn blob_list_tags(self, _msg: ListTagsRequest) -> impl Stream + Send + 'static { tracing::info!("blob_list_tags"); Gen::new(|co| async move { let tags = self.inner.db.tags().await.unwrap(); @@ -414,7 +415,7 @@ impl Handler { for item in tags { if let Ok((name, HashAndFormat { hash, format })) = item { tracing::info!("{:?} {} {:?}", name, hash, format); - co.yield_(ListTagsResponse { name, hash, format }).await; + co.yield_(TagInfo { name, hash, format }).await; } } }) @@ -649,7 +650,6 @@ impl Handler { msg: BlobAddPathRequest, progress: flume::Sender, ) -> anyhow::Result<()> { - use crate::rpc_protocol::WrapOption; use iroh_bytes::store::ImportMode; use std::collections::BTreeMap; @@ -780,8 +780,8 @@ impl Handler { res } - async fn node_status(self, _: NodeStatusRequest) -> RpcResult { - Ok(NodeStatusResponse { + async fn node_status(self, _: NodeStatusRequest) -> RpcResult { + Ok(NodeStatus { addr: self.inner.endpoint.my_addr().await?, listen_addrs: self .inner @@ -792,6 +792,11 @@ impl Handler { }) } + #[allow(clippy::unused_async)] + async fn node_id(self, _: NodeIdRequest) -> RpcResult { + Ok(self.inner.secret_key.public()) + } + #[allow(clippy::unused_async)] async fn node_shutdown(self, request: NodeShutdownRequest) { if request.force { diff --git a/iroh/src/node/rpc_status.rs b/iroh/src/node/rpc_status.rs index f5ea1b0981..00ad0c8ab2 100644 --- a/iroh/src/node/rpc_status.rs +++ b/iroh/src/node/rpc_status.rs @@ -16,7 +16,7 @@ pub enum RpcStatus { /// The port we are connected on. port: u16, /// Actual connected RPC client. - client: crate::client::quic::RpcClient, + client: crate::client::QuicRpcClient, }, } @@ -40,7 +40,7 @@ impl RpcStatus { .await .context("read rpc lock file")?; let running_rpc_port = u16::from_le_bytes(buffer); - if let Ok(client) = crate::client::quic::connect_raw(running_rpc_port).await { + if let Ok(client) = crate::client::quic_connect_raw(running_rpc_port).await { return Ok(RpcStatus::Running { port: running_rpc_port, client, diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 14ab810946..4af54cb922 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -7,7 +7,7 @@ //! response, while others like provide have a stream of responses. //! //! Note that this is subject to change. The RPC protocol is not yet stable. -use std::{collections::BTreeMap, net::SocketAddr, path::PathBuf}; +use std::{collections::BTreeMap, path::PathBuf}; use bytes::Bytes; use derive_more::{From, TryInto}; @@ -21,12 +21,13 @@ use iroh_bytes::{ use iroh_net::{ key::PublicKey, magic_endpoint::{ConnectionInfo, NodeAddr}, + NodeId, }; use iroh_sync::{ actor::OpenState, store::{DownloadPolicy, Query}, - Author, PeerIdBytes, {AuthorId, CapabilityKind, Entry, NamespaceId, SignedEntry}, + Author, AuthorId, CapabilityKind, DocTicket, Entry, NamespaceId, PeerIdBytes, SignedEntry, }; use quic_rpc::{ message::{BidiStreaming, BidiStreamingMsg, Msg, RpcMsg, ServerStreaming, ServerStreamingMsg}, @@ -39,13 +40,17 @@ pub use iroh_base::rpc::{RpcError, RpcResult}; use iroh_bytes::store::{ExportFormat, ExportMode}; pub use iroh_bytes::{provider::AddProgress, store::ValidateProgress}; -use crate::sync_engine::LiveEvent; -pub use crate::ticket::DocTicket; +use crate::{ + client::{ + blobs::{BlobInfo, CollectionInfo, DownloadMode, IncompleteBlobInfo, WrapOption}, + docs::ShareMode, + node::NodeStatus, + tags::TagInfo, + }, + sync_engine::LiveEvent, +}; pub use iroh_bytes::util::SetTagOption; -/// A 32-byte key or token -pub type KeyBytes = [u8; 32]; - /// A request to the node to provide the data at the given path /// /// Will produce a stream of [`AddProgress`] messages. @@ -66,23 +71,11 @@ pub struct BlobAddPathRequest { pub wrap: WrapOption, } -/// Whether to wrap the added data in a collection. -#[derive(Debug, Serialize, Deserialize)] -pub enum WrapOption { - /// Do not wrap the file or directory. - NoWrap, - /// Wrap the file or directory in a collection. - Wrap { - /// Override the filename in the wrapping collection. - name: Option, - }, -} - -impl Msg for BlobAddPathRequest { +impl Msg for BlobAddPathRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for BlobAddPathRequest { +impl ServerStreamingMsg for BlobAddPathRequest { type Response = BlobAddPathResponse; } @@ -107,29 +100,15 @@ pub struct BlobDownloadRequest { pub nodes: Vec, /// Optional tag to tag the data with. pub tag: SetTagOption, - /// Whether to directly start the download or add it to the downlod queue. + /// Whether to directly start the download or add it to the download queue. pub mode: DownloadMode, } -/// Set the mode for whether to directly start the download or add it to the download queue. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum DownloadMode { - /// Start the download right away. - /// - /// No concurrency limits or queuing will be applied. It is up to the user to manage download - /// concurrency. - Direct, - /// Queue the download. - /// - /// The download queue will be processed in-order, while respecting the downloader concurrency limits. - Queued, -} - -impl Msg for BlobDownloadRequest { +impl Msg for BlobDownloadRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for BlobDownloadRequest { +impl ServerStreamingMsg for BlobDownloadRequest { type Response = BlobDownloadResponse; } @@ -156,11 +135,11 @@ pub struct BlobExportRequest { pub mode: ExportMode, } -impl Msg for BlobExportRequest { +impl Msg for BlobExportRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for BlobExportRequest { +impl ServerStreamingMsg for BlobExportRequest { type Response = BlobExportResponse; } @@ -175,11 +154,11 @@ pub struct BlobConsistencyCheckRequest { pub repair: bool, } -impl Msg for BlobConsistencyCheckRequest { +impl Msg for BlobConsistencyCheckRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for BlobConsistencyCheckRequest { +impl ServerStreamingMsg for BlobConsistencyCheckRequest { type Response = ConsistencyCheckProgress; } @@ -190,11 +169,11 @@ pub struct BlobValidateRequest { pub repair: bool, } -impl Msg for BlobValidateRequest { +impl Msg for BlobValidateRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for BlobValidateRequest { +impl ServerStreamingMsg for BlobValidateRequest { type Response = ValidateProgress; } @@ -202,46 +181,24 @@ impl ServerStreamingMsg for BlobValidateRequest { #[derive(Debug, Serialize, Deserialize)] pub struct BlobListRequest; -/// A response to a list blobs request -#[derive(Debug, Serialize, Deserialize)] -pub struct BlobListResponse { - /// Location of the blob - pub path: String, - /// The hash of the blob - pub hash: Hash, - /// The size of the blob - pub size: u64, -} - -impl Msg for BlobListRequest { +impl Msg for BlobListRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for BlobListRequest { - type Response = RpcResult; +impl ServerStreamingMsg for BlobListRequest { + type Response = RpcResult; } /// List all blobs, including collections #[derive(Debug, Serialize, Deserialize)] pub struct BlobListIncompleteRequest; -/// A response to a list blobs request -#[derive(Debug, Serialize, Deserialize)] -pub struct BlobListIncompleteResponse { - /// The size we got - pub size: u64, - /// The size we expect - pub expected_size: u64, - /// The hash of the blob - pub hash: Hash, -} - -impl Msg for BlobListIncompleteRequest { +impl Msg for BlobListIncompleteRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for BlobListIncompleteRequest { - type Response = RpcResult; +impl ServerStreamingMsg for BlobListIncompleteRequest { + type Response = RpcResult; } /// List all collections @@ -250,30 +207,12 @@ impl ServerStreamingMsg for BlobListIncompleteRequest { #[derive(Debug, Serialize, Deserialize)] pub struct BlobListCollectionsRequest; -/// A response to a list collections request -#[derive(Debug, Serialize, Deserialize)] -pub struct BlobListCollectionsResponse { - /// Tag of the collection - pub tag: Tag, - - /// Hash of the collection - pub hash: Hash, - /// Number of children in the collection - /// - /// This is an optional field, because the data is not always available. - pub total_blobs_count: Option, - /// Total size of the raw data referred to by all links - /// - /// This is an optional field, because the data is not always available. - pub total_blobs_size: Option, -} - -impl Msg for BlobListCollectionsRequest { +impl Msg for BlobListCollectionsRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for BlobListCollectionsRequest { - type Response = RpcResult; +impl ServerStreamingMsg for BlobListCollectionsRequest { + type Response = RpcResult; } /// List all collections @@ -282,23 +221,12 @@ impl ServerStreamingMsg for BlobListCollectionsRequest { #[derive(Debug, Serialize, Deserialize)] pub struct ListTagsRequest; -/// A response to a list collections request -#[derive(Debug, Serialize, Deserialize)] -pub struct ListTagsResponse { - /// Name of the tag - pub name: Tag, - /// Format of the data - pub format: BlobFormat, - /// Hash of the data - pub hash: Hash, -} - -impl Msg for ListTagsRequest { +impl Msg for ListTagsRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for ListTagsRequest { - type Response = ListTagsResponse; +impl ServerStreamingMsg for ListTagsRequest { + type Response = TagInfo; } /// Delete a blob @@ -308,7 +236,7 @@ pub struct BlobDeleteBlobRequest { pub hash: Hash, } -impl RpcMsg for BlobDeleteBlobRequest { +impl RpcMsg for BlobDeleteBlobRequest { type Response = RpcResult<()>; } @@ -319,7 +247,7 @@ pub struct DeleteTagRequest { pub name: Tag, } -impl RpcMsg for DeleteTagRequest { +impl RpcMsg for DeleteTagRequest { type Response = RpcResult<()>; } @@ -330,7 +258,7 @@ pub struct BlobGetCollectionRequest { pub hash: Hash, } -impl RpcMsg for BlobGetCollectionRequest { +impl RpcMsg for BlobGetCollectionRequest { type Response = RpcResult; } @@ -361,7 +289,7 @@ pub struct CreateCollectionResponse { pub tag: Tag, } -impl RpcMsg for CreateCollectionRequest { +impl RpcMsg for CreateCollectionRequest { type Response = RpcResult; } @@ -379,11 +307,11 @@ pub struct NodeConnectionsResponse { pub conn_info: ConnectionInfo, } -impl Msg for NodeConnectionsRequest { +impl Msg for NodeConnectionsRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for NodeConnectionsRequest { +impl ServerStreamingMsg for NodeConnectionsRequest { type Response = RpcResult; } @@ -401,7 +329,7 @@ pub struct NodeConnectionInfoResponse { pub conn_info: Option, } -impl RpcMsg for NodeConnectionInfoRequest { +impl RpcMsg for NodeConnectionInfoRequest { type Response = RpcResult; } @@ -412,40 +340,35 @@ pub struct NodeShutdownRequest { pub force: bool, } -impl RpcMsg for NodeShutdownRequest { +impl RpcMsg for NodeShutdownRequest { type Response = (); } -/// A request to get information about the identity of the node -/// -/// See [`NodeStatusResponse`] for the response. +/// A request to get information about the status of the node. #[derive(Serialize, Deserialize, Debug)] pub struct NodeStatusRequest; -impl RpcMsg for NodeStatusRequest { - type Response = RpcResult; +impl RpcMsg for NodeStatusRequest { + type Response = RpcResult; } -/// The response to a version request +/// A request to get information the identity of the node. #[derive(Serialize, Deserialize, Debug)] -pub struct NodeStatusResponse { - /// The node id and socket addresses of this node. - pub addr: NodeAddr, - /// The bound listening addresses of the node - pub listen_addrs: Vec, - /// The version of the node - pub version: String, +pub struct NodeIdRequest; + +impl RpcMsg for NodeIdRequest { + type Response = RpcResult; } /// A request to watch for the node status #[derive(Serialize, Deserialize, Debug)] pub struct NodeWatchRequest; -impl Msg for NodeWatchRequest { +impl Msg for NodeWatchRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for NodeWatchRequest { +impl ServerStreamingMsg for NodeWatchRequest { type Response = NodeWatchResponse; } @@ -469,11 +392,11 @@ pub struct VersionResponse { #[derive(Serialize, Deserialize, Debug)] pub struct AuthorListRequest {} -impl Msg for AuthorListRequest { +impl Msg for AuthorListRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for AuthorListRequest { +impl ServerStreamingMsg for AuthorListRequest { type Response = RpcResult; } @@ -488,7 +411,7 @@ pub struct AuthorListResponse { #[derive(Serialize, Deserialize, Debug)] pub struct AuthorCreateRequest; -impl RpcMsg for AuthorCreateRequest { +impl RpcMsg for AuthorCreateRequest { type Response = RpcResult; } @@ -506,7 +429,7 @@ pub struct AuthorDeleteRequest { pub author: AuthorId, } -impl RpcMsg for AuthorDeleteRequest { +impl RpcMsg for AuthorDeleteRequest { type Response = RpcResult; } @@ -521,7 +444,7 @@ pub struct AuthorExportRequest { pub author: AuthorId, } -impl RpcMsg for AuthorExportRequest { +impl RpcMsg for AuthorExportRequest { type Response = RpcResult; } @@ -539,7 +462,7 @@ pub struct AuthorImportRequest { pub author: Author, } -impl RpcMsg for AuthorImportRequest { +impl RpcMsg for AuthorImportRequest { type Response = RpcResult; } @@ -550,15 +473,6 @@ pub struct AuthorImportResponse { pub author_id: AuthorId, } -/// Intended capability for document share tickets -#[derive(Serialize, Deserialize, Debug, Clone)] -pub enum ShareMode { - /// Read-only access - Read, - /// Write access - Write, -} - /// Subscribe to events for a document. #[derive(Serialize, Deserialize, Debug)] pub struct DocSubscribeRequest { @@ -566,11 +480,11 @@ pub struct DocSubscribeRequest { pub doc_id: NamespaceId, } -impl Msg for DocSubscribeRequest { +impl Msg for DocSubscribeRequest { type Pattern = TryServerStreaming; } -impl TryServerStreamingMsg for DocSubscribeRequest { +impl TryServerStreamingMsg for DocSubscribeRequest { type Item = DocSubscribeResponse; type ItemError = RpcError; type CreateError = RpcError; @@ -587,11 +501,11 @@ pub struct DocSubscribeResponse { #[derive(Serialize, Deserialize, Debug)] pub struct DocListRequest {} -impl Msg for DocListRequest { +impl Msg for DocListRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for DocListRequest { +impl ServerStreamingMsg for DocListRequest { type Response = RpcResult; } @@ -608,7 +522,7 @@ pub struct DocListResponse { #[derive(Serialize, Deserialize, Debug)] pub struct DocCreateRequest {} -impl RpcMsg for DocCreateRequest { +impl RpcMsg for DocCreateRequest { type Response = RpcResult; } @@ -623,7 +537,7 @@ pub struct DocCreateResponse { #[derive(Serialize, Deserialize, Debug)] pub struct DocImportRequest(pub DocTicket); -impl RpcMsg for DocImportRequest { +impl RpcMsg for DocImportRequest { type Response = RpcResult; } @@ -645,7 +559,7 @@ pub struct DocShareRequest { pub addr_options: AddrInfoOptions, } -impl RpcMsg for DocShareRequest { +impl RpcMsg for DocShareRequest { type Response = RpcResult; } @@ -660,7 +574,7 @@ pub struct DocStatusRequest { pub doc_id: NamespaceId, } -impl RpcMsg for DocStatusRequest { +impl RpcMsg for DocStatusRequest { type Response = RpcResult; } @@ -679,7 +593,7 @@ pub struct DocOpenRequest { pub doc_id: NamespaceId, } -impl RpcMsg for DocOpenRequest { +impl RpcMsg for DocOpenRequest { type Response = RpcResult; } @@ -694,7 +608,7 @@ pub struct DocCloseRequest { pub doc_id: NamespaceId, } -impl RpcMsg for DocCloseRequest { +impl RpcMsg for DocCloseRequest { type Response = RpcResult; } @@ -711,7 +625,7 @@ pub struct DocStartSyncRequest { pub peers: Vec, } -impl RpcMsg for DocStartSyncRequest { +impl RpcMsg for DocStartSyncRequest { type Response = RpcResult; } @@ -726,7 +640,7 @@ pub struct DocLeaveRequest { pub doc_id: NamespaceId, } -impl RpcMsg for DocLeaveRequest { +impl RpcMsg for DocLeaveRequest { type Response = RpcResult; } @@ -741,7 +655,7 @@ pub struct DocDropRequest { pub doc_id: NamespaceId, } -impl RpcMsg for DocDropRequest { +impl RpcMsg for DocDropRequest { type Response = RpcResult; } @@ -764,7 +678,7 @@ pub struct DocSetRequest { pub value: Bytes, } -impl RpcMsg for DocSetRequest { +impl RpcMsg for DocSetRequest { type Response = RpcResult; } @@ -797,11 +711,11 @@ pub struct DocImportFileRequest { pub in_place: bool, } -impl Msg for DocImportFileRequest { +impl Msg for DocImportFileRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for DocImportFileRequest { +impl ServerStreamingMsg for DocImportFileRequest { type Response = DocImportFileResponse; } @@ -867,11 +781,11 @@ pub struct DocExportFileRequest { pub mode: ExportMode, } -impl Msg for DocExportFileRequest { +impl Msg for DocExportFileRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for DocExportFileRequest { +impl ServerStreamingMsg for DocExportFileRequest { type Response = DocExportFileResponse; } @@ -893,7 +807,7 @@ pub struct DocDelRequest { pub prefix: Bytes, } -impl RpcMsg for DocDelRequest { +impl RpcMsg for DocDelRequest { type Response = RpcResult; } @@ -919,7 +833,7 @@ pub struct DocSetHashRequest { pub size: u64, } -impl RpcMsg for DocSetHashRequest { +impl RpcMsg for DocSetHashRequest { type Response = RpcResult; } @@ -936,11 +850,11 @@ pub struct DocGetManyRequest { pub query: Query, } -impl Msg for DocGetManyRequest { +impl Msg for DocGetManyRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for DocGetManyRequest { +impl ServerStreamingMsg for DocGetManyRequest { type Response = RpcResult; } @@ -964,7 +878,7 @@ pub struct DocGetExactRequest { pub include_empty: bool, } -impl RpcMsg for DocGetExactRequest { +impl RpcMsg for DocGetExactRequest { type Response = RpcResult; } @@ -984,7 +898,7 @@ pub struct DocSetDownloadPolicyRequest { pub policy: DownloadPolicy, } -impl RpcMsg for DocSetDownloadPolicyRequest { +impl RpcMsg for DocSetDownloadPolicyRequest { type Response = RpcResult; } @@ -999,7 +913,7 @@ pub struct DocGetDownloadPolicyRequest { pub doc_id: NamespaceId, } -impl RpcMsg for DocGetDownloadPolicyRequest { +impl RpcMsg for DocGetDownloadPolicyRequest { type Response = RpcResult; } @@ -1017,7 +931,7 @@ pub struct DocGetSyncPeersRequest { pub doc_id: NamespaceId, } -impl RpcMsg for DocGetSyncPeersRequest { +impl RpcMsg for DocGetSyncPeersRequest { type Response = RpcResult; } @@ -1039,11 +953,11 @@ pub struct BlobReadAtRequest { pub len: Option, } -impl Msg for BlobReadAtRequest { +impl Msg for BlobReadAtRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for BlobReadAtRequest { +impl ServerStreamingMsg for BlobReadAtRequest { type Response = RpcResult; } @@ -1080,11 +994,11 @@ pub enum BlobAddStreamUpdate { Abort, } -impl Msg for BlobAddStreamRequest { +impl Msg for BlobAddStreamRequest { type Pattern = BidiStreaming; } -impl BidiStreamingMsg for BlobAddStreamRequest { +impl BidiStreamingMsg for BlobAddStreamRequest { type Update = BlobAddStreamUpdate; type Response = BlobAddStreamResponse; } @@ -1097,7 +1011,7 @@ pub struct BlobAddStreamResponse(pub AddProgress); #[derive(Serialize, Deserialize, Debug)] pub struct NodeStatsRequest {} -impl RpcMsg for NodeStatsRequest { +impl RpcMsg for NodeStatsRequest { type Response = RpcResult; } @@ -1119,13 +1033,14 @@ pub struct NodeStatsResponse { /// The RPC service for the iroh provider process. #[derive(Debug, Clone)] -pub struct ProviderService; +pub struct RpcService; /// The request enum, listing all possible requests. #[allow(missing_docs)] #[derive(strum::Display, Debug, Serialize, Deserialize, From, TryInto)] -pub enum ProviderRequest { +pub enum Request { NodeStatus(NodeStatusRequest), + NodeId(NodeIdRequest), NodeStats(NodeStatsRequest), NodeShutdown(NodeShutdownRequest), NodeConnections(NodeConnectionsRequest), @@ -1182,8 +1097,9 @@ pub enum ProviderRequest { /// The response enum, listing all possible responses. #[allow(missing_docs, clippy::large_enum_variant)] #[derive(Debug, Serialize, Deserialize, From, TryInto)] -pub enum ProviderResponse { - NodeStatus(RpcResult), +pub enum Response { + NodeStatus(RpcResult), + NodeId(RpcResult), NodeStats(RpcResult), NodeConnections(RpcResult), NodeConnectionInfo(RpcResult), @@ -1193,9 +1109,9 @@ pub enum ProviderResponse { BlobReadAt(RpcResult), BlobAddStream(BlobAddStreamResponse), BlobAddPath(BlobAddPathResponse), - BlobList(RpcResult), - BlobListIncomplete(RpcResult), - BlobListCollections(RpcResult), + BlobList(RpcResult), + BlobListIncomplete(RpcResult), + BlobListCollections(RpcResult), BlobDownload(BlobDownloadResponse), BlobFsck(ConsistencyCheckProgress), BlobExport(BlobExportResponse), @@ -1203,7 +1119,7 @@ pub enum ProviderResponse { CreateCollection(RpcResult), BlobGetCollection(RpcResult), - ListTags(ListTagsResponse), + ListTags(TagInfo), DeleteTag(RpcResult<()>), DocOpen(RpcResult), @@ -1236,7 +1152,7 @@ pub enum ProviderResponse { AuthorDelete(RpcResult), } -impl Service for ProviderService { - type Req = ProviderRequest; - type Res = ProviderResponse; +impl Service for RpcService { + type Req = Request; + type Res = Response; } diff --git a/iroh/src/sync_engine.rs b/iroh/src/sync_engine.rs index 69d426ac9e..ec21f8be0a 100644 --- a/iroh/src/sync_engine.rs +++ b/iroh/src/sync_engine.rs @@ -26,7 +26,6 @@ use live::{LiveActor, ToLiveActor}; pub use self::live::SyncEvent; pub use self::state::{Origin, SyncReason}; -pub use iroh_sync::net::SYNC_ALPN; /// Capacity of the channel for the [`ToLiveActor`] messages. const ACTOR_CHANNEL_CAP: usize = 64; @@ -43,6 +42,7 @@ pub struct SyncEngine { pub(crate) endpoint: MagicEndpoint, pub(crate) sync: SyncHandle, to_live_actor: mpsc::Sender, + #[allow(dead_code)] actor_handle: SharedAbortingJoinHandle<()>, #[debug("ContentStatusCallback")] content_status_cb: ContentStatusCallback, @@ -53,7 +53,7 @@ impl SyncEngine { /// /// This will spawn two tokio tasks for the live sync coordination and gossip actors, and a /// thread for the [`iroh_sync::actor::SyncHandle`]. - pub fn spawn( + pub(crate) fn spawn( endpoint: MagicEndpoint, gossip: Gossip, replica_store: iroh_sync::store::Store, @@ -108,7 +108,7 @@ impl SyncEngine { /// /// If `peers` is non-empty, it will both do an initial set-reconciliation sync with each peer, /// and join an iroh-gossip swarm with these peers to receive and broadcast document updates. - pub async fn start_sync(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { + async fn start_sync(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { let (reply, reply_rx) = oneshot::channel(); self.to_live_actor .send(ToLiveActor::StartSync { @@ -121,25 +121,11 @@ impl SyncEngine { Ok(()) } - /// Join and sync with a set of peers for a document that is already syncing. - pub async fn join_peers(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { - let (reply, reply_rx) = oneshot::channel(); - self.to_live_actor - .send(ToLiveActor::JoinPeers { - namespace, - peers, - reply, - }) - .await?; - reply_rx.await??; - Ok(()) - } - /// Stop the live sync for a document and leave the gossip swarm. /// /// If `kill_subscribers` is true, all existing event subscribers will be dropped. This means /// they will receive `None` and no further events in case of rejoining the document. - pub async fn leave(&self, namespace: NamespaceId, kill_subscribers: bool) -> Result<()> { + async fn leave(&self, namespace: NamespaceId, kill_subscribers: bool) -> Result<()> { let (reply, reply_rx) = oneshot::channel(); self.to_live_actor .send(ToLiveActor::Leave { @@ -153,7 +139,7 @@ impl SyncEngine { } /// Subscribe to replica and sync progress events. - pub async fn subscribe( + async fn subscribe( &self, namespace: NamespaceId, ) -> Result> + Unpin + 'static> { @@ -190,7 +176,7 @@ impl SyncEngine { } /// Handle an incoming iroh-sync connection. - pub async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { + pub(super) async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { self.to_live_actor .send(ToLiveActor::HandleConnection { conn }) .await?; @@ -201,14 +187,6 @@ impl SyncEngine { self.to_live_actor.send(ToLiveActor::Shutdown).await?; Ok(()) } - - /// Shutdown the sync engine. - pub async fn shutdown(self) -> Result<()> { - self.to_live_actor.send(ToLiveActor::Shutdown).await?; - - self.actor_handle.await.map_err(|e| anyhow::anyhow!(e))?; - Ok(()) - } } pub(crate) fn entry_to_content_status(entry: io::Result) -> ContentStatus { diff --git a/iroh/src/sync_engine/live.rs b/iroh/src/sync_engine/live.rs index f45d99f651..d22c77f950 100644 --- a/iroh/src/sync_engine/live.rs +++ b/iroh/src/sync_engine/live.rs @@ -60,12 +60,6 @@ pub enum ToLiveActor { #[debug("onsehot::Sender")] reply: sync::oneshot::Sender>, }, - JoinPeers { - namespace: NamespaceId, - peers: Vec, - #[debug("onsehot::Sender")] - reply: sync::oneshot::Sender>, - }, Leave { namespace: NamespaceId, kill_subscribers: bool, @@ -305,14 +299,6 @@ impl LiveActor { let res = self.leave(namespace, kill_subscribers).await; reply.send(res).ok(); } - ToLiveActor::JoinPeers { - namespace, - peers, - reply, - } => { - let res = self.join_peers(namespace, peers).await; - reply.send(res).ok(); - } ToLiveActor::Subscribe { namespace, sender, diff --git a/iroh/src/sync_engine/rpc.rs b/iroh/src/sync_engine/rpc.rs index 605f875f2b..6749d715b0 100644 --- a/iroh/src/sync_engine/rpc.rs +++ b/iroh/src/sync_engine/rpc.rs @@ -3,9 +3,10 @@ use anyhow::anyhow; use futures_lite::Stream; use iroh_bytes::{store::Store as BaoStore, BlobFormat}; -use iroh_sync::{Author, NamespaceSecret}; +use iroh_sync::{Author, DocTicket, NamespaceSecret}; use tokio_stream::StreamExt; +use crate::client::docs::ShareMode; use crate::rpc_protocol::{ AuthorDeleteRequest, AuthorDeleteResponse, AuthorExportRequest, AuthorExportResponse, AuthorImportRequest, AuthorImportResponse, DocGetSyncPeersRequest, DocGetSyncPeersResponse, @@ -21,7 +22,7 @@ use crate::{ DocSetDownloadPolicyRequest, DocSetDownloadPolicyResponse, DocSetHashRequest, DocSetHashResponse, DocSetRequest, DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocStartSyncResponse, DocStatusRequest, DocStatusResponse, - DocSubscribeRequest, DocSubscribeResponse, DocTicket, RpcResult, ShareMode, + DocSubscribeRequest, DocSubscribeResponse, RpcResult, }, sync_engine::SyncEngine, }; diff --git a/iroh/src/ticket.rs b/iroh/src/ticket.rs deleted file mode 100644 index cbc5b6a695..0000000000 --- a/iroh/src/ticket.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Tickets that iroh supports -mod doc; - -pub use doc::DocTicket; -pub use iroh_base::ticket::{BlobTicket, NodeTicket}; diff --git a/iroh/src/util/fs.rs b/iroh/src/util/fs.rs index 0eefe33bc1..d1af9650b0 100644 --- a/iroh/src/util/fs.rs +++ b/iroh/src/util/fs.rs @@ -11,7 +11,7 @@ use iroh_net::key::SecretKey; use tokio::io::AsyncWriteExt; use walkdir::WalkDir; -use crate::rpc_protocol::WrapOption; +use crate::client::blobs::WrapOption; /// A data source #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] diff --git a/iroh/tests/provide.rs b/iroh/tests/provide.rs index 938dfacfea..ec9eade08a 100644 --- a/iroh/tests/provide.rs +++ b/iroh/tests/provide.rs @@ -5,14 +5,11 @@ use std::{ time::{Duration, Instant}, }; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use bytes::Bytes; use futures_lite::FutureExt; -use iroh::{ - dial::Options, - node::{Builder, Event}, -}; -use iroh_net::{key::SecretKey, NodeId}; +use iroh::node::{Builder, Event}; +use iroh_net::{defaults::default_relay_map, key::SecretKey, NodeAddr, NodeId}; use quic_rpc::transport::misc::DummyServerEndpoint; use rand::RngCore; use tokio::sync::mpsc; @@ -31,6 +28,18 @@ use iroh_bytes::{ BlobFormat, Hash, }; +/// Create a new endpoint and dial a peer, returning the connection. +async fn dial(secret_key: SecretKey, peer: NodeAddr) -> anyhow::Result { + let endpoint = iroh_net::MagicEndpoint::builder() + .secret_key(secret_key) + .bind(0) + .await?; + endpoint + .connect(peer, iroh::bytes::protocol::ALPN) + .await + .context("failed to connect to provider") +} + fn test_node(db: D) -> Builder { let store = iroh_sync::store::Store::memory(); iroh::node::Builder::with_db_and_store(db, store, iroh::node::StorageConfig::Mem).bind_port(0) @@ -118,19 +127,14 @@ async fn empty_files() -> Result<()> { /// Create new get options with the given node id and addresses, using a /// randomly generated secret key. -fn get_options(node_id: NodeId, addrs: Vec) -> iroh::dial::Options { - let relay_map = iroh_net::defaults::default_relay_map(); +fn get_options(node_id: NodeId, addrs: Vec) -> (SecretKey, NodeAddr) { + let relay_map = default_relay_map(); let peer = iroh_net::NodeAddr::from_parts( node_id, relay_map.nodes().next().map(|n| n.url.clone()), addrs, ); - iroh::dial::Options { - secret_key: SecretKey::generate(), - peer, - keylog: false, - relay_map: Some(relay_map), - } + (SecretKey::generate(), peer) } #[tokio::test(flavor = "multi_thread")] @@ -153,12 +157,12 @@ async fn multiple_clients() -> Result<()> { tasks.push(node.local_pool_handle().spawn_pinned(move || { async move { - let opts = get_options(peer_id, addrs); + let (secret_key, peer) = get_options(peer_id, addrs); let expected_data = &content; let expected_name = name; let request = GetRequest::all(hash); let (collection, children, _stats) = - run_collection_get_request(opts, request).await?; + run_collection_get_request(secret_key, peer, request).await?; assert_eq!(expected_name, &collection[0].0); assert_eq!(&file_hash, &collection[0].1); assert_eq!(expected_data, &children[&0]); @@ -232,9 +236,10 @@ where .await?; let addrs = node.local_endpoint_addresses().await?; - let opts = get_options(node.node_id(), addrs); + let (secret_key, peer) = get_options(node.node_id(), addrs); let request = GetRequest::all(collection_hash); - let (collection, children, _stats) = run_collection_get_request(opts, request).await?; + let (collection, children, _stats) = + run_collection_get_request(secret_key, peer, request).await?; assert_eq!(num_blobs, collection.len()); for (i, (expected_name, expected_hash)) in expects.iter().enumerate() { let (name, hash) = &collection[i]; @@ -327,9 +332,11 @@ async fn test_server_close() { }) .await .unwrap(); - let opts = get_options(peer_id, node_addr); + let (secret_key, peer) = get_options(peer_id, node_addr); let request = GetRequest::all(hash); - let (_collection, _children, _stats) = run_collection_get_request(opts, request).await.unwrap(); + let (_collection, _children, _stats) = run_collection_get_request(secret_key, peer, request) + .await + .unwrap(); // Unwrap the JoinHandle, then the result of the Provider tokio::time::timeout(Duration::from_secs(10), async move { @@ -387,9 +394,9 @@ async fn test_ipv6() { let addrs = node.local_endpoint_addresses().await.unwrap(); let peer_id = node.node_id(); tokio::time::timeout(Duration::from_secs(10), async move { - let opts = get_options(peer_id, addrs); + let (secret_key, peer) = get_options(peer_id, addrs); let request = GetRequest::all(hash); - run_collection_get_request(opts, request).await + run_collection_get_request(secret_key, peer, request).await }) .await .expect("timeout") @@ -415,9 +422,9 @@ async fn test_not_found() { let addrs = node.local_endpoint_addresses().await.unwrap(); let peer_id = node.node_id(); tokio::time::timeout(Duration::from_secs(10), async move { - let opts = get_options(peer_id, addrs); + let (secret_key, peer) = get_options(peer_id, addrs); let request = GetRequest::single(hash); - let res = run_collection_get_request(opts, request).await; + let res = run_collection_get_request(secret_key, peer, request).await; if let Err(cause) = res { if let Some(e) = cause.downcast_ref::() { if let DecodeError::NotFound = e { @@ -458,9 +465,9 @@ async fn test_chunk_not_found_1() { let addrs = node.local_endpoint_addresses().await.unwrap(); let peer_id = node.node_id(); tokio::time::timeout(Duration::from_secs(10), async move { - let opts = get_options(peer_id, addrs); + let (secret_key, peer) = get_options(peer_id, addrs); let request = GetRequest::single(hash); - let res = run_collection_get_request(opts, request).await; + let res = run_collection_get_request(secret_key, peer, request).await; if let Err(cause) = res { if let Some(e) = cause.downcast_ref::() { if let DecodeError::NotFound = e { @@ -489,16 +496,7 @@ async fn test_run_ticket() { let ticket = node.ticket(hash, BlobFormat::HashSeq).await.unwrap(); tokio::time::timeout(Duration::from_secs(10), async move { let request = GetRequest::all(hash); - run_collection_get_request( - Options { - secret_key: SecretKey::generate(), - peer: ticket.node_addr().clone(), - keylog: false, - relay_map: Some(iroh_net::defaults::default_relay_map()), - }, - request, - ) - .await + run_collection_get_request(SecretKey::generate(), ticket.node_addr().clone(), request).await }) .await .expect("timeout") @@ -518,10 +516,11 @@ fn validate_children(collection: Collection, children: BTreeMap) -> } async fn run_collection_get_request( - opts: iroh::dial::Options, + secret_key: SecretKey, + peer: NodeAddr, request: GetRequest, ) -> anyhow::Result<(Collection, BTreeMap, Stats)> { - let connection = iroh::dial::dial(opts).await?; + let connection = dial(secret_key, peer).await?; let initial = fsm::start(connection, request); let connected = initial.next().await?; let ConnectedNext::StartRoot(fsm_at_start_root) = connected.next().await? else { @@ -538,9 +537,10 @@ async fn test_run_fsm() { let addrs = node.local_endpoint_addresses().await.unwrap(); let peer_id = node.node_id(); tokio::time::timeout(Duration::from_secs(10), async move { - let opts = get_options(peer_id, addrs); + let (secret_key, peer) = get_options(peer_id, addrs); let request = GetRequest::all(hash); - let (collection, children, _) = run_collection_get_request(opts, request).await?; + let (collection, children, _) = + run_collection_get_request(secret_key, peer, request).await?; validate_children(collection, children)?; anyhow::Ok(()) }) @@ -587,7 +587,8 @@ async fn test_size_request_blob() { let peer_id = node.node_id(); tokio::time::timeout(Duration::from_secs(10), async move { let request = GetRequest::last_chunk(hash); - let connection = iroh::dial::dial(get_options(peer_id, addrs)).await?; + let (secret_key, peer) = get_options(peer_id, addrs); + let connection = dial(secret_key, peer).await?; let response = fsm::start(connection, request); let connected = response.next().await?; let ConnectedNext::StartRoot(start) = connected.next().await? else { @@ -623,8 +624,9 @@ async fn test_collection_stat() { hash, RangeSpecSeq::from_ranges_infinite([ChunkRanges::all(), ranges]), ); - let opts = get_options(peer_id, addrs); - let (_collection, items, _stats) = run_collection_get_request(opts, request).await?; + let (secret_key, peer) = get_options(peer_id, addrs); + let (_collection, items, _stats) = + run_collection_get_request(secret_key, peer, request).await?; // we should get the first <=1024 bytes and the last chunk of each child // so now we know the size and can guess the type by inspecting the header assert_eq!(items.len(), 2); diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 8e0ffcc817..d466dc1463 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -10,12 +10,14 @@ use bytes::Bytes; use futures_lite::Stream; use futures_util::{FutureExt, StreamExt, TryStreamExt}; use iroh::{ - client::{mem::Doc, Entry, LiveEvent}, + base::node_addr::AddrInfoOptions, + client::{ + docs::{Entry, LiveEvent, ShareMode}, + MemDoc, + }, + net::key::{PublicKey, SecretKey}, node::{Builder, Node}, - rpc_protocol::ShareMode, }; -use iroh_base::node_addr::AddrInfoOptions; -use iroh_net::key::{PublicKey, SecretKey}; use quic_rpc::transport::misc::DummyServerEndpoint; use rand::{CryptoRng, Rng, SeedableRng}; use tracing::{debug, error_span, info, Instrument}; @@ -952,14 +954,14 @@ async fn sync_big() -> Result<()> { } /// Get all entries of a document. -async fn get_all(doc: &Doc) -> anyhow::Result> { +async fn get_all(doc: &MemDoc) -> anyhow::Result> { let entries = doc.get_many(Query::all()).await?; let entries = entries.collect::>().await; entries.into_iter().collect() } /// Get all entries of a document with the blob content. -async fn get_all_with_content(doc: &Doc) -> anyhow::Result> { +async fn get_all_with_content(doc: &MemDoc) -> anyhow::Result> { let entries = doc.get_many(Query::all()).await?; let entries = entries.and_then(|entry| async { let content = entry.content_bytes(doc).await; @@ -971,7 +973,7 @@ async fn get_all_with_content(doc: &Doc) -> anyhow::Result> } async fn publish( - docs: &[Doc], + docs: &[MemDoc], expected: &mut Vec, n: usize, cb: impl Fn(usize, usize) -> (AuthorId, String, String), @@ -1030,7 +1032,7 @@ async fn wait_for_events( } async fn assert_all_docs( - docs: &[Doc], + docs: &[MemDoc], node_ids: &[PublicKey], expected: &Vec, label: &str, @@ -1143,12 +1145,12 @@ async fn sync_drop_doc() -> Result<()> { Ok(()) } -async fn assert_latest(doc: &Doc, key: &[u8], value: &[u8]) { +async fn assert_latest(doc: &MemDoc, key: &[u8], value: &[u8]) { let content = get_latest(doc, key).await.unwrap(); assert_eq!(content, value.to_vec()); } -async fn get_latest(doc: &Doc, key: &[u8]) -> anyhow::Result> { +async fn get_latest(doc: &MemDoc, key: &[u8]) -> anyhow::Result> { let query = Query::single_latest_per_key().key_exact(key); let entry = doc .get_many(query)