From 217ac06b2128af9721ed6780a6bb2f0092a46ace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BCdiger=20Klaehn?= Date: Mon, 19 Aug 2024 20:55:45 +0300 Subject: [PATCH] refactor(iroh-blobs)!: expand docs (#2638) ## Description Extend iroh-blobs docs a bit ## Breaking Changes - There are some fn name changes, but they are for unpublished fns. - transfer_collection has been removed. It was very lowlevel and used in none of the examples, so I doubt anybody was using it outside. ## Notes & open questions ## Change checklist - [ ] Self-review. - [ ] Documentation updates following the [style guide](https://rust-lang.github.io/rfcs/1574-more-api-documentation-conventions.html#appendix-a-full-conventions-text), if relevant. - [ ] Tests if relevant. - [ ] All breaking changes documented. --------- Co-authored-by: Friedel Ziegelmayer --- iroh-blobs/src/lib.rs | 25 ++++++++++++++++++++++++- iroh-blobs/src/provider.rs | 30 ++++++++++++++++++------------ iroh-gossip/src/lib.rs | 7 ++++++- iroh/src/client/blobs.rs | 13 +++++++++++++ iroh/src/client/blobs/batch.rs | 2 +- iroh/tests/sync.rs | 1 + 6 files changed, 63 insertions(+), 15 deletions(-) diff --git a/iroh-blobs/src/lib.rs b/iroh-blobs/src/lib.rs index ce2788d5a9..2bd1624fd2 100644 --- a/iroh-blobs/src/lib.rs +++ b/iroh-blobs/src/lib.rs @@ -1,5 +1,28 @@ //! Blobs layer for iroh. - +//! +//! The crate is designed to be used from the [iroh] crate, which provides a +//! [high level interface](https://docs.rs/iroh/latest/iroh/client/blobs/index.html), +//! but can also be used standalone. +//! +//! It implements a [protocol] for streaming content-addressed data transfer using +//! [BLAKE3] verified streaming. +//! +//! It also provides a [store] interface for storage of blobs and outboards, +//! as well as a [persistent](crate::store::fs) and a [memory](crate::store::mem) +//! store implementation. +//! +//! To implement a server, the [provider] module provides helpers for handling +//! connections and individual requests given a store. +//! +//! To perform get requests, the [get] module provides utilities to perform +//! requests and store the result in a store, as well as a low level state +//! machine for executing requests. +//! +//! The [downloader] module provides a component to download blobs from +//! multiple sources and store them in a store. +//! +//! [BLAKE3]: https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf +//! [iroh]: https://docs.rs/iroh #![deny(missing_docs, rustdoc::broken_intra_doc_links)] #![recursion_limit = "256"] diff --git a/iroh-blobs/src/provider.rs b/iroh-blobs/src/provider.rs index a6d6c6a386..4144004a1f 100644 --- a/iroh-blobs/src/provider.rs +++ b/iroh-blobs/src/provider.rs @@ -197,19 +197,21 @@ pub async fn read_request(mut reader: RecvStream) -> Result { Ok(request) } -/// Transfers the collection & blob data. +/// Transfers a blob or hash sequence to the client. /// -/// First, it transfers the collection data & its associated outboard encoding data. Then it sequentially transfers each individual blob data & its associated outboard -/// encoding data. +/// The difference to [`handle_get`] is that we already have a reader for the +/// root blob and outboard. /// -/// Will fail if there is an error writing to the getter or reading from -/// the database. +/// First, it transfers the root blob. Then, if needed, it sequentially +/// transfers each individual blob data. /// -/// If a blob from the collection cannot be found in the database, the transfer will gracefully -/// close the writer, and return with `Ok(SentStatus::NotFound)`. +/// The transfer fail if there is an error writing to the writer or reading from +/// the database. /// -/// If the transfer does _not_ end in error, the buffer will be empty and the writer is gracefully closed. -pub async fn transfer_collection( +/// If a blob from the hash sequence cannot be found in the database, the +/// transfer will return with [`SentStatus::NotFound`]. If the transfer completes +/// successfully, it will return with [`SentStatus::Sent`]. +pub(crate) async fn transfer_hash_seq( request: GetRequest, // Store from which to fetch blobs. db: &D, @@ -358,7 +360,9 @@ pub trait CustomEventSender: std::fmt::Debug + Sync + Send + 'static { fn try_send(&self, event: Event); } -/// A possibly disabled sender for events. +/// A sender for events related to blob transfers. +/// +/// The sender is disabled by default. #[derive(Debug, Clone, Default)] pub struct EventSender { inner: Option>, @@ -458,7 +462,9 @@ async fn handle_stream(db: D, reader: RecvStream, writer: ResponseWriter } } -/// Handle a single standard get request. +/// Handle a single get request. +/// +/// Requires the request, a database, and a writer. pub async fn handle_get( db: D, request: GetRequest, @@ -482,7 +488,7 @@ pub async fn handle_get( let mut stats = Box::::default(); let t0 = std::time::Instant::now(); // 5. Transfer data! - let res = transfer_collection( + let res = transfer_hash_seq( request, &db, &mut writer, diff --git a/iroh-gossip/src/lib.rs b/iroh-gossip/src/lib.rs index 9c6dd3f27e..d67a14f4a1 100644 --- a/iroh-gossip/src/lib.rs +++ b/iroh-gossip/src/lib.rs @@ -1,5 +1,10 @@ //! Broadcast messages to peers subscribed to a topic - +//! +//! The crate is designed to be used from the [iroh] crate, which provides a +//! [high level interface](https://docs.rs/iroh/latest/iroh/client/gossip/index.html), +//! but can also be used standalone. +//! +//! [iroh]: https://docs.rs/iroh #![deny(missing_docs, rustdoc::broken_intra_doc_links)] pub mod metrics; diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs index d30518a7ed..eb1f462769 100644 --- a/iroh/src/client/blobs.rs +++ b/iroh/src/client/blobs.rs @@ -46,6 +46,19 @@ //! - [`validate`](Client::validate) validates the locally stored data against //! their BLAKE3 hashes. //! - [`delete_blob`](Client::delete_blob) deletes a blob from the local store. +//! +//! ### Batch operations +//! +//! For complex update operations, there is a [`batch`](Client::batch) API that +//! allows you to add multiple blobs in a single logical batch. +//! +//! Operations in a batch return [temporary tags](crate::blobs::TempTag) that +//! protect the added data from garbage collection as long as the batch is +//! alive. +//! +//! To store the data permanently, a temp tag needs to be upgraded to a +//! permanent tag using [`persist`](crate::client::blobs::Batch::persist) or +//! [`persist_to`](crate::client::blobs::Batch::persist_to). use std::{ future::Future, io, diff --git a/iroh/src/client/blobs/batch.rs b/iroh/src/client/blobs/batch.rs index d6e70d090d..8633110e35 100644 --- a/iroh/src/client/blobs/batch.rs +++ b/iroh/src/client/blobs/batch.rs @@ -442,7 +442,7 @@ impl Batch { /// Upgrades a temp tag to a persistent tag with either a specific name or /// an automatically generated name. - pub async fn upgrade_with_opts(&self, tt: TempTag, opts: SetTagOption) -> Result { + pub async fn persist_with_opts(&self, tt: TempTag, opts: SetTagOption) -> Result { match opts { SetTagOption::Auto => self.persist(tt).await, SetTagOption::Named(tag) => { diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 026d14c6fc..67638c6869 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -576,6 +576,7 @@ async fn test_sync_via_relay() -> Result<()> { #[tokio::test] #[cfg(feature = "test-utils")] +#[ignore = "flaky"] async fn sync_restart_node() -> Result<()> { let mut rng = test_rng(b"sync_restart_node"); setup_logging();