From 690c20e6be6b5413da673a5ad473458907f8ed0c Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 21 Mar 2024 17:13:11 +0200 Subject: [PATCH] Rename blobs db from file::Store to fs::Store for consistency with the docs store Also rename the feature flag to be consistent. --- iroh-bytes/Cargo.toml | 4 ++-- iroh-bytes/src/store.rs | 6 +++--- iroh-bytes/src/store/bao_file.rs | 5 +++-- iroh-bytes/src/store/{file.rs => fs.rs} | 2 +- iroh-bytes/src/store/{file => fs}/import_flat_store.rs | 2 +- iroh-bytes/src/store/{file => fs}/tables.rs | 0 iroh-bytes/src/store/{file => fs}/test_support.rs | 0 iroh-bytes/src/store/{file => fs}/tests.rs | 0 iroh-bytes/src/store/{file => fs}/util.rs | 6 +++++- iroh-bytes/src/store/{file => fs}/validate.rs | 2 +- iroh-cli/src/commands/doctor.rs | 2 +- iroh-cli/src/commands/start.rs | 2 +- iroh-cli/tests/cli.rs | 8 ++++---- iroh/Cargo.toml | 2 +- iroh/src/node.rs | 6 +++--- iroh/src/node/builder.rs | 10 +++++----- iroh/src/util/path.rs | 2 +- iroh/tests/gc.rs | 10 +++++----- 18 files changed, 37 insertions(+), 32 deletions(-) rename iroh-bytes/src/store/{file.rs => fs.rs} (99%) rename iroh-bytes/src/store/{file => fs}/import_flat_store.rs (99%) rename iroh-bytes/src/store/{file => fs}/tables.rs (100%) rename iroh-bytes/src/store/{file => fs}/test_support.rs (100%) rename iroh-bytes/src/store/{file => fs}/tests.rs (100%) rename iroh-bytes/src/store/{file => fs}/util.rs (97%) rename iroh-bytes/src/store/{file => fs}/validate.rs (99%) diff --git a/iroh-bytes/Cargo.toml b/iroh-bytes/Cargo.toml index 2cbc6fce24..5e93aa9d97 100644 --- a/iroh-bytes/Cargo.toml +++ b/iroh-bytes/Cargo.toml @@ -62,8 +62,8 @@ rustls = { version = "0.21.0", default-features = false, features = ["quic"] } tempfile = "3.10.0" [features] -default = ["file-db"] -file-db = ["reflink-copy", "redb"] +default = ["fs-store"] +fs-store = ["reflink-copy", "redb"] downloader = ["iroh-net", "parking_lot", "tokio-util/time"] metrics = ["iroh-metrics"] diff --git a/iroh-bytes/src/store.rs b/iroh-bytes/src/store.rs index f42204bf29..0e8f35d301 100644 --- a/iroh-bytes/src/store.rs +++ b/iroh-bytes/src/store.rs @@ -1,14 +1,14 @@ //! Implementations of blob stores use crate::{BlobFormat, Hash, HashAndFormat}; -#[cfg(feature = "file-db")] +#[cfg(feature = "fs-store")] mod bao_file; pub mod mem; mod mutable_mem_storage; pub mod readonly_mem; -#[cfg(feature = "file-db")] -pub mod file; +#[cfg(feature = "fs-store")] +pub mod fs; mod traits; pub use traits::*; diff --git a/iroh-bytes/src/store/bao_file.rs b/iroh-bytes/src/store/bao_file.rs index 5a2bccddee..259f5d3750 100644 --- a/iroh-bytes/src/store/bao_file.rs +++ b/iroh-bytes/src/store/bao_file.rs @@ -134,6 +134,7 @@ fn create_read_write(path: impl AsRef) -> io::Result { .read(true) .write(true) .create(true) + .truncate(false) .open(path) } @@ -276,7 +277,7 @@ impl BaoFileStorage { /// Take the storage out, leaving an empty storage in its place. /// /// Be careful to put somethign back in its place, or you will lose data. - #[cfg(feature = "file-db")] + #[cfg(feature = "fs-store")] pub fn take(&mut self) -> Self { std::mem::take(self) } @@ -524,7 +525,7 @@ impl BaoFileHandle { /// Transform the storage in place. If the transform fails, the storage will /// be an immutable empty storage. - #[cfg(feature = "file-db")] + #[cfg(feature = "fs-store")] pub(crate) fn transform( &self, f: impl FnOnce(BaoFileStorage) -> io::Result, diff --git a/iroh-bytes/src/store/file.rs b/iroh-bytes/src/store/fs.rs similarity index 99% rename from iroh-bytes/src/store/file.rs rename to iroh-bytes/src/store/fs.rs index 87d4d3b00c..de636e645c 100644 --- a/iroh-bytes/src/store/file.rs +++ b/iroh-bytes/src/store/fs.rs @@ -100,7 +100,7 @@ mod validate; use crate::{ store::{ bao_file::{BaoFileStorage, CompleteStorage}, - file::{ + fs::{ tables::BaoFilePart, util::{overwrite_and_sync, read_and_remove, ProgressReader}, }, diff --git a/iroh-bytes/src/store/file/import_flat_store.rs b/iroh-bytes/src/store/fs/import_flat_store.rs similarity index 99% rename from iroh-bytes/src/store/file/import_flat_store.rs rename to iroh-bytes/src/store/fs/import_flat_store.rs index 66b0252c5b..601b417307 100644 --- a/iroh-bytes/src/store/file/import_flat_store.rs +++ b/iroh-bytes/src/store/fs/import_flat_store.rs @@ -9,7 +9,7 @@ use std::{ }; use crate::{ - store::file::{tables::Tables, DataLocation, EntryState, OutboardLocation}, + store::fs::{tables::Tables, DataLocation, EntryState, OutboardLocation}, util::{raw_outboard_size, Tag}, IROH_BLOCK_SIZE, }; diff --git a/iroh-bytes/src/store/file/tables.rs b/iroh-bytes/src/store/fs/tables.rs similarity index 100% rename from iroh-bytes/src/store/file/tables.rs rename to iroh-bytes/src/store/fs/tables.rs diff --git a/iroh-bytes/src/store/file/test_support.rs b/iroh-bytes/src/store/fs/test_support.rs similarity index 100% rename from iroh-bytes/src/store/file/test_support.rs rename to iroh-bytes/src/store/fs/test_support.rs diff --git a/iroh-bytes/src/store/file/tests.rs b/iroh-bytes/src/store/fs/tests.rs similarity index 100% rename from iroh-bytes/src/store/file/tests.rs rename to iroh-bytes/src/store/fs/tests.rs diff --git a/iroh-bytes/src/store/file/util.rs b/iroh-bytes/src/store/fs/util.rs similarity index 97% rename from iroh-bytes/src/store/file/util.rs rename to iroh-bytes/src/store/fs/util.rs index 4d6fd8ca2b..39dcd43ce9 100644 --- a/iroh-bytes/src/store/file/util.rs +++ b/iroh-bytes/src/store/fs/util.rs @@ -49,7 +49,11 @@ pub fn overwrite_and_sync(path: &Path, data: &[u8]) -> io::Result // std::fs::create_dir_all(path.parent().unwrap()).unwrap(); // tracing::error!("{}", path.parent().unwrap().display()); // tracing::error!("{}", path.parent().unwrap().metadata().unwrap().is_dir()); - let mut file = OpenOptions::new().write(true).create(true).open(path)?; + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(path)?; file.write_all(data)?; // todo: figure out if it is safe to not sync here file.sync_all()?; diff --git a/iroh-bytes/src/store/file/validate.rs b/iroh-bytes/src/store/fs/validate.rs similarity index 99% rename from iroh-bytes/src/store/file/validate.rs rename to iroh-bytes/src/store/fs/validate.rs index 1d6df236d1..1d208ae613 100644 --- a/iroh-bytes/src/store/file/validate.rs +++ b/iroh-bytes/src/store/fs/validate.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use redb::ReadableTable; -use crate::store::{file::tables::BaoFilePart, ValidateLevel, ValidateProgress}; +use crate::store::{fs::tables::BaoFilePart, ValidateLevel, ValidateProgress}; use super::{ raw_outboard_size, tables::Tables, ActorResult, ActorState, DataLocation, EntryState, Hash, diff --git a/iroh-cli/src/commands/doctor.rs b/iroh-cli/src/commands/doctor.rs index 1c6f23d37c..a3d861b93d 100644 --- a/iroh-cli/src/commands/doctor.rs +++ b/iroh-cli/src/commands/doctor.rs @@ -974,7 +974,7 @@ pub async fn run(command: Commands, config: &NodeConfig) -> anyhow::Result<()> { } Commands::TicketInspect { ticket } => inspect_ticket(&ticket), Commands::ValidateBlobStore { path, repair } => { - let blob_store = iroh::bytes::store::file::Store::load(path).await?; + let blob_store = iroh::bytes::store::fs::Store::load(path).await?; let (send, mut recv) = sync::mpsc::channel(1); let task = tokio::spawn(async move { while let Some(msg) = recv.recv().await { diff --git a/iroh-cli/src/commands/start.rs b/iroh-cli/src/commands/start.rs index 9dc6fea59f..1c6bc15191 100644 --- a/iroh-cli/src/commands/start.rs +++ b/iroh-cli/src/commands/start.rs @@ -124,7 +124,7 @@ where pub(crate) async fn start_node( iroh_data_root: &Path, derp_map: Option, -) -> Result> { +) -> Result> { let rpc_status = RpcStatus::load(iroh_data_root).await?; match rpc_status { RpcStatus::Running { port, .. } => { diff --git a/iroh-cli/tests/cli.rs b/iroh-cli/tests/cli.rs index df804c6013..512a57aadc 100644 --- a/iroh-cli/tests/cli.rs +++ b/iroh-cli/tests/cli.rs @@ -107,7 +107,7 @@ fn cli_provide_tree() -> Result<()> { test_provide_get_loop(Input::Path(path), Output::Path) } -#[cfg(feature = "file-db")] +#[cfg(feature = "fs-store")] #[test] #[ignore = "flaky"] fn cli_provide_tree_resume() -> Result<()> { @@ -213,7 +213,7 @@ fn cli_provide_tree_resume() -> Result<()> { Ok(()) } -#[cfg(feature = "file-db")] +#[cfg(feature = "fs-store")] #[test] #[ignore = "flaky"] fn cli_provide_file_resume() -> Result<()> { @@ -439,14 +439,14 @@ async fn cli_provide_persistence() -> anyhow::Result<()> { provide(&foo_path)?; // should have some data now let db_path = IrohPaths::BaoStoreDir.with_root(&iroh_data_dir); - let db = iroh::bytes::store::file::Store::load(&db_path).await?; + let db = iroh::bytes::store::fs::Store::load(&db_path).await?; let blobs: Vec> = db.blobs().await.unwrap().collect::>(); drop(db); assert_eq!(blobs.len(), 3); provide(&bar_path)?; // should have more data now - let db = iroh::bytes::store::file::Store::load(&db_path).await?; + let db = iroh::bytes::store::fs::Store::load(&db_path).await?; let blobs = db.blobs().await.unwrap().collect::>(); drop(db); assert_eq!(blobs.len(), 6); diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 7a4a95f944..29db36e8d2 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -59,7 +59,7 @@ indicatif = { version = "0.17", features = ["tokio"], optional = true } [features] default = ["metrics"] metrics = ["iroh-metrics", "iroh-bytes/metrics"] -file-db = ["iroh-bytes/file-db"] +fs-store = ["iroh-bytes/fs-store"] test = [] examples = ["dep:clap", "dep:indicatif"] diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 5f5cc0b01a..93723872e8 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -124,7 +124,7 @@ pub enum Event { pub type MemNode = Node; /// Persistent node. -pub type FsNode = Node; +pub type FsNode = Node; impl MemNode { /// Returns a new builder for the [`Node`], by default configured to run in memory. @@ -142,7 +142,7 @@ impl FsNode { /// Once done with the builder call [`Builder::spawn`] to create the node. pub async fn persistent( root: impl AsRef, - ) -> Result> { + ) -> Result> { Builder::default().persist(root).await } } @@ -278,7 +278,7 @@ impl NodeInner { } } -#[cfg(all(test, feature = "file-db"))] +#[cfg(all(test, feature = "fs-store"))] mod tests { use std::path::Path; use std::time::Duration; diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index c27eabe5b0..ed6aa9e769 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -134,25 +134,25 @@ where pub async fn persist( self, root: impl AsRef, - ) -> Result> { + ) -> Result> { let root = root.as_ref(); let blob_dir = IrohPaths::BaoStoreDir.with_root(root); tokio::fs::create_dir_all(&blob_dir).await?; - let blobs_store = iroh_bytes::store::file::Store::load(&blob_dir) + let blobs_store = iroh_bytes::store::fs::Store::load(&blob_dir) .await .with_context(|| format!("Failed to load iroh database from {}", blob_dir.display()))?; let docs_store = iroh_sync::store::fs::Store::new(IrohPaths::DocsDatabase.with_root(root))?; let v0 = blobs_store - .import_flat_store(iroh_bytes::store::file::FlatStorePaths { + .import_flat_store(iroh_bytes::store::fs::FlatStorePaths { complete: root.join("blobs.v0"), partial: root.join("blobs-partial.v0"), meta: root.join("blobs-meta.v0"), }) .await?; let v1 = blobs_store - .import_flat_store(iroh_bytes::store::file::FlatStorePaths { + .import_flat_store(iroh_bytes::store::fs::FlatStorePaths { complete: root.join("blobs.v1").join("complete"), partial: root.join("blobs.v1").join("partial"), meta: root.join("blobs.v1").join("meta"), @@ -161,7 +161,7 @@ where if v0 || v1 { tracing::info!("flat data was imported - reapply inline options"); blobs_store - .update_inline_options(iroh_bytes::store::file::InlineOptions::default(), true) + .update_inline_options(iroh_bytes::store::fs::InlineOptions::default(), true) .await?; } diff --git a/iroh/src/util/path.rs b/iroh/src/util/path.rs index 058379a33c..b9d7093f4a 100644 --- a/iroh/src/util/path.rs +++ b/iroh/src/util/path.rs @@ -9,7 +9,7 @@ pub enum IrohPaths { /// Path to the node's secret key for the [`iroh_net::key::PublicKey`]. #[strum(serialize = "keypair")] SecretKey, - /// Path to the node's [file based blob store](iroh_bytes::store::file::Store). + /// Path to the node's [file based blob store](iroh_bytes::store::fs::Store). #[strum(serialize = "blobs")] BaoStoreDir, /// Path to the [iroh-sync document database](iroh_sync::store::fs::Store) diff --git a/iroh/tests/gc.rs b/iroh/tests/gc.rs index e70a6d4fce..28998a3c6e 100644 --- a/iroh/tests/gc.rs +++ b/iroh/tests/gc.rs @@ -185,7 +185,7 @@ async fn gc_hashseq_impl() -> Result<()> { Ok(()) } -#[cfg(feature = "file-db")] +#[cfg(feature = "fs-store")] mod file { use super::*; use std::{io, path::PathBuf, time::Duration}; @@ -242,7 +242,7 @@ mod file { async fn redb_doc_import_stress() -> Result<()> { let _ = tracing_subscriber::fmt::try_init(); let dir = testdir!(); - let bao_store = iroh_bytes::store::file::Store::load(dir.join("store")).await?; + let bao_store = iroh_bytes::store::fs::Store::load(dir.join("store")).await?; let node = wrap_in_node(bao_store.clone(), Duration::from_secs(10)).await; let client = node.client(); let doc = client.docs.create().await?; @@ -285,7 +285,7 @@ mod file { let path = data_path(dir.clone()); let outboard_path = outboard_path(dir.clone()); - let bao_store = iroh_bytes::store::file::Store::load(dir.clone()).await?; + let bao_store = iroh_bytes::store::fs::Store::load(dir.clone()).await?; let node = wrap_in_node(bao_store.clone(), Duration::from_millis(100)).await; let evs = attach_db_events(&node).await; let data1 = create_test_data(10000000); @@ -448,7 +448,7 @@ mod file { let path = data_path(dir.clone()); let outboard_path = outboard_path(dir.clone()); - let bao_store = iroh_bytes::store::file::Store::load(dir.clone()).await?; + let bao_store = iroh_bytes::store::fs::Store::load(dir.clone()).await?; let node = wrap_in_node(bao_store.clone(), Duration::from_millis(10)).await; let evs = attach_db_events(&node).await; @@ -482,7 +482,7 @@ mod file { let _ = tracing_subscriber::fmt::try_init(); let dir = testdir!(); - let bao_store = iroh_bytes::store::file::Store::load(dir.clone()).await?; + let bao_store = iroh_bytes::store::fs::Store::load(dir.clone()).await?; let node = wrap_in_node(bao_store.clone(), Duration::from_secs(1)).await; let evs = attach_db_events(&node).await;