diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f864f0f..33d6bea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: - name: Install cargo-hack run: cargo install cargo-hack - name: Apply clippy lints - run: cargo hack clippy --each-feature --exclude-no-default-features + run: cargo hack clippy --each-feature --exclude-no-default-features --exclude-features tracing # Run tests on some extra platforms cross: @@ -141,13 +141,15 @@ jobs: - name: Install Rust # --no-self-update is necessary because the windows environment cannot self-update rustup.exe. run: rustup update stable --no-self-update && rustup default stable + - name: Install cargo-hack + run: cargo install cargo-hack - name: Cache ~/.cargo uses: actions/cache@v3 with: path: ~/.cargo key: ${{ runner.os }}-coverage-dotcargo - name: Run build - run: cargo build --all-features + run: cargo hack build --feature-powerset --exclude-no-default-features --group-features xxhash64,xxhash3 --exclude-features tracing test: name: test diff --git a/Cargo.toml b/Cargo.toml index 0485a24..f75d5c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "orderwal" -version = "0.3.1" +version = "0.3.2" edition = "2021" repository = "https://github.com/al8n/orderwal" homepage = "https://github.com/al8n/orderwal" @@ -23,11 +23,13 @@ std = ["rarena-allocator/default", "crossbeam-skiplist/default", "bitflags/defau xxhash3 = ["dbutils/xxhash3", "std"] xxhash64 = ["dbutils/xxhash64", "std"] +tracing = ["dep:tracing", "dbutils/tracing"] + [dependencies] among = { version = "0.1", default-features = false, features = ["either"] } bitflags = { version = "1", default-features = false } dbutils = { version = "0.4", default-features = false, features = ["crc32fast"] } -rarena-allocator = { version = "0.3", default-features = false, features = ["memmap"] } +rarena-allocator = { version = "0.4", default-features = false, features = ["memmap"] } crossbeam-skiplist = { version = "0.1", default-features = false, package = "crossbeam-skiplist-pr1132" } paste = "1" thiserror = "1" @@ -38,7 +40,7 @@ smallvec-wrapper = { version = "0.1", optional = true, default-features = false, smol_str = { version = "0.3", default-features = false, optional = true } faststr = { version = "0.2", default-features = false, optional = true } -tracing = { version = "0.1", optional = true } +tracing = { version = "0.1", default-features = false, optional = true } [dev-dependencies] arbitrary = { version = "1", features = ["derive"] } diff --git a/src/builder.rs b/src/builder.rs index 5dc3f51..38206f4 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -126,6 +126,50 @@ impl Builder { self.opts.reserved() } + /// Set if lock the meta of the WAL in the memory to prevent OS from swapping out the header of WAL. + /// When using memory map backed WAL, the meta of the WAL + /// is in the header, meta is frequently accessed, + /// lock (`mlock` on the header) the meta can reduce the page fault, + /// but yes, this means that one WAL will have one page are locked in memory, + /// and will not be swapped out. So, this is a trade-off between performance and memory usage. + /// + /// Default is `true`. + /// + /// This configuration has no effect on windows and vec backed WAL. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::Builder; + /// + /// let opts = Builder::new().with_lock_meta(false); + /// ``` + #[inline] + pub const fn with_lock_meta(mut self, lock_meta: bool) -> Self { + self.opts.lock_meta = lock_meta; + self + } + + /// Get if lock the meta of the WAL in the memory to prevent OS from swapping out the header of WAL. + /// When using memory map backed WAL, the meta of the WAL + /// is in the header, meta is frequently accessed, + /// lock (`mlock` on the header) the meta can reduce the page fault, + /// but yes, this means that one WAL will have one page are locked in memory, + /// and will not be swapped out. So, this is a trade-off between performance and memory usage. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::Builder; + /// + /// let opts = Builder::new().with_lock_meta(false); + /// assert_eq!(opts.lock_meta(), false); + /// ``` + #[inline] + pub const fn lock_meta(&self) -> bool { + self.opts.lock_meta + } + /// Returns the magic version. /// /// The default value is `0`. diff --git a/src/lib.rs b/src/lib.rs index 24eaf76..f5360cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,8 +22,7 @@ extern crate std; pub use dbutils::{ checksum::{self, Crc32}, - traits::{Ascend, Descend}, - CheapClone, Comparator, + Ascend, CheapClone, Comparator, Descend, }; #[cfg(feature = "xxhash3")] diff --git a/src/options.rs b/src/options.rs index 80d366e..97ee7c6 100644 --- a/src/options.rs +++ b/src/options.rs @@ -8,6 +8,7 @@ pub struct Options { cap: Option, reserved: u32, + pub(crate) lock_meta: bool, pub(crate) read: bool, pub(crate) write: bool, pub(crate) create_new: bool, @@ -50,6 +51,7 @@ impl Options { huge: None, cap: None, reserved: 0, + lock_meta: false, read: false, write: false, create_new: false, @@ -102,6 +104,50 @@ impl Options { self.reserved } + /// Set if lock the meta of the WAL in the memory to prevent OS from swapping out the header of WAL. + /// When using memory map backed WAL, the meta of the WAL + /// is in the header, meta is frequently accessed, + /// lock (`mlock` on the header) the meta can reduce the page fault, + /// but yes, this means that one WAL will have one page are locked in memory, + /// and will not be swapped out. So, this is a trade-off between performance and memory usage. + /// + /// Default is `true`. + /// + /// This configuration has no effect on windows and vec backed WAL. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::Options; + /// + /// let opts = Options::new().with_lock_meta(false); + /// ``` + #[inline] + pub const fn with_lock_meta(mut self, lock_meta: bool) -> Self { + self.lock_meta = lock_meta; + self + } + + /// Get if lock the meta of the WAL in the memory to prevent OS from swapping out the header of WAL. + /// When using memory map backed WAL, the meta of the WAL + /// is in the header, meta is frequently accessed, + /// lock (`mlock` on the header) the meta can reduce the page fault, + /// but yes, this means that one WAL will have one page are locked in memory, + /// and will not be swapped out. So, this is a trade-off between performance and memory usage. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::Options; + /// + /// let opts = Options::new().with_lock_meta(false); + /// assert_eq!(opts.lock_meta(), false); + /// ``` + #[inline] + pub const fn lock_meta(&self) -> bool { + self.lock_meta + } + /// Returns the magic version. /// /// The default value is `0`. @@ -664,7 +710,8 @@ impl ArenaOptionsExt for super::ArenaOptions { .with_append(opts.append()) .with_stack(opts.stack()) .with_populate(opts.populate()) - .with_huge(opts.huge()); + .with_huge(opts.huge()) + .with_lock_meta(opts.lock_meta()); if let Some(cap) = opts.cap { new.with_capacity(cap) diff --git a/src/swmr/generic/tests/constructor.rs b/src/swmr/generic/tests/constructor.rs index 617cf7d..e6946a5 100644 --- a/src/swmr/generic/tests/constructor.rs +++ b/src/swmr/generic/tests/constructor.rs @@ -230,6 +230,7 @@ fn construct_with_small_capacity_map_anon() { } #[test] +#[cfg_attr(miri, ignore)] fn construct_with_small_capacity_map_file() { let dir = tempdir().unwrap(); let path = dir diff --git a/src/wal/sealed.rs b/src/wal/sealed.rs index 1aaa8db..c67b330 100644 --- a/src/wal/sealed.rs +++ b/src/wal/sealed.rs @@ -352,9 +352,10 @@ pub trait Sealed: Constructor { if self.options().sync_on_write() && is_ondisk { allocator - .flush_range(buf.offset(), elen as usize) + .flush_header_and_range(buf.offset(), elen as usize) .map_err(|e| Among::Right(e.into()))?; } + buf.detach(); let cmp = self.comparator().cheap_clone(); let ptr = buf.as_ptr().add(ko); @@ -395,7 +396,7 @@ trait SealedExt: Sealed { let buf_cap = buf.capacity(); if self.options().sync_on_write() && allocator.is_ondisk() { - allocator.flush_range(buf.offset(), buf_cap)?; + allocator.flush_header_and_range(buf.offset(), buf_cap)?; } buf.detach(); Ok(())