diff --git a/examples/zero_cost.rs b/examples/zero_cost.rs index 36de356..4536578 100644 --- a/examples/zero_cost.rs +++ b/examples/zero_cost.rs @@ -1,6 +1,10 @@ use std::{cmp, sync::Arc, thread::spawn}; -use orderwal::{swmr::generic::*, utils::*, OpenOptions, Options}; +use orderwal::{ + swmr::generic::{Comparable, Equivalent, GenericBuilder, KeyRef, Type, TypeRef}, + utils::*, + OpenOptions, +}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] struct Person { @@ -132,15 +136,15 @@ fn main() { .collect::>(); let mut wal = unsafe { - GenericOrderWal::::map_mut( - &path, - Options::new(), - OpenOptions::new() - .create_new(Some(1024 * 1024)) - .write(true) - .read(true), - ) - .unwrap() + GenericBuilder::new() + .map_mut::( + &path, + OpenOptions::new() + .create_new(Some(1024 * 1024)) + .write(true) + .read(true), + ) + .unwrap() }; // Create 100 readers diff --git a/src/wal/builder.rs b/src/builder.rs similarity index 92% rename from src/wal/builder.rs rename to src/builder.rs index 0eb2d1a..0426644 100644 --- a/src/wal/builder.rs +++ b/src/builder.rs @@ -1,3 +1,6 @@ +use checksum::BuildChecksumer; +use wal::{sealed::Constructor, Wal}; + use super::*; /// A write-ahead log builder. @@ -29,7 +32,7 @@ impl Builder { impl Builder { /// Returns a new write-ahead log builder with the new comparator /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::{Builder, Ascend}; @@ -47,7 +50,7 @@ impl Builder { /// Returns a new write-ahead log builder with the new checksumer /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::{Builder, Crc32}; @@ -65,7 +68,7 @@ impl Builder { /// Returns a new write-ahead log builder with the new options /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::{Builder, Options}; @@ -88,7 +91,7 @@ impl Builder { /// /// The default reserved is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -108,7 +111,7 @@ impl Builder { /// /// The default reserved is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -126,7 +129,7 @@ impl Builder { /// /// The default value is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -143,7 +146,7 @@ impl Builder { /// /// The default value is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -160,7 +163,7 @@ impl Builder { /// /// The default value is `u16::MAX`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -177,7 +180,7 @@ impl Builder { /// /// The default value is `u32::MAX`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -194,7 +197,7 @@ impl Builder { /// /// The default value is `true`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -221,7 +224,7 @@ impl Builder { /// /// The default value is `None`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -240,7 +243,7 @@ impl Builder { /// /// The default value is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -256,7 +259,7 @@ impl Builder { /// Sets the maximum key length. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -272,7 +275,7 @@ impl Builder { /// Sets the maximum value length. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -300,7 +303,7 @@ impl Builder { /// /// The default value is `None`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -318,7 +321,7 @@ impl Builder { /// /// The default value is `true`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -336,7 +339,7 @@ impl Builder { /// /// The default value is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Builder; @@ -354,7 +357,7 @@ impl Builder { impl Builder { /// Creates a new in-memory write-ahead log backed by an aligned vec. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::{swmr::OrderWal, Builder}; @@ -373,12 +376,12 @@ impl Builder { arena_options(opts.reserved()).with_capacity(opts.capacity()), ) .map_err(Error::from_insufficient_space)?; - >::new_in(arena, opts, cmp, cks).map(W::from_core) + >::new_in(arena, opts, cmp, cks).map(W::from_core) } /// Creates a new in-memory write-ahead log but backed by an anonymous mmap. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::{swmr::OrderWal, Builder}; @@ -396,9 +399,7 @@ impl Builder { let mmap_opts = MmapOptions::new().len(opts.capacity()); ::map_anon(arena_options(opts.reserved()), mmap_opts) .map_err(Into::into) - .and_then(|arena| { - >::new_in(arena, opts, cmp, cks).map(W::from_core) - }) + .and_then(|arena| >::new_in(arena, opts, cmp, cks).map(W::from_core)) } /// Opens a write-ahead log backed by a file backed memory map in read-only mode. @@ -433,7 +434,7 @@ impl Builder { /// }; pub unsafe fn map(self, path: P) -> Result where - C: Comparator + CheapClone, + C: Comparator + CheapClone + 'static, S: BuildChecksumer, P: AsRef, W: Wal, @@ -479,16 +480,16 @@ impl Builder { ) -> Result> where PB: FnOnce() -> Result, - C: Comparator + CheapClone, + C: Comparator + CheapClone + 'static, S: BuildChecksumer, W: Wal, - W::Pointer: Ord, + W::Pointer: Ord + 'static, { let open_options = OpenOptions::default().read(true); let Self { opts, cmp, cks } = self; - <>::Allocator as Allocator>::map_with_path_builder( + <>::Allocator as Allocator>::map_with_path_builder( path_builder, arena_options(opts.reserved()), open_options, @@ -496,8 +497,8 @@ impl Builder { ) .map_err(|e| e.map_right(Into::into)) .and_then(|arena| { - >::replay(arena, Options::new(), true, cmp, cks) - .map(>::from_core) + >::replay(arena, Options::new(), true, cmp, cks) + .map(>::from_core) .map_err(Either::Right) }) } @@ -528,7 +529,7 @@ impl Builder { /// ``` pub unsafe fn map_mut(self, path: P, open_opts: OpenOptions) -> Result where - C: Comparator + CheapClone, + C: Comparator + CheapClone + 'static, S: BuildChecksumer, P: AsRef, W: Wal, @@ -571,7 +572,7 @@ impl Builder { ) -> Result> where PB: FnOnce() -> Result, - C: Comparator + CheapClone, + C: Comparator + CheapClone + 'static, S: BuildChecksumer, W: Wal, { @@ -590,9 +591,9 @@ impl Builder { .map_err(Into::into) .and_then(|arena| { if !exist { - >::new_in(arena, opts, cmp, cks).map(W::from_core) + >::new_in(arena, opts, cmp, cks).map(W::from_core) } else { - >::replay(arena, opts, false, cmp, cks).map(W::from_core) + >::replay(arena, opts, false, cmp, cks).map(W::from_core) } }) .map_err(Either::Right) diff --git a/src/entry.rs b/src/entry.rs new file mode 100644 index 0000000..d73277b --- /dev/null +++ b/src/entry.rs @@ -0,0 +1,407 @@ +use among::Among; +use crossbeam_skiplist::set::Entry as SetEntry; +use rarena_allocator::either::Either; + +use super::{ + pointer::{GenericPointer, Pointer}, + wal::r#type::{Type, TypeRef}, + KeyBuilder, ValueBuilder, +}; + +/// An entry in the write-ahead log. +pub struct Entry { + pub(crate) key: K, + pub(crate) value: V, + pub(crate) pointer: Option>, +} + +impl Entry { + /// Creates a new entry. + #[inline] + pub const fn new(key: K, value: V) -> Self { + Self { + key, + value, + pointer: None, + } + } + + /// Returns the key. + #[inline] + pub const fn key(&self) -> &K { + &self.key + } + + /// Returns the value. + #[inline] + pub const fn value(&self) -> &V { + &self.value + } + + /// Consumes the entry and returns the key and value. + #[inline] + pub fn into_components(self) -> (K, V) { + (self.key, self.value) + } +} + +/// An entry in the write-ahead log. +pub struct EntryWithKeyBuilder { + pub(crate) kb: KeyBuilder, + pub(crate) value: V, + pub(crate) pointer: Option>, +} + +impl EntryWithKeyBuilder { + /// Creates a new entry. + #[inline] + pub const fn new(kb: KeyBuilder, value: V) -> Self { + Self { + kb, + value, + pointer: None, + } + } + + /// Returns the key. + #[inline] + pub const fn key_builder(&self) -> &KeyBuilder { + &self.kb + } + + /// Returns the value. + #[inline] + pub const fn value(&self) -> &V { + &self.value + } + + /// Consumes the entry and returns the key and value. + #[inline] + pub fn into_components(self) -> (KeyBuilder, V) { + (self.kb, self.value) + } +} + +/// An entry in the write-ahead log. +pub struct EntryWithValueBuilder { + pub(crate) key: K, + pub(crate) vb: ValueBuilder, + pub(crate) pointer: Option>, +} + +impl EntryWithValueBuilder { + /// Creates a new entry. + #[inline] + pub const fn new(key: K, vb: ValueBuilder) -> Self { + Self { + key, + vb, + pointer: None, + } + } + + /// Returns the key. + #[inline] + pub const fn value_builder(&self) -> &ValueBuilder { + &self.vb + } + + /// Returns the value. + #[inline] + pub const fn key(&self) -> &K { + &self.key + } + + /// Consumes the entry and returns the key and value. + #[inline] + pub fn into_components(self) -> (K, ValueBuilder) { + (self.key, self.vb) + } +} + +/// An entry in the write-ahead log. +pub struct EntryWithBuilders { + pub(crate) kb: KeyBuilder, + pub(crate) vb: ValueBuilder, + pub(crate) pointer: Option>, +} + +impl EntryWithBuilders { + /// Creates a new entry. + #[inline] + pub const fn new(kb: KeyBuilder, vb: ValueBuilder) -> Self { + Self { + kb, + vb, + pointer: None, + } + } + + /// Returns the value builder. + #[inline] + pub const fn value_builder(&self) -> &ValueBuilder { + &self.vb + } + + /// Returns the key builder. + #[inline] + pub const fn key_builder(&self) -> &KeyBuilder { + &self.kb + } + + /// Consumes the entry and returns the key and value. + #[inline] + pub fn into_components(self) -> (KeyBuilder, ValueBuilder) { + (self.kb, self.vb) + } +} + +/// An entry in the [`GenericOrderWal`](crate::swmr::GenericOrderWal). +#[repr(transparent)] +pub struct Generic<'a, T> { + value: Either, &'a [u8]>, +} + +impl<'a, T: Type> Generic<'a, T> { + /// Returns the value contained in the generic. + #[inline] + pub const fn value(&self) -> Either<&GenericVariant<'a, T>, &'a [u8]> { + match &self.value { + Either::Left(v) => Either::Left(v), + Either::Right(v) => Either::Right(v), + } + } + + /// Creates a new generic from bytes for querying or inserting into the [`GenericOrderWal`](crate::swmr::GenericOrderWal). + /// + /// ## Safety + /// - the `slice` must the same as the one returned by [`T::encode`](Type::encode). + #[inline] + pub const unsafe fn from_slice(slice: &'a [u8]) -> Self { + Self { + value: Either::Right(slice), + } + } + + #[inline] + pub(crate) fn into_among(self) -> Among { + match self.value { + Either::Left(val) => match val { + GenericVariant::Owned(val) => Among::Left(val), + GenericVariant::Ref(val) => Among::Middle(val), + }, + Either::Right(val) => Among::Right(val), + } + } + + #[inline] + pub(crate) fn encoded_len(&self) -> usize { + match &self.value { + Either::Left(val) => val.encoded_len(), + Either::Right(val) => val.len(), + } + } + + #[inline] + pub(crate) fn encode(&self, buf: &mut [u8]) -> Result<(), T::Error> { + match &self.value { + Either::Left(val) => match val { + GenericVariant::Owned(val) => val.encode(buf), + GenericVariant::Ref(val) => val.encode(buf), + }, + Either::Right(val) => { + buf.copy_from_slice(val); + Ok(()) + } + } + } +} + +impl<'a, T> From> for Generic<'a, T> { + #[inline] + fn from(value: GenericVariant<'a, T>) -> Self { + Self { + value: Either::Left(value), + } + } +} + +impl<'a, T> From<&'a T> for Generic<'a, T> { + #[inline] + fn from(value: &'a T) -> Self { + Self { + value: Either::Left(GenericVariant::Ref(value)), + } + } +} + +impl From for Generic<'_, T> { + #[inline] + fn from(value: T) -> Self { + Self { + value: Either::Left(GenericVariant::Owned(value)), + } + } +} + +/// The kind of a generic type. +pub enum GenericVariant<'a, T> { + /// An owned `T`. + Owned(T), + /// A reference of `T`. + Ref(&'a T), +} + +impl GenericVariant<'_, T> { + #[inline] + fn encoded_len(&self) -> usize { + match self { + Self::Owned(val) => val.encoded_len(), + Self::Ref(val) => val.encoded_len(), + } + } +} + +impl From for GenericVariant<'_, T> { + fn from(value: T) -> Self { + Self::Owned(value) + } +} + +impl<'a, T> From<&'a T> for GenericVariant<'a, T> { + fn from(value: &'a T) -> Self { + Self::Ref(value) + } +} + +/// An entry in the generic write-ahead log. +pub struct GenericEntryRefMut<'a, K, V> { + pub(crate) key: Generic<'a, K>, + pub(crate) value: Generic<'a, V>, + pub(crate) pointer: Option<&'a mut GenericPointer>, +} + +/// An entry in the generic write-ahead log. +pub struct GenericEntry<'a, K, V> { + pub(crate) key: Generic<'a, K>, + pub(crate) value: Generic<'a, V>, + pub(crate) pointer: Option>, +} + +impl<'a, K, V> GenericEntry<'a, K, V> { + /// Creates a new entry. + #[inline] + pub fn new(key: impl Into>, value: impl Into>) -> Self { + Self { + key: key.into(), + value: value.into(), + pointer: None, + } + } + + /// Returns the key. + #[inline] + pub const fn key(&self) -> Either<&GenericVariant<'a, K>, &'a [u8]> { + match &self.key.value { + Either::Left(v) => Either::Left(v), + Either::Right(v) => Either::Right(v), + } + } + + /// Returns the value. + #[inline] + pub const fn value(&self) -> Either<&GenericVariant<'a, V>, &'a [u8]> { + match &self.value.value { + Either::Left(v) => Either::Left(v), + Either::Right(v) => Either::Right(v), + } + } + + /// Consumes the entry and returns the key and value. + #[inline] + pub fn into_components(self) -> (Generic<'a, K>, Generic<'a, V>) { + (self.key, self.value) + } + + #[inline] + pub(crate) fn as_ref_mut(&mut self) -> GenericEntryRefMut<'_, K, V> { + let k = match &self.key.value { + Either::Left(v) => match v { + GenericVariant::Owned(k) => Generic { value: Either::Left(GenericVariant::Ref(k)) }, + GenericVariant::Ref(k) => Generic { value: Either::Left(GenericVariant::Ref(*k)) }, + }, + Either::Right(v) => Generic { value: Either::Right(v) }, + }; + + let v = match &self.value.value { + Either::Left(v) => match v { + GenericVariant::Owned(v) => Generic { value: Either::Left(GenericVariant::Ref(v)) }, + GenericVariant::Ref(v) => Generic { value: Either::Left(GenericVariant::Ref(*v)) }, + }, + Either::Right(v) => Generic { value: Either::Right(v) }, + }; + + GenericEntryRefMut { + key: k, + value: v, + pointer: self.pointer.as_mut(), + } + } +} + +/// The reference to an entry in the [`GenericOrderWal`](super::GenericOrderWal). +#[repr(transparent)] +pub struct GenericEntryRef<'a, K, V> { + ent: SetEntry<'a, GenericPointer>, +} + +impl<'a, K, V> core::fmt::Debug for GenericEntryRef<'a, K, V> +where + K: Type, + K::Ref<'a>: core::fmt::Debug, + V: Type, + V::Ref<'a>: core::fmt::Debug, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("EntryRef") + .field("key", &self.key()) + .field("value", &self.value()) + .finish() + } +} + +impl Clone for GenericEntryRef<'_, K, V> { + #[inline] + fn clone(&self) -> Self { + Self { + ent: self.ent.clone(), + } + } +} + +impl<'a, K, V> GenericEntryRef<'a, K, V> { + #[inline] + pub(super) fn new(ent: SetEntry<'a, GenericPointer>) -> Self { + Self { ent } + } +} + +impl<'a, K, V> GenericEntryRef<'a, K, V> +where + K: Type, + V: Type, +{ + /// Returns the key of the entry. + #[inline] + pub fn key(&self) -> K::Ref<'a> { + let p = self.ent.value(); + unsafe { TypeRef::from_slice(p.as_key_slice()) } + } + + /// Returns the value of the entry. + #[inline] + pub fn value(&self) -> V::Ref<'a> { + let p = self.ent.value(); + unsafe { TypeRef::from_slice(p.as_value_slice()) } + } +} diff --git a/src/lib.rs b/src/lib.rs index 044e3ae..9d989d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,7 @@ #![deny(missing_docs)] #![allow(clippy::type_complexity)] -use core::{borrow::Borrow, cmp, marker::PhantomData, mem, slice}; +use core::{borrow::Borrow, marker::PhantomData, mem}; pub use among; use among::Among; @@ -23,7 +23,7 @@ pub use rarena_allocator::OpenOptions; extern crate std; pub use dbutils::{ - checksum::{BuildChecksumer, Checksumer, Crc32}, + checksum::{self, Crc32}, Ascend, CheapClone, Comparator, Descend, }; @@ -72,14 +72,20 @@ pub mod error; mod buffer; pub use buffer::*; +mod builder; +pub use builder::Builder; + +mod entry; +pub use entry::*; + /// Utilities. pub mod utils; use utils::*; mod wal; -pub use wal::{ - Batch, BatchWithBuilders, BatchWithKeyBuilder, BatchWithValueBuilder, Builder, ImmutableWal, Wal, -}; +// pub use wal::{ +// Batch, BatchWithBuilders, BatchWithKeyBuilder, BatchWithValueBuilder, ImmutableWal, Wal, +// }; mod options; pub use options::Options; @@ -90,6 +96,8 @@ pub mod swmr; /// An ordered write-ahead Log implementation. pub mod unsync; +mod pointer; + bitflags::bitflags! { /// The flags of the entry. struct Flags: u8 { @@ -99,230 +107,3 @@ bitflags::bitflags! { const BATCHING = 0b00000010; } } - -#[doc(hidden)] -pub struct Pointer { - /// The pointer to the start of the entry. - ptr: *const u8, - /// The length of the key. - key_len: usize, - /// The length of the value. - value_len: usize, - cmp: C, -} - -unsafe impl Send for Pointer {} -unsafe impl Sync for Pointer {} - -impl Pointer { - #[inline] - const fn new(key_len: usize, value_len: usize, ptr: *const u8, cmp: C) -> Self { - Self { - ptr, - key_len, - value_len, - cmp, - } - } - - #[inline] - const fn as_key_slice<'a>(&self) -> &'a [u8] { - if self.key_len == 0 { - return &[]; - } - - // SAFETY: `ptr` is a valid pointer to `len` bytes. - unsafe { slice::from_raw_parts(self.ptr, self.key_len) } - } - - #[inline] - const fn as_value_slice<'a, 'b: 'a>(&'a self) -> &'b [u8] { - if self.value_len == 0 { - return &[]; - } - - // SAFETY: `ptr` is a valid pointer to `len` bytes. - unsafe { slice::from_raw_parts(self.ptr.add(self.key_len), self.value_len) } - } -} - -impl PartialEq for Pointer { - fn eq(&self, other: &Self) -> bool { - self - .cmp - .compare(self.as_key_slice(), other.as_key_slice()) - .is_eq() - } -} - -impl Eq for Pointer {} - -impl PartialOrd for Pointer { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Pointer { - fn cmp(&self, other: &Self) -> cmp::Ordering { - self.cmp.compare(self.as_key_slice(), other.as_key_slice()) - } -} - -impl Borrow for Pointer -where - [u8]: Borrow, - Q: ?Sized + Ord, -{ - fn borrow(&self) -> &Q { - self.as_key_slice().borrow() - } -} - -/// An entry in the write-ahead log. -pub struct Entry { - key: K, - value: V, - pointer: Option>, -} - -impl Entry { - /// Creates a new entry. - #[inline] - pub const fn new(key: K, value: V) -> Self { - Self { - key, - value, - pointer: None, - } - } - - /// Returns the key. - #[inline] - pub const fn key(&self) -> &K { - &self.key - } - - /// Returns the value. - #[inline] - pub const fn value(&self) -> &V { - &self.value - } - - /// Consumes the entry and returns the key and value. - #[inline] - pub fn into_components(self) -> (K, V) { - (self.key, self.value) - } -} - -/// An entry in the write-ahead log. -pub struct EntryWithKeyBuilder { - kb: KeyBuilder, - value: V, - pointer: Option>, -} - -impl EntryWithKeyBuilder { - /// Creates a new entry. - #[inline] - pub const fn new(kb: KeyBuilder, value: V) -> Self { - Self { - kb, - value, - pointer: None, - } - } - - /// Returns the key. - #[inline] - pub const fn key_builder(&self) -> &KeyBuilder { - &self.kb - } - - /// Returns the value. - #[inline] - pub const fn value(&self) -> &V { - &self.value - } - - /// Consumes the entry and returns the key and value. - #[inline] - pub fn into_components(self) -> (KeyBuilder, V) { - (self.kb, self.value) - } -} - -/// An entry in the write-ahead log. -pub struct EntryWithValueBuilder { - key: K, - vb: ValueBuilder, - pointer: Option>, -} - -impl EntryWithValueBuilder { - /// Creates a new entry. - #[inline] - pub const fn new(key: K, vb: ValueBuilder) -> Self { - Self { - key, - vb, - pointer: None, - } - } - - /// Returns the key. - #[inline] - pub const fn value_builder(&self) -> &ValueBuilder { - &self.vb - } - - /// Returns the value. - #[inline] - pub const fn key(&self) -> &K { - &self.key - } - - /// Consumes the entry and returns the key and value. - #[inline] - pub fn into_components(self) -> (K, ValueBuilder) { - (self.key, self.vb) - } -} - -/// An entry in the write-ahead log. -pub struct EntryWithBuilders { - kb: KeyBuilder, - vb: ValueBuilder, - pointer: Option>, -} - -impl EntryWithBuilders { - /// Creates a new entry. - #[inline] - pub const fn new(kb: KeyBuilder, vb: ValueBuilder) -> Self { - Self { - kb, - vb, - pointer: None, - } - } - - /// Returns the value builder. - #[inline] - pub const fn value_builder(&self) -> &ValueBuilder { - &self.vb - } - - /// Returns the key builder. - #[inline] - pub const fn key_builder(&self) -> &KeyBuilder { - &self.kb - } - - /// Consumes the entry and returns the key and value. - #[inline] - pub fn into_components(self) -> (KeyBuilder, ValueBuilder) { - (self.kb, self.vb) - } -} diff --git a/src/options.rs b/src/options.rs index cefac44..092460a 100644 --- a/src/options.rs +++ b/src/options.rs @@ -21,7 +21,7 @@ impl Options { /// Create a new `Options` instance. /// /// - /// # Example + /// ## Example /// /// **Note:** If you are creating in-memory WAL, then you must specify the capacity. /// @@ -50,7 +50,7 @@ impl Options { /// /// The default reserved is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -70,7 +70,7 @@ impl Options { /// /// The default reserved is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -88,7 +88,7 @@ impl Options { /// /// The default value is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -105,7 +105,7 @@ impl Options { /// /// The default value is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -122,7 +122,7 @@ impl Options { /// /// The default value is `u16::MAX`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -139,7 +139,7 @@ impl Options { /// /// The default value is `u32::MAX`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -156,7 +156,7 @@ impl Options { /// /// The default value is `true`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -183,7 +183,7 @@ impl Options { /// /// The default value is `None`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -202,7 +202,7 @@ impl Options { /// /// The default value is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -218,7 +218,7 @@ impl Options { /// Sets the maximum key length. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -234,7 +234,7 @@ impl Options { /// Sets the maximum value length. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -262,7 +262,7 @@ impl Options { /// /// The default value is `None`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -280,7 +280,7 @@ impl Options { /// /// The default value is `true`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; @@ -298,7 +298,7 @@ impl Options { /// /// The default value is `0`. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::Options; diff --git a/src/pointer.rs b/src/pointer.rs new file mode 100644 index 0000000..9963a18 --- /dev/null +++ b/src/pointer.rs @@ -0,0 +1,177 @@ +use core::{borrow::Borrow, cmp, marker::PhantomData, slice}; + +use dbutils::Comparator; + +use super::wal::r#type::{KeyRef, Type}; + +#[doc(hidden)] +pub struct Pointer { + /// The pointer to the start of the entry. + ptr: *const u8, + /// The length of the key. + key_len: usize, + /// The length of the value. + value_len: usize, + cmp: C, +} + +unsafe impl Send for Pointer {} +unsafe impl Sync for Pointer {} + +impl Pointer { + #[inline] + pub(crate) const fn new(key_len: usize, value_len: usize, ptr: *const u8, cmp: C) -> Self { + Self { + ptr, + key_len, + value_len, + cmp, + } + } + + #[inline] + pub const fn as_key_slice<'a>(&self) -> &'a [u8] { + if self.key_len == 0 { + return &[]; + } + + // SAFETY: `ptr` is a valid pointer to `len` bytes. + unsafe { slice::from_raw_parts(self.ptr, self.key_len) } + } + + #[inline] + pub const fn as_value_slice<'a, 'b: 'a>(&'a self) -> &'b [u8] { + if self.value_len == 0 { + return &[]; + } + + // SAFETY: `ptr` is a valid pointer to `len` bytes. + unsafe { slice::from_raw_parts(self.ptr.add(self.key_len), self.value_len) } + } +} + +impl PartialEq for Pointer { + fn eq(&self, other: &Self) -> bool { + self + .cmp + .compare(self.as_key_slice(), other.as_key_slice()) + .is_eq() + } +} + +impl Eq for Pointer {} + +impl PartialOrd for Pointer { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Pointer { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.cmp.compare(self.as_key_slice(), other.as_key_slice()) + } +} + +impl Borrow for Pointer +where + [u8]: Borrow, + Q: ?Sized + Ord, +{ + fn borrow(&self) -> &Q { + self.as_key_slice().borrow() + } +} + +impl super::wal::sealed::Pointer for Pointer { + type Comparator = C; + + #[inline] + fn new(klen: usize, vlen: usize, ptr: *const u8, cmp: C) -> Self { + Pointer::::new(klen, vlen, ptr, cmp) + } +} + +#[doc(hidden)] +#[derive(Debug)] +pub struct GenericPointer { + /// The pointer to the start of the entry. + ptr: *const u8, + /// The length of the key. + key_len: usize, + /// The length of the value. + value_len: usize, + _m: PhantomData<(fn() -> K, fn() -> V)>, +} + +impl crate::wal::sealed::Pointer for GenericPointer { + type Comparator = (); + + #[inline] + fn new(klen: usize, vlen: usize, ptr: *const u8, _cmp: Self::Comparator) -> Self { + Self::new(klen, vlen, ptr) + } +} + +impl PartialEq for GenericPointer { + fn eq(&self, other: &Self) -> bool { + self.as_key_slice() == other.as_key_slice() + } +} + +impl Eq for GenericPointer {} + +impl PartialOrd for GenericPointer +where + K: Type + Ord, + for<'a> K::Ref<'a>: KeyRef<'a, K>, +{ + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for GenericPointer +where + K: Type + Ord, + for<'a> K::Ref<'a>: KeyRef<'a, K>, +{ + fn cmp(&self, other: &Self) -> cmp::Ordering { + as KeyRef>::compare_binary(self.as_key_slice(), other.as_key_slice()) + } +} + +unsafe impl Send for GenericPointer {} +unsafe impl Sync for GenericPointer {} + +impl GenericPointer { + #[inline] + pub(crate) const fn new(key_len: usize, value_len: usize, ptr: *const u8) -> Self { + Self { + ptr, + key_len, + value_len, + _m: PhantomData, + } + } + + #[inline] + pub const fn as_key_slice<'a>(&self) -> &'a [u8] { + if self.key_len == 0 { + return &[]; + } + + // SAFETY: `ptr` is a valid pointer to `len` bytes. + unsafe { slice::from_raw_parts(self.ptr, self.key_len) } + } + + #[inline] + pub const fn as_value_slice<'a, 'b: 'a>(&'a self) -> &'b [u8] { + if self.value_len == 0 { + return &[]; + } + + // SAFETY: `ptr` is a valid pointer to `len` bytes. + unsafe { slice::from_raw_parts(self.ptr.add(self.key_len), self.value_len) } + } +} diff --git a/src/swmr.rs b/src/swmr.rs index 763bab8..c3758f8 100644 --- a/src/swmr.rs +++ b/src/swmr.rs @@ -1,7 +1,7 @@ /// The ordered write-ahead log only supports bytes. pub mod wal; -pub use wal::OrderWal; +pub use wal::{Builder, OrderWal}; /// The generic implementation of the ordered write-ahead log. pub mod generic; -pub use generic::GenericOrderWal; +pub use generic::{GenericBuilder, GenericOrderWal}; diff --git a/src/swmr/generic.rs b/src/swmr/generic.rs index 38ea395..5280ad3 100644 --- a/src/swmr/generic.rs +++ b/src/swmr/generic.rs @@ -6,22 +6,27 @@ use std::{ use among::Among; use crossbeam_skiplist::SkipSet; -use dbutils::checksum::{BuildChecksumer, Checksumer, Crc32}; -pub use dbutils::equivalent::*; +use dbutils::{ + checksum::{BuildChecksumer, Checksumer, Crc32}, + leb128::encoded_u64_varint_len, +}; use rarena_allocator::{either::Either, sync::Arena, Allocator, Memory, MmapOptions, OpenOptions}; use crate::{ arena_options, check, entry_size, error::{self, Error}, + merge_lengths, + pointer::GenericPointer, wal::sealed::Constructor, - Flags, Options, HEADER_SIZE, STATUS_SIZE, + Flags, Options, CHECKSUM_SIZE, HEADER_SIZE, STATUS_SIZE, }; -mod entry; -pub use entry::*; +pub use crate::{ + entry::{Generic, GenericEntry, GenericEntryRef, GenericVariant}, + wal::{r#type::*, GenericBatch}, +}; -mod traits; -pub use traits::*; +pub use dbutils::equivalent::{Comparable, Equivalent}; mod reader; pub use reader::*; @@ -29,138 +34,22 @@ pub use reader::*; mod iter; pub use iter::*; -#[cfg(all( - test, - any( - all_tests, - test_swmr_generic_constructor, - test_swmr_generic_insert, - test_swmr_generic_get, - test_swmr_generic_iters, - ) -))] -mod tests; - +mod builder; +pub use builder::*; + +// #[cfg(all( +// test, +// any( +// all_tests, +// test_swmr_generic_constructor, +// test_swmr_generic_insert, +// test_swmr_generic_get, +// test_swmr_generic_iters, +// ) +// ))] +#[cfg(test)] #[doc(hidden)] -#[derive(Debug)] -pub struct Pointer { - /// The pointer to the start of the entry. - ptr: *const u8, - /// The length of the key. - key_len: usize, - /// The length of the value. - value_len: usize, - _m: PhantomData<(K, V)>, -} - -impl crate::wal::sealed::Pointer for Pointer { - type Comparator = (); - - #[inline] - fn new(klen: usize, vlen: usize, ptr: *const u8, _cmp: Self::Comparator) -> Self { - Self::new(klen, vlen, ptr) - } -} - -impl PartialEq for Pointer { - fn eq(&self, other: &Self) -> bool { - self.as_key_slice() == other.as_key_slice() - } -} - -impl Eq for Pointer {} - -impl PartialOrd for Pointer -where - K: Type + Ord, - for<'a> K::Ref<'a>: KeyRef<'a, K>, -{ - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Pointer -where - K: Type + Ord, - for<'a> K::Ref<'a>: KeyRef<'a, K>, -{ - fn cmp(&self, other: &Self) -> cmp::Ordering { - as KeyRef>::compare_binary(self.as_key_slice(), other.as_key_slice()) - } -} - -unsafe impl Send for Pointer {} -unsafe impl Sync for Pointer {} - -impl Pointer { - #[inline] - const fn new(key_len: usize, value_len: usize, ptr: *const u8) -> Self { - Self { - ptr, - key_len, - value_len, - _m: PhantomData, - } - } - - #[inline] - const fn as_key_slice<'a>(&self) -> &'a [u8] { - if self.key_len == 0 { - return &[]; - } - - // SAFETY: `ptr` is a valid pointer to `len` bytes. - unsafe { slice::from_raw_parts(self.ptr, self.key_len) } - } - - #[inline] - const fn as_value_slice<'a, 'b: 'a>(&'a self) -> &'b [u8] { - if self.value_len == 0 { - return &[]; - } - - // SAFETY: `ptr` is a valid pointer to `len` bytes. - unsafe { slice::from_raw_parts(self.ptr.add(self.key_len), self.value_len) } - } -} - -/// An entry in the generic write-ahead log. -pub struct GenericEntry { - key: K, - value: V, - pointer: Option>, -} - -impl GenericEntry { - /// Creates a new entry. - #[inline] - pub const fn new(key: K, value: V) -> Self { - Self { - key, - value, - pointer: None, - } - } - - /// Returns the key. - #[inline] - pub const fn key(&self) -> &K { - &self.key - } - - /// Returns the value. - #[inline] - pub const fn value(&self) -> &V { - &self.value - } - - /// Consumes the entry and returns the key and value. - #[inline] - pub fn into_components(self) -> (K, V) { - (self.key, self.value) - } -} +pub mod tests; struct PartialPointer { key_len: usize, @@ -217,22 +106,22 @@ impl PartialPointer { } } -impl<'a, K, V> Equivalent> for PartialPointer +impl<'a, K, V> Equivalent> for PartialPointer where K: Type + Ord, K::Ref<'a>: KeyRef<'a, K>, { - fn equivalent(&self, key: &Pointer) -> bool { + fn equivalent(&self, key: &GenericPointer) -> bool { self.compare(key).is_eq() } } -impl<'a, K, V> Comparable> for PartialPointer +impl<'a, K, V> Comparable> for PartialPointer where K: Type + Ord, K::Ref<'a>: KeyRef<'a, K>, { - fn compare(&self, p: &Pointer) -> cmp::Ordering { + fn compare(&self, p: &GenericPointer) -> cmp::Ordering { unsafe { let kr: K::Ref<'_> = TypeRef::from_slice(p.as_key_slice()); let or: K::Ref<'_> = TypeRef::from_slice(self.as_key_slice()); @@ -257,24 +146,24 @@ impl<'a, K, Q: ?Sized> Ref<'a, K, Q> { } } -impl<'a, K, Q, V> Equivalent> for Ref<'a, K, Q> +impl<'a, K, Q, V> Equivalent> for Ref<'a, K, Q> where K: Type + Ord, K::Ref<'a>: KeyRef<'a, K>, Q: ?Sized + Ord + Comparable>, { - fn equivalent(&self, key: &Pointer) -> bool { + fn equivalent(&self, key: &GenericPointer) -> bool { self.compare(key).is_eq() } } -impl<'a, K, Q, V> Comparable> for Ref<'a, K, Q> +impl<'a, K, Q, V> Comparable> for Ref<'a, K, Q> where K: Type + Ord, K::Ref<'a>: KeyRef<'a, K>, Q: ?Sized + Ord + Comparable>, { - fn compare(&self, p: &Pointer) -> cmp::Ordering { + fn compare(&self, p: &GenericPointer) -> cmp::Ordering { let kr = unsafe { TypeRef::from_slice(p.as_key_slice()) }; KeyRef::compare(&kr, self.key).reverse() } @@ -296,24 +185,24 @@ impl<'a, K, Q: ?Sized> Owned<'a, K, Q> { } } -impl<'a, K, Q, V> Equivalent> for Owned<'a, K, Q> +impl<'a, K, Q, V> Equivalent> for Owned<'a, K, Q> where K: Type + Ord, K::Ref<'a>: KeyRef<'a, K>, Q: ?Sized + Ord + Comparable + Comparable>, { - fn equivalent(&self, key: &Pointer) -> bool { + fn equivalent(&self, key: &GenericPointer) -> bool { self.compare(key).is_eq() } } -impl<'a, K, Q, V> Comparable> for Owned<'a, K, Q> +impl<'a, K, Q, V> Comparable> for Owned<'a, K, Q> where K: Type + Ord, K::Ref<'a>: KeyRef<'a, K>, Q: ?Sized + Ord + Comparable + Comparable>, { - fn compare(&self, p: &Pointer) -> cmp::Ordering { + fn compare(&self, p: &GenericPointer) -> cmp::Ordering { let kr = unsafe { as TypeRef<'_>>::from_slice(p.as_key_slice()) }; KeyRef::compare(&kr, self.key).reverse() } @@ -322,21 +211,17 @@ where #[doc(hidden)] pub struct GenericOrderWalCore { arena: Arena, - map: SkipSet>, + map: SkipSet>, opts: Options, cks: S, } -impl crate::wal::sealed::WalCore<(), S> for GenericOrderWalCore -where - K: 'static, - V: 'static, -{ +impl crate::wal::sealed::WalCore<(), S> for GenericOrderWalCore { type Allocator = Arena; - type Base = SkipSet>; + type Base = SkipSet>; - type Pointer = Pointer; + type Pointer = GenericPointer; #[inline] fn construct(arena: Self::Allocator, base: Self::Base, opts: Options, _cmp: (), cks: S) -> Self { @@ -361,21 +246,21 @@ impl GenericOrderWalCore { } #[inline] - fn first(&self) -> Option> + fn first(&self) -> Option> where K: Type + Ord, for<'b> K::Ref<'b>: KeyRef<'b, K>, { - self.map.front().map(EntryRef::new) + self.map.front().map(GenericEntryRef::new) } #[inline] - fn last(&self) -> Option> + fn last(&self) -> Option> where K: Type + Ord, for<'b> K::Ref<'b>: KeyRef<'b, K>, { - self.map.back().map(EntryRef::new) + self.map.back().map(GenericEntryRef::new) } #[inline] @@ -424,16 +309,12 @@ impl GenericOrderWalCore { } } -impl Constructor<(), S> for GenericOrderWal -where - K: 'static, - V: 'static, -{ +impl Constructor<(), S> for GenericOrderWal { type Allocator = Arena; type Core = GenericOrderWalCore; - type Pointer = Pointer; + type Pointer = GenericPointer; fn allocator(&self) -> &Self::Allocator { &self.core.arena @@ -477,33 +358,33 @@ where } #[inline] - fn get<'a, Q>(&'a self, key: &'a Q) -> Option> + fn get<'a, Q>(&'a self, key: &'a Q) -> Option> where Q: ?Sized + Ord + Comparable> + Comparable, { self .map .get::>(&Owned::new(key)) - .map(EntryRef::new) + .map(GenericEntryRef::new) } #[inline] - fn get_by_ref<'a, Q>(&'a self, key: &'a Q) -> Option> + fn get_by_ref<'a, Q>(&'a self, key: &'a Q) -> Option> where Q: ?Sized + Ord + Comparable>, { self .map .get::>(&Ref::new(key)) - .map(EntryRef::new) + .map(GenericEntryRef::new) } #[inline] - unsafe fn get_by_bytes(&self, key: &[u8]) -> Option> { + unsafe fn get_by_bytes(&self, key: &[u8]) -> Option> { self .map .get(&PartialPointer::new(key.len(), key.as_ptr())) - .map(EntryRef::new) + .map(GenericEntryRef::new) } } @@ -517,40 +398,6 @@ pub struct GenericOrderWal { ro: bool, } -impl GenericOrderWal -where - K: 'static, - V: 'static, -{ - /// Creates a new in-memory write-ahead log backed by an aligned vec with the given capacity and options. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::GenericOrderWal, Options}; - /// - /// let wal = GenericOrderWal::::new(Options::new().with_capacity(1024)).unwrap(); - /// ``` - #[inline] - pub fn new(opts: Options) -> Result { - Self::with_checksumer(opts, Crc32::default()) - } - - /// Creates a new in-memory write-ahead log backed by an anonymous memory map with the given options. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::GenericOrderWal, Options}; - /// - /// let wal = GenericOrderWal::::map_anon(Options::new().with_capacity(1024)).unwrap(); - /// ``` - #[inline] - pub fn map_anon(opts: Options) -> Result { - Self::map_anon_with_checksumer(opts, Crc32::default()) - } -} - impl GenericOrderWal where K: Type + Ord + 'static, @@ -558,13 +405,13 @@ where { /// Returns the first key-value pair in the map. The key in this pair is the minimum key in the wal. #[inline] - pub fn first(&self) -> Option> { + pub fn first(&self) -> Option> { self.core.first() } /// Returns the last key-value pair in the map. The key in this pair is the maximum key in the wal. #[inline] - pub fn last(&self) -> Option> { + pub fn last(&self) -> Option> { self.core.last() } @@ -601,136 +448,6 @@ where } } -impl GenericOrderWal -where - K: Type + Ord + 'static, - for<'a> ::Ref<'a>: KeyRef<'a, K>, - V: Type + 'static, -{ - /// Creates a new write-ahead log backed by a file backed memory map with the given options. - /// - /// ## Safety - /// - /// All file-backed memory map constructors are marked `unsafe` because of the potential for - /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or - /// out of process. Applications must consider the risk and take appropriate precautions when - /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. - /// unlinked) files exist but are platform specific and limited. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::{GenericOrderWal, generic::*}, OpenOptions, Options}; - /// - /// # let dir = tempfile::tempdir().unwrap(); - /// # let path = dir - /// # .path() - /// # .join("generic_wal_map_mut"); - /// - /// let mut wal = unsafe { - /// GenericOrderWal::::map_mut( - /// &path, - /// Options::new(), - /// OpenOptions::new() - /// .create_new(Some(1024)) - /// .write(true) - /// .read(true), - /// ) - /// .unwrap() - /// }; - /// ``` - #[inline] - pub unsafe fn map_mut>( - path: P, - opts: Options, - open_options: OpenOptions, - ) -> Result { - Self::map_mut_with_path_builder::<_, ()>(|| dummy_path_builder(path), opts, open_options) - .map_err(|e| e.unwrap_right()) - } - - /// Creates a new write-ahead log backed by a file backed memory map with the given options. - /// - /// ## Safety - /// - /// All file-backed memory map constructors are marked `unsafe` because of the potential for - /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or - /// out of process. Applications must consider the risk and take appropriate precautions when - /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. - /// unlinked) files exist but are platform specific and limited. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::{GenericOrderWal, generic::*}, OpenOptions, Options}; - /// - /// let dir = tempfile::tempdir().unwrap(); - /// - /// let mut wal = unsafe { - /// GenericOrderWal::::map_mut_with_path_builder::<_, ()>( - /// || { - /// Ok(dir.path().join("generic_wal_map_mut_with_path_builder")) - /// }, - /// Options::new(), - /// OpenOptions::new() - /// .create_new(Some(1024)) - /// .write(true) - /// .read(true), - /// ) - /// .unwrap() - /// }; - /// ``` - #[inline] - pub unsafe fn map_mut_with_path_builder( - pb: PB, - opts: Options, - open_options: OpenOptions, - ) -> Result> - where - PB: FnOnce() -> Result, - { - Self::map_mut_with_path_builder_and_checksumer(pb, opts, open_options, Crc32::default()) - } - - /// Open a write-ahead log backed by a file backed memory map in read only mode. - /// - /// ## Safety - /// - /// All file-backed memory map constructors are marked `unsafe` because of the potential for - /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or - /// out of process. Applications must consider the risk and take appropriate precautions when - /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. - /// unlinked) files exist but are platform specific and limited. - #[inline] - pub unsafe fn map>( - path: P, - opts: Options, - ) -> Result, Error> { - Self::map_with_path_builder::<_, ()>(|| dummy_path_builder(path), opts) - .map_err(|e| e.unwrap_right()) - } - - /// Open a write-ahead log backed by a file backed memory map in read only mode. - /// - /// ## Safety - /// - /// All file-backed memory map constructors are marked `unsafe` because of the potential for - /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or - /// out of process. Applications must consider the risk and take appropriate precautions when - /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. - /// unlinked) files exist but are platform specific and limited. - #[inline] - pub unsafe fn map_with_path_builder( - pb: PB, - opts: Options, - ) -> Result, Either> - where - PB: FnOnce() -> Result, - { - Self::map_with_path_builder_and_checksumer(pb, opts, Crc32::default()) - } -} - impl GenericOrderWal where K: 'static, @@ -750,7 +467,7 @@ where /// Returns the reserved space in the WAL. /// - /// # Safety + /// ## Safety /// - The writer must ensure that the returned slice is not modified. /// - This method is not thread-safe, so be careful when using it. #[inline] @@ -764,7 +481,7 @@ where /// Returns the mutable reference to the reserved slice. /// - /// # Safety + /// ## Safety /// - The caller must ensure that the there is no others accessing reserved slice for either read or write. /// - This method is not thread-safe, so be careful when using it. #[inline] @@ -787,236 +504,6 @@ where pub fn is_empty(&self) -> bool { self.core.is_empty() } - - /// Creates a new in-memory write-ahead log backed by an aligned vec with the given options and [`Checksumer`]. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::GenericOrderWal, Options, Crc32}; - /// - /// let wal = GenericOrderWal::::with_checksumer(Options::new().with_capacity(1024), Crc32::default()); - /// ``` - pub fn with_checksumer(opts: Options, cks: S) -> Result { - let arena = Arena::new(arena_options(opts.reserved()).with_capacity(opts.capacity())) - .map_err(Error::from_insufficient_space)?; - - Self::new_in(arena, opts, (), cks).map(Self::from_core) - } - - /// Creates a new in-memory write-ahead log backed by an anonymous memory map with the given options and [`Checksumer`]. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::GenericOrderWal, Options, Crc32}; - /// - /// let wal = GenericOrderWal::::map_anon_with_checksumer(Options::new().with_capacity(1024), Crc32::default()).unwrap(); - /// ``` - pub fn map_anon_with_checksumer(opts: Options, cks: S) -> Result { - let arena = Arena::map_anon( - arena_options(opts.reserved()), - MmapOptions::new().len(opts.capacity()), - )?; - - Self::new_in(arena, opts, (), cks).map(Self::from_core) - } -} - -impl GenericOrderWal -where - K: Type + Ord + 'static, - for<'a> ::Ref<'a>: KeyRef<'a, K>, - V: Type + 'static, - S: BuildChecksumer, -{ - /// Returns a write-ahead log backed by a file backed memory map with the given options and [`Checksumer`]. - /// - /// ## Safety - /// - /// All file-backed memory map constructors are marked `unsafe` because of the potential for - /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or - /// out of process. Applications must consider the risk and take appropriate precautions when - /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. - /// unlinked) files exist but are platform specific and limited. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::{GenericOrderWal, generic::*}, Crc32, OpenOptions, Options}; - /// - /// # let dir = tempfile::tempdir().unwrap(); - /// # let path = dir - /// # .path() - /// # .join("generic_wal_map_mut_with_checksumer"); - /// - /// let mut wal = unsafe { - /// GenericOrderWal::::map_mut_with_checksumer( - /// &path, - /// Options::new(), - /// OpenOptions::new() - /// .create_new(Some(1024)) - /// .write(true) - /// .read(true), - /// Crc32::default(), - /// ) - /// .unwrap() - /// }; - /// ``` - #[inline] - pub unsafe fn map_mut_with_checksumer>( - path: P, - opts: Options, - open_options: OpenOptions, - cks: S, - ) -> Result { - Self::map_mut_with_path_builder_and_checksumer::<_, ()>( - || dummy_path_builder(path), - opts, - open_options, - cks, - ) - .map_err(|e| e.unwrap_right()) - } - - /// Returns a write-ahead log backed by a file backed memory map with the given options and [`Checksumer`]. - /// - /// ## Safety - /// - /// All file-backed memory map constructors are marked `unsafe` because of the potential for - /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or - /// out of process. Applications must consider the risk and take appropriate precautions when - /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. - /// unlinked) files exist but are platform specific and limited. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::{GenericOrderWal, generic::*}, Crc32, OpenOptions, Options}; - /// - /// let dir = tempfile::tempdir().unwrap(); - /// - /// let mut wal = unsafe { - /// GenericOrderWal::::map_mut_with_path_builder_and_checksumer::<_, ()>( - /// || { - /// Ok(dir.path().join("generic_wal_map_mut_with_path_builder_and_checksumer")) - /// }, - /// Options::new(), - /// OpenOptions::new() - /// .create_new(Some(1024)) - /// .write(true) - /// .read(true), - /// Crc32::default(), - /// ) - /// .unwrap() - /// }; - /// ``` - pub unsafe fn map_mut_with_path_builder_and_checksumer( - path_builder: PB, - opts: Options, - open_options: OpenOptions, - cks: S, - ) -> Result> - where - PB: FnOnce() -> Result, - { - let path = path_builder().map_err(Either::Left)?; - let exist = path.exists(); - let arena = Arena::map_mut_with_path_builder( - || Ok(path), - arena_options(opts.reserved()), - open_options, - MmapOptions::new(), - ) - .map_err(|e| e.map_right(Into::into))?; - - if !exist { - return Self::new_in(arena, opts, (), cks) - .map(Self::from_core) - .map_err(Either::Right); - } - - Self::replay(arena, opts, false, (), cks) - .map(Self::from_core) - .map_err(Either::Right) - } - - /// Open a write-ahead log backed by a file backed memory map in read only mode with the given [`Checksumer`]. - /// - /// ## Safety - /// - /// All file-backed memory map constructors are marked `unsafe` because of the potential for - /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or - /// out of process. Applications must consider the risk and take appropriate precautions when - /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. - /// unlinked) files exist but are platform specific and limited. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::{GenericOrderWal, generic::*}, Crc32, OpenOptions, Options}; - /// - /// # let dir = tempfile::tempdir().unwrap(); - /// # let path = dir - /// # .path() - /// # .join("generic_wal_map_mut_with_checksumer"); - /// - /// # let mut wal = unsafe { - /// # GenericOrderWal::::map_mut_with_checksumer( - /// # &path, - /// # Options::new(), - /// # OpenOptions::new() - /// # .create_new(Some(1024)) - /// # .write(true) - /// # .read(true), - /// # Crc32::default(), - /// # ) - /// # .unwrap() - /// # }; - /// - /// let reader = unsafe { GenericOrderWal::::map_with_checksumer(&path, Options::new(), Crc32::default()).unwrap() }; - /// ``` - #[inline] - pub unsafe fn map_with_checksumer>( - path: P, - opts: Options, - cks: S, - ) -> Result, Error> { - Self::map_with_path_builder_and_checksumer::<_, ()>(|| dummy_path_builder(path), opts, cks) - .map_err(|e| e.unwrap_right()) - } - - /// Open a write-ahead log backed by a file backed memory map in read only mode with the given [`Checksumer`]. - /// - /// ## Safety - /// - /// All file-backed memory map constructors are marked `unsafe` because of the potential for - /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or - /// out of process. Applications must consider the risk and take appropriate precautions when - /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. - /// unlinked) files exist but are platform specific and limited. - #[inline] - pub unsafe fn map_with_path_builder_and_checksumer( - path_builder: PB, - opts: Options, - cks: S, - ) -> Result, Either> - where - PB: FnOnce() -> Result, - { - let open_options = OpenOptions::default().read(true); - let arena = Arena::map_with_path_builder( - path_builder, - arena_options(opts.reserved()), - open_options, - MmapOptions::new(), - ) - .map_err(|e| e.map_right(Into::into))?; - - Self::replay(arena, opts, true, (), cks) - .map(|core| GenericWalReader::new(Arc::new(core))) - .map_err(Either::Right) - } } impl GenericOrderWal @@ -1036,7 +523,7 @@ where /// Returns `true` if the key exists in the WAL. /// - /// # Safety + /// ## Safety /// - The given `key` must be valid to construct to `K::Ref` without remaining. #[inline] pub unsafe fn contains_key_by_bytes(&self, key: &[u8]) -> bool { @@ -1054,7 +541,7 @@ where /// Gets the value associated with the key. #[inline] - pub fn get<'a, Q>(&'a self, key: &'a Q) -> Option> + pub fn get<'a, Q>(&'a self, key: &'a Q) -> Option> where Q: ?Sized + Ord + Comparable> + Comparable, { @@ -1063,7 +550,7 @@ where /// Gets the value associated with the key. #[inline] - pub fn get_by_ref<'a, Q>(&'a self, key: &'a Q) -> Option> + pub fn get_by_ref<'a, Q>(&'a self, key: &'a Q) -> Option> where Q: ?Sized + Ord + Comparable>, { @@ -1072,10 +559,10 @@ where /// Gets the value associated with the key. /// - /// # Safety + /// ## Safety /// - The given `key` must be valid to construct to `K::Ref` without remaining. #[inline] - pub unsafe fn get_by_bytes(&self, key: &[u8]) -> Option> { + pub unsafe fn get_by_bytes(&self, key: &[u8]) -> Option> { self.core.get_by_bytes(key) } } @@ -1089,146 +576,55 @@ where { /// Gets or insert the key value pair. #[inline] - pub fn get_or_insert( - &mut self, - key: &K, - value: &V, - ) -> Either, Result<(), Among>> { - let ent = self - .core - .map - .get(&Owned::new(key)) - .map(|e| Either::Left(EntryRef::new(e))); - - match ent { - Some(e) => e, - None => { - let p = self.insert_in(Among::Middle(key), Among::Middle(value)); - Either::Right(p) - } - } - } - - /// Gets or insert the key value pair. - #[inline] - pub fn get_or_insert_with( + pub fn get_or_insert<'a>( &mut self, - key: &K, - value: impl FnOnce() -> V, - ) -> Either, Result<(), Among>> { - let ent = self - .core - .map - .get(&Ref::new(key)) - .map(|e| Either::Left(EntryRef::new(e))); - - match ent { - Some(e) => e, - None => { - let p = self.insert_in(Among::Middle(key), Among::Left(value())); - Either::Right(p) + key: impl Into>, + value: impl Into>, + ) -> Either, Result<(), Among>> { + let key: Generic<'a, K> = key.into(); + let map = &self.core.map; + let ent = match key.value() { + Either::Left(key) => { + let k = match key { + GenericVariant::Owned(k) => k, + GenericVariant::Ref(k) => *k, + }; + + map.get(&Owned::new(k)) } - } - } - - /// Gets or insert the key value pair. - /// - /// # Safety - /// - The given `key` and `value` must be valid to construct to `K::Ref` and `V::Ref` without remaining. - #[inline] - pub unsafe fn get_by_bytes_or_insert_bytes( - &mut self, - key: &[u8], - value: &[u8], - ) -> Either, Result<(), Error>> { - let ent = self - .core - .map - .get(&PartialPointer::new(key.len(), key.as_ptr())) - .map(|e| Either::Left(EntryRef::new(e))); + Either::Right(key) => map.get(&PartialPointer::new(key.len(), key.as_ptr())), + }; - match ent { + match ent.map(|e| Either::Left(GenericEntryRef::new(e))) { Some(e) => e, - None => match self.insert_in(Among::Right(key), Among::Right(value)) { - Ok(_) => Either::Right(Ok(())), - Err(Among::Right(e)) => Either::Right(Err(e)), - _ => unreachable!(), - }, + None => Either::Right(self.insert_in(key.into_among(), value.into().into_among())), } } - /// Gets or insert the key value pair. - /// - /// # Safety - /// - The given `value` must be valid to construct to `V::Ref` without remaining. - #[inline] - pub unsafe fn get_or_insert_bytes( - &mut self, - key: &K, - value: &[u8], - ) -> Either, Result<(), Either>> { - let ent = self - .core - .map - .get(&Owned::new(key)) - .map(|e| Either::Left(EntryRef::new(e))); - - match ent { - Some(e) => e, - None => match self.insert_in(Among::Middle(key), Among::Right(value)) { - Ok(_) => Either::Right(Ok(())), - Err(e) => Either::Right(Err(e.into_left_right())), - }, - } - } - - /// Gets or insert the key value pair. - /// - /// # Safety - /// - The given `key` must be valid to construct to `K::Ref` without remaining. - #[inline] - pub unsafe fn get_by_bytes_or_insert( - &mut self, - key: &[u8], - value: &V, - ) -> Either, Result<(), Either>> { - let ent = self - .core - .map - .get(&PartialPointer::new(key.len(), key.as_ptr())) - .map(|e| Either::Left(EntryRef::new(e))); - - match ent { - Some(e) => e, - None => match self.insert_in(Among::Right(key), Among::Middle(value)) { - Ok(_) => Either::Right(Ok(())), - Err(e) => Either::Right(Err(e.into_middle_right())), - }, - } - } - - /// Gets or insert the key value pair. - /// - /// # Safety - /// - The given `key` must be valid to construct to `K::Ref` without remaining. + /// Gets or insert the key with a value builder. #[inline] - pub unsafe fn get_by_bytes_or_insert_with( + pub fn get_or_insert_with<'a>( &mut self, - key: &[u8], - value: impl FnOnce() -> V, - ) -> Either, Result<(), Either>> { - let ent = self - .core - .map - .get(&PartialPointer::new(key.len(), key.as_ptr())) - .map(|e| Either::Left(EntryRef::new(e))); + key: impl Into>, + value: impl FnOnce() -> Generic<'a, V>, + ) -> Either, Result<(), Among>> { + let key: Generic<'a, K> = key.into(); + let map = &self.core.map; + let ent = match key.value() { + Either::Left(key) => { + let k = match key { + GenericVariant::Owned(k) => k, + GenericVariant::Ref(k) => *k, + }; + + map.get(&Owned::new(k)) + } + Either::Right(key) => map.get(&PartialPointer::new(key.len(), key.as_ptr())), + }; - match ent { + match ent.map(|e| Either::Left(GenericEntryRef::new(e))) { Some(e) => e, - None => match self.insert_in(Among::Right(key), Among::Left(value())) { - Ok(_) => Either::Right(Ok(())), - Err(e) => Either::Right(Err(e.into_middle_right())), - }, + None => Either::Right(self.insert_in(key.into_among(), value().into_among())), } } } @@ -1241,181 +637,81 @@ where S: BuildChecksumer, { /// Inserts a key-value pair into the write-ahead log. - #[inline] - pub fn insert(&mut self, key: &K, val: &V) -> Result<(), Among> { - self.insert_in(Among::Middle(key), Among::Middle(val)) - } - - /// Inserts a key in structured format and value in bytes format into the write-ahead log directly. - /// - /// # Safety - /// - The given `value` must be valid to construct to `V::Ref` without remaining. - /// - /// # Example - /// - /// See [`insert_bytes`](GenericOrderWal::insert_bytes) for more details. - #[inline] - pub unsafe fn insert_key_with_value_bytes( - &mut self, - key: &K, - value: &[u8], - ) -> Result<(), Either> { - self - .insert_in(Among::Middle(key), Among::Right(value)) - .map_err(Among::into_left_right) - } - - /// Inserts a key in bytes format and value in structured format into the write-ahead log directly. - /// - /// # Safety - /// - The given `key` and `value` must be valid to construct to `K::Ref` and `V::Ref` without remaining. - /// - /// # Example - /// - /// ```rust - /// use orderwal::{swmr::{*, generic::*}, utils::*, Options}; - /// use std::cmp; - /// - /// #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] - /// struct Person { - /// id: u64, - /// name: String, - /// } - /// - /// #[derive(Debug)] - /// struct PersonRef<'a> { - /// id: u64, - /// name: &'a str, - /// } - /// - /// impl<'a> PartialEq for PersonRef<'a> { - /// fn eq(&self, other: &Self) -> bool { - /// self.id == other.id && self.name == other.name - /// } - /// } /// - /// impl<'a> Eq for PersonRef<'a> {} + /// ## Example /// - /// impl<'a> PartialOrd for PersonRef<'a> { - /// fn partial_cmp(&self, other: &Self) -> Option { - /// Some(self.cmp(other)) - /// } - /// } + /// Here are three examples of how flexible the `insert` method is: /// - /// impl<'a> Ord for PersonRef<'a> { - /// fn cmp(&self, other: &Self) -> cmp::Ordering { - /// self - /// .id - /// .cmp(&other.id) - /// .then_with(|| self.name.cmp(other.name)) - /// } - /// } + /// The `Person` struct implementation can be found [here](crate::swmr::generic::tests::Person). /// - /// impl Equivalent for PersonRef<'_> { - /// fn equivalent(&self, key: &Person) -> bool { - /// self.id == key.id && self.name == key.name - /// } - /// } + /// 1. **Inserting an owned key-value pair with structured key and value** /// - /// impl Comparable for PersonRef<'_> { - /// fn compare(&self, key: &Person) -> std::cmp::Ordering { - /// self.id.cmp(&key.id).then_with(|| self.name.cmp(&key.name)) - /// } - /// } + /// ```rust + /// use orderwal::swmr::{*, generic::*}; + /// # use orderwal::swmr::generic::tests::Person; /// - /// impl Equivalent> for Person { - /// fn equivalent(&self, key: &PersonRef<'_>) -> bool { - /// self.id == key.id && self.name == key.name - /// } - /// } + /// let person = Person { + /// id: 1, + /// name: "Alice".to_string(), + /// }; /// - /// impl Comparable> for Person { - /// fn compare(&self, key: &PersonRef<'_>) -> std::cmp::Ordering { - /// self - /// .id - /// .cmp(&key.id) - /// .then_with(|| self.name.as_str().cmp(key.name)) - /// } - /// } + /// let mut wal = GenericBuilder::new().with_capacity(1024).alloc::().unwrap(); + /// let value = "Hello, Alice!".to_string(); + /// wal.insert(&person, value); + /// ``` /// - /// impl<'a> KeyRef<'a, Person> for PersonRef<'a> { - /// fn compare(&self, a: &Q) -> cmp::Ordering - /// where - /// Q: ?Sized + Ord + Comparable, - /// { - /// Comparable::compare(a, self).reverse() - /// } + /// 2. **Inserting a key-value pair, key is a reference, value is owned** + /// + /// ```rust + /// use orderwal::swmr::{*, generic::*}; + /// # use orderwal::swmr::generic::tests::Person; /// - /// fn compare_binary(this: &[u8], other: &[u8]) -> cmp::Ordering { - /// let (this_id_size, this_id) = decode_u64_varint(this).unwrap(); - /// let (other_id_size, other_id) = decode_u64_varint(other).unwrap(); + /// let mut wal = GenericBuilder::new().with_capacity(1024).alloc::().unwrap(); /// - /// PersonRef { - /// id: this_id, - /// name: std::str::from_utf8(&this[this_id_size..]).unwrap(), - /// } - /// .cmp(&PersonRef { - /// id: other_id, - /// name: std::str::from_utf8(&other[other_id_size..]).unwrap(), - /// }) - /// } - /// } + /// let person = Person { + /// id: 1, + /// name: "Alice".to_string(), + /// }; /// - /// impl Type for Person { - /// type Ref<'a> = PersonRef<'a>; - /// type Error = EncodeVarintError; + /// wal.insert(&person, "value".to_string()); + /// ``` /// - /// fn encoded_len(&self) -> usize { - /// encoded_u64_varint_len(self.id) + self.name.len() - /// } + /// 3. **Inserting a key-value pair, both of them are in encoded format** /// - /// fn encode(&self, buf: &mut [u8]) -> Result<(), Self::Error> { - /// let id_size = encode_u64_varint(self.id, buf)?; - /// buf[id_size..].copy_from_slice(self.name.as_bytes()); - /// Ok(()) - /// } - /// } + /// ```rust + /// use orderwal::swmr::{*, generic::*}; + /// # use orderwal::swmr::generic::tests::Person; + /// + /// let mut wal = GenericBuilder::new().with_capacity(1024).alloc::().unwrap(); /// - /// impl<'a> TypeRef<'a> for PersonRef<'a> { - /// unsafe fn from_slice(src: &'a [u8]) -> Self { - /// let (id_size, id) = decode_u64_varint(src).unwrap(); - /// let name = std::str::from_utf8(&src[id_size..]).unwrap(); - /// PersonRef { id, name } - /// } - /// } + /// let person = Person { + /// id: 1, + /// name: "Alice".to_string(), + /// }.encode_into_vec(); /// - /// let mut wal = GenericOrderWal::::new(Options::new().with_capacity(1024)).unwrap(); /// - /// # let key = Person { - /// # id: 1, - /// # name: "Alice".to_string(), - /// # }; - /// - /// - /// # let mut person = vec![0; key.encoded_len()]; - /// # key.encode(&mut person).unwrap(); - /// - /// // Assume `person` comes from somewhere else, e.g. from the network. - /// unsafe { - /// wal.insert_bytes(person.as_ref(), b"Hello, Alice!").unwrap(); - /// } - /// ``` + /// unsafe { + /// let key = Generic::from_slice(person.as_ref()); + /// let value = Generic::from_slice("Hello, Alice!".as_bytes()); + /// wal.insert(key, value).unwrap(); + /// } + /// ``` + /// ### Inserting a key in bytes format #[inline] - pub unsafe fn insert_bytes(&mut self, key: &[u8], value: &[u8]) -> Result<(), Error> { - self - .insert_in(Among::Right(key), Among::Right(value)) - .map_err(|e| match e { - Among::Right(e) => e, - _ => unreachable!(), - }) + pub fn insert<'a>( + &mut self, + key: impl Into>, + val: impl Into>, + ) -> Result<(), Among> { + self.insert_in(key.into().into_among(), val.into().into_among()) } /// Inserts a key in bytes format and value in structured format into the write-ahead log directly. /// - /// # Safety + /// ## Safety /// - The given `key` must be valid to construct to `K::Ref` without remaining. /// - /// # Example + /// ## Example /// /// See [`insert_bytes`](GenericOrderWal::insert_bytes) for more details. #[inline] @@ -1429,6 +725,112 @@ where .map_err(Among::into_middle_right) } + /// Inserts a batch of entries into the write-ahead log. + pub fn insert_batch<'a, B: GenericBatch<'a, Key = K, Value = V>>( + &mut self, + batch: &mut B, + ) -> Result<(), Among> + { + let mut batch_encoded_size = 0u64; + + let mut num_entries = 0u32; + { + for ent in batch.iter_mut() { + let klen = ent.key.encoded_len(); + let vlen = ent.value.encoded_len(); + crate::utils::check_batch_entry( + klen, + vlen, + self.core.opts.maximum_key_size(), + self.core.opts.maximum_value_size(), + ) + .map_err(Among::Right)?; + let merged_len = merge_lengths(klen as u32, vlen as u32); + batch_encoded_size += klen as u64 + vlen as u64 + encoded_u64_varint_len(merged_len) as u64; + + num_entries += 1; + } + } + + // safe to cast batch_encoded_size to u32 here, we already checked it's less than capacity (less than u32::MAX). + let batch_meta = merge_lengths(num_entries, batch_encoded_size as u32); + let batch_meta_size = encoded_u64_varint_len(batch_meta); + let allocator = self.allocator(); + let remaining = allocator.remaining() as u64; + let total_size = + STATUS_SIZE as u64 + batch_meta_size as u64 + batch_encoded_size + CHECKSUM_SIZE as u64; + if total_size > remaining { + return Err(Among::Right(Error::insufficient_space( + total_size, + remaining as u32, + ))); + } + + let mut buf = allocator + .alloc_bytes(total_size as u32) + .map_err(|e| Among::Right(Error::from_insufficient_space(e)))?; + + unsafe { + let committed_flag = Flags::BATCHING | Flags::COMMITTED; + let mut cks = self.core.cks.build_checksumer(); + let flag = Flags::BATCHING; + buf.put_u8_unchecked(flag.bits); + buf.put_u64_varint_unchecked(batch_meta); + let mut cursor = 1 + batch_meta_size; + + for ent in batch.iter_mut() { + let klen = ent.key.encoded_len(); + let vlen = ent.value.encoded_len(); + let merged_kv_len = merge_lengths(klen as u32, vlen as u32); + let merged_kv_len_size = encoded_u64_varint_len(merged_kv_len); + + let remaining = buf.remaining(); + if remaining < merged_kv_len_size + klen + vlen { + return Err(Among::Right(Error::larger_batch_size(total_size as u32))); + } + + let ent_len_size = buf.put_u64_varint_unchecked(merged_kv_len); + let ko = cursor as usize + ent_len_size; + let ptr = buf.as_mut_ptr().add(ko); + + let key_buf = slice::from_raw_parts_mut(ptr, klen); + ent.key.encode(key_buf).map_err(Among::Left)?; + let value_buf = slice::from_raw_parts_mut(ptr.add(klen), vlen); + ent.value.encode(value_buf).map_err(Among::Middle)?; + + cursor += ent_len_size + klen + vlen; + // if let Some(pointer) = ent.pointer { + // *pointer = GenericPointer::new(klen, vlen, ptr); + // } + ent.pointer = Some(GenericPointer::new(klen, vlen, ptr)); + } + + if (cursor + CHECKSUM_SIZE) as u64 != total_size { + return Err(Among::Right(Error::batch_size_mismatch( + total_size as u32 - CHECKSUM_SIZE as u32, + cursor as u32, + ))); + } + + cks.update(&[committed_flag.bits]); + cks.update(&buf[1..]); + let checksum = cks.digest(); + buf.put_u64_le_unchecked(checksum); + + // commit the entry + buf[0] = committed_flag.bits; + let buf_cap = buf.capacity(); + + if self.core.opts.sync_on_write() && allocator.is_ondisk() { + allocator + .flush_range(buf.offset(), buf_cap) + .map_err(|e| Among::Right(e.into()))?; + } + buf.detach(); + Ok(()) + } + } + fn insert_in( &self, key: Among, @@ -1488,7 +890,7 @@ where } buf.detach(); - let p = Pointer::new(klen, vlen, buf.as_ptr().add(ko)); + let p = GenericPointer::new(klen, vlen, buf.as_ptr().add(ko)); self.core.map.insert(p); Ok(()) } diff --git a/src/swmr/generic/builder.rs b/src/swmr/generic/builder.rs new file mode 100644 index 0000000..712aac3 --- /dev/null +++ b/src/swmr/generic/builder.rs @@ -0,0 +1,588 @@ +use std::path::{Path, PathBuf}; + +use rarena_allocator::sync::Arena; + +use super::*; + +/// A write-ahead log builder. +pub struct GenericBuilder { + pub(super) opts: Options, + pub(super) cks: S, +} + +impl Default for GenericBuilder { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl GenericBuilder { + /// Returns a new write-ahead log builder with the given options. + #[inline] + pub fn new() -> Self { + Self { + opts: Options::default(), + cks: Crc32::default(), + } + } +} + +impl GenericBuilder { + /// Returns a new write-ahead log builder with the new checksumer + /// + /// ## Example + /// + /// ```rust + /// use orderwal::{GenericBuilder, Crc32}; + /// + /// let opts = GenericBuilder::new().with_checksumer(Crc32::new()); + /// ``` + #[inline] + pub fn with_checksumer(self, cks: NS) -> GenericBuilder { + GenericBuilder { + opts: self.opts, + cks, + } + } + + /// Returns a new write-ahead log builder with the new options + /// + /// ## Example + /// + /// ```rust + /// use orderwal::{GenericBuilder, Options}; + /// + /// let opts = GenericBuilder::new().with_options(Options::default()); + /// ``` + #[inline] + pub fn with_options(self, opts: Options) -> Self { + Self { + opts, + cks: self.cks, + } + } + + /// Set the reserved bytes of the WAL. + /// + /// The `reserved` is used to configure the start position of the WAL. This is useful + /// when you want to add some bytes as your own WAL's header. + /// + /// The default reserved is `0`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let opts = GenericBuilder::new().with_reserved(8); + /// ``` + #[inline] + pub const fn with_reserved(mut self, reserved: u32) -> Self { + self.opts = self.opts.with_reserved(reserved); + self + } + + /// Get the reserved of the WAL. + /// + /// The `reserved` is used to configure the start position of the WAL. This is useful + /// when you want to add some bytes as your own WAL's header. + /// + /// The default reserved is `0`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let opts = GenericBuilder::new().with_reserved(8); + /// + /// assert_eq!(opts.reserved(), 8); + /// ``` + #[inline] + pub const fn reserved(&self) -> u32 { + self.opts.reserved() + } + + /// Returns the magic version. + /// + /// The default value is `0`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_magic_version(1); + /// assert_eq!(options.magic_version(), 1); + /// ``` + #[inline] + pub const fn magic_version(&self) -> u16 { + self.opts.magic_version() + } + + /// Returns the capacity of the WAL. + /// + /// The default value is `0`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_capacity(1000); + /// assert_eq!(options.capacity(), 1000); + /// ``` + #[inline] + pub const fn capacity(&self) -> u32 { + self.opts.capacity() + } + + /// Returns the maximum key length. + /// + /// The default value is `u16::MAX`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_maximum_key_size(1024); + /// assert_eq!(options.maximum_key_size(), 1024); + /// ``` + #[inline] + pub const fn maximum_key_size(&self) -> u32 { + self.opts.maximum_key_size() + } + + /// Returns the maximum value length. + /// + /// The default value is `u32::MAX`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_maximum_value_size(1024); + /// assert_eq!(options.maximum_value_size(), 1024); + /// ``` + #[inline] + pub const fn maximum_value_size(&self) -> u32 { + self.opts.maximum_value_size() + } + + /// Returns `true` if the WAL syncs on write. + /// + /// The default value is `true`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new(); + /// assert_eq!(options.sync_on_write(), true); + /// ``` + #[inline] + pub const fn sync_on_write(&self) -> bool { + self.opts.sync_on_write() + } + + /// Returns the bits of the page size. + /// + /// Configures the anonymous memory map to be allocated using huge pages. + /// + /// This option corresponds to the `MAP_HUGETLB` flag on Linux. It has no effect on Windows. + /// + /// The size of the requested page can be specified in page bits. + /// If not provided, the system default is requested. + /// The requested length should be a multiple of this, or the mapping will fail. + /// + /// This option has no effect on file-backed memory maps. + /// + /// The default value is `None`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_huge(64); + /// assert_eq!(options.huge(), Some(64)); + /// ``` + #[inline] + pub const fn huge(&self) -> Option { + self.opts.huge() + } + + /// Sets the capacity of the WAL. + /// + /// This configuration will be ignored when using file-backed memory maps. + /// + /// The default value is `0`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_capacity(100); + /// assert_eq!(options.capacity(), 100); + /// ``` + #[inline] + pub const fn with_capacity(mut self, cap: u32) -> Self { + self.opts = self.opts.with_capacity(cap); + self + } + + /// Sets the maximum key length. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_maximum_key_size(1024); + /// assert_eq!(options.maximum_key_size(), 1024); + /// ``` + #[inline] + pub const fn with_maximum_key_size(mut self, size: u32) -> Self { + self.opts = self.opts.with_maximum_key_size(size); + self + } + + /// Sets the maximum value length. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_maximum_value_size(1024); + /// assert_eq!(options.maximum_value_size(), 1024); + /// ``` + #[inline] + pub const fn with_maximum_value_size(mut self, size: u32) -> Self { + self.opts = self.opts.with_maximum_value_size(size); + self + } + + /// Returns the bits of the page size. + /// + /// Configures the anonymous memory map to be allocated using huge pages. + /// + /// This option corresponds to the `MAP_HUGETLB` flag on Linux. It has no effect on Windows. + /// + /// The size of the requested page can be specified in page bits. + /// If not provided, the system default is requested. + /// The requested length should be a multiple of this, or the mapping will fail. + /// + /// This option has no effect on file-backed memory maps. + /// + /// The default value is `None`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_huge(64); + /// assert_eq!(options.huge(), Some(64)); + /// ``` + #[inline] + pub const fn with_huge(mut self, page_bits: u8) -> Self { + self.opts = self.opts.with_huge(page_bits); + self + } + + /// Sets the WAL to sync on write. + /// + /// The default value is `true`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_sync_on_write(false); + /// assert_eq!(options.sync_on_write(), false); + /// ``` + #[inline] + pub const fn with_sync_on_write(mut self, sync: bool) -> Self { + self.opts = self.opts.with_sync_on_write(sync); + self + } + + /// Sets the magic version. + /// + /// The default value is `0`. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::GenericBuilder; + /// + /// let options = GenericBuilder::new().with_magic_version(1); + /// assert_eq!(options.magic_version(), 1); + /// ``` + #[inline] + pub const fn with_magic_version(mut self, version: u16) -> Self { + self.opts = self.opts.with_magic_version(version); + self + } +} + +impl GenericBuilder { + /// Creates a new in-memory write-ahead log backed by an aligned vec with the given capacity and options. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::swmr::{GenericOrderWal, GenericBuilder}; + /// + /// let wal = GenericBuilder::new().with_capacity(1024).alloc::().unwrap(); + /// ``` + #[inline] + pub fn alloc(self) -> Result, Error> { + let Self { opts, cks } = self; + let arena = Arena::new(arena_options(opts.reserved()).with_capacity(opts.capacity())) + .map_err(Error::from_insufficient_space)?; + + GenericOrderWal::new_in(arena, opts, (), cks).map(GenericOrderWal::from_core) + } + + /// Creates a new in-memory write-ahead log backed by an anonymous memory map with the given options. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::swmr::{GenericOrderWal, GenericBuilder}; + /// + /// let wal = GenericBuilder::new().with_capacity(1024).map_anon::().unwrap(); + /// ``` + #[inline] + pub fn map_anon(self) -> Result, Error> { + let Self { opts, cks } = self; + let arena = Arena::map_anon( + arena_options(opts.reserved()), + MmapOptions::new().len(opts.capacity()), + )?; + + GenericOrderWal::new_in(arena, opts, (), cks).map(GenericOrderWal::from_core) + } + + /// Open a write-ahead log backed by a file backed memory map in read only mode. + /// + /// ## Safety + /// + /// All file-backed memory map constructors are marked `unsafe` because of the potential for + /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or + /// out of process. Applications must consider the risk and take appropriate precautions when + /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. + /// unlinked) files exist but are platform specific and limited. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::{swmr::{GenericOrderWal, GenericBuilder, generic::*}, OpenOptions}; + /// # let dir = tempfile::tempdir().unwrap(); + /// # let path = dir + /// # .path() + /// # .join("generic_wal_map_mut_with_checksumer"); + /// # + /// # let mut wal = unsafe { + /// # GenericBuilder::new() + /// # .map_mut::( + /// # &path, + /// # OpenOptions::new() + /// # .create_new(Some(1024)) + /// # .write(true) + /// # .read(true), + /// # ) + /// # .unwrap() + /// # }; + /// + /// let reader = unsafe { GenericBuilder::new().map::(path).unwrap() }; + /// ``` + #[inline] + pub unsafe fn map>(self, path: P) -> Result, Error> + where + K: Type + Ord + 'static, + for<'a> K::Ref<'a>: KeyRef<'a, K>, + V: 'static, + { + self + .map_with_path_builder::(|| dummy_path_builder(path)) + .map_err(|e| e.unwrap_right()) + } + + /// Open a write-ahead log backed by a file backed memory map in read only mode with the given [`Checksumer`]. + /// + /// ## Safety + /// + /// All file-backed memory map constructors are marked `unsafe` because of the potential for + /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or + /// out of process. Applications must consider the risk and take appropriate precautions when + /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. + /// unlinked) files exist but are platform specific and limited. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::{swmr::{GenericOrderWal, GenericBuilder, generic::*}, OpenOptions}; + /// # let dir = tempfile::tempdir().unwrap(); + /// # let path = dir + /// # .path() + /// # .join("generic_wal_map_mut_with_checksumer"); + /// # + /// # let mut wal = unsafe { + /// # GenericBuilder::new() + /// # .map_mut::( + /// # &path, + /// # OpenOptions::new() + /// # .create_new(Some(1024)) + /// # .write(true) + /// # .read(true), + /// # ) + /// # .unwrap() + /// # }; + /// + /// let reader = unsafe { GenericBuilder::new().map_with_path_builder::(|| Ok(path)).unwrap() }; + /// ``` + #[inline] + pub unsafe fn map_with_path_builder( + self, + path_builder: PB, + ) -> Result, Either> + where + K: Type + Ord + 'static, + for<'a> K::Ref<'a>: KeyRef<'a, K>, + V: 'static, + PB: FnOnce() -> Result, + { + let Self { cks, opts } = self; + let open_options = OpenOptions::default().read(true); + let arena = Arena::map_with_path_builder( + path_builder, + arena_options(opts.reserved()), + open_options, + MmapOptions::new(), + ) + .map_err(|e| e.map_right(Into::into))?; + + GenericOrderWal::replay(arena, opts, true, (), cks) + .map(|core| GenericWalReader::new(Arc::new(core))) + .map_err(Either::Right) + } + + /// Creates a new write-ahead log backed by a file backed memory map with the given options. + /// + /// ## Safety + /// + /// All file-backed memory map constructors are marked `unsafe` because of the potential for + /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or + /// out of process. Applications must consider the risk and take appropriate precautions when + /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. + /// unlinked) files exist but are platform specific and limited. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::{swmr::{GenericOrderWal, GenericBuilder generic::*}, OpenOptions}; + /// + /// # let dir = tempfile::tempdir().unwrap(); + /// # let path = dir + /// # .path() + /// # .join("generic_wal_map_mut"); + /// + /// let mut wal = unsafe { + /// GenericBuilder::new().map_mut::( + /// &path, + /// OpenOptions::new() + /// .create_new(Some(1024)) + /// .write(true) + /// .read(true), + /// ) + /// .unwrap() + /// }; + /// ``` + #[inline] + pub unsafe fn map_mut>( + self, + path: P, + open_options: OpenOptions, + ) -> Result, Error> + where + K: Type + Ord + 'static, + for<'a> K::Ref<'a>: KeyRef<'a, K>, + V: 'static, + { + self + .map_mut_with_path_builder::(|| dummy_path_builder(path), open_options) + .map_err(|e| e.unwrap_right()) + } + + /// Returns a write-ahead log backed by a file backed memory map with the given options and [`Checksumer`]. + /// + /// ## Safety + /// + /// All file-backed memory map constructors are marked `unsafe` because of the potential for + /// *Undefined Behavior* (UB) using the map if the underlying file is subsequently modified, in or + /// out of process. Applications must consider the risk and take appropriate precautions when + /// using file-backed maps. Solutions such as file permissions, locks or process-private (e.g. + /// unlinked) files exist but are platform specific and limited. + /// + /// ## Example + /// + /// ```rust + /// use orderwal::{swmr::{GenericOrderWal, GenericBuilder, generic::*}, OpenOptions}; + /// + /// let dir = tempfile::tempdir().unwrap(); + /// + /// let mut wal = unsafe { + /// GenericBuilder::new().map_mut_with_path_builder::<( + /// || { + /// Ok(dir.path().join("generic_wal_map_mut_with_path_builder_and_checksumer")) + /// }, + /// OpenOptions::new() + /// .create_new(Some(1024)) + /// .write(true) + /// .read(true), + /// ) + /// .unwrap() + /// }; + /// ``` + pub unsafe fn map_mut_with_path_builder( + self, + path_builder: PB, + open_options: OpenOptions, + ) -> Result, Either> + where + K: Type + Ord + 'static, + for<'a> K::Ref<'a>: KeyRef<'a, K>, + V: 'static, + PB: FnOnce() -> Result, + { + let Self { opts, cks } = self; + let path = path_builder().map_err(Either::Left)?; + let exist = path.exists(); + let arena = Arena::map_mut_with_path_builder( + || Ok(path), + arena_options(opts.reserved()), + open_options, + MmapOptions::new(), + ) + .map_err(|e| e.map_right(Into::into))?; + + if !exist { + return GenericOrderWal::new_in(arena, opts, (), cks) + .map(GenericOrderWal::from_core) + .map_err(Either::Right); + } + + GenericOrderWal::replay(arena, opts, false, (), cks) + .map(GenericOrderWal::from_core) + .map_err(Either::Right) + } +} diff --git a/src/swmr/generic/entry.rs b/src/swmr/generic/entry.rs deleted file mode 100644 index dd98c62..0000000 --- a/src/swmr/generic/entry.rs +++ /dev/null @@ -1,59 +0,0 @@ -use crossbeam_skiplist::set::Entry as SetEntry; - -use super::{Pointer, Type, TypeRef}; - -/// The reference to an entry in the [`GenericOrderWal`](super::GenericOrderWal). -pub struct EntryRef<'a, K, V> { - ent: SetEntry<'a, Pointer>, -} - -impl<'a, K, V> core::fmt::Debug for EntryRef<'a, K, V> -where - K: Type, - K::Ref<'a>: core::fmt::Debug, - V: Type, - V::Ref<'a>: core::fmt::Debug, -{ - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("EntryRef") - .field("key", &self.key()) - .field("value", &self.value()) - .finish() - } -} - -impl Clone for EntryRef<'_, K, V> { - #[inline] - fn clone(&self) -> Self { - Self { - ent: self.ent.clone(), - } - } -} - -impl<'a, K, V> EntryRef<'a, K, V> { - #[inline] - pub(super) fn new(ent: SetEntry<'a, Pointer>) -> Self { - Self { ent } - } -} - -impl<'a, K, V> EntryRef<'a, K, V> -where - K: Type, - V: Type, -{ - /// Returns the key of the entry. - #[inline] - pub fn key(&self) -> K::Ref<'a> { - let p = self.ent.value(); - unsafe { TypeRef::from_slice(p.as_key_slice()) } - } - - /// Returns the value of the entry. - #[inline] - pub fn value(&self) -> V::Ref<'a> { - let p = self.ent.value(); - unsafe { TypeRef::from_slice(p.as_value_slice()) } - } -} diff --git a/src/swmr/generic/iter.rs b/src/swmr/generic/iter.rs index 9093870..477ba6b 100644 --- a/src/swmr/generic/iter.rs +++ b/src/swmr/generic/iter.rs @@ -2,7 +2,7 @@ use core::ops::Bound; use crossbeam_skiplist::Comparable; -use super::{EntryRef, KeyRef, Owned, Pointer, Ref, Type}; +use super::{GenericEntryRef, GenericPointer as Pointer, KeyRef, Owned, Ref, Type}; type SetRefRange<'a, Q, K, V> = crossbeam_skiplist::set::Range< 'a, @@ -34,11 +34,11 @@ where K: Type + Ord, for<'b> K::Ref<'b>: KeyRef<'b, K>, { - type Item = EntryRef<'a, K, V>; + type Item = GenericEntryRef<'a, K, V>; #[inline] fn next(&mut self) -> Option { - self.iter.next().map(|ptr| EntryRef::new(ptr)) + self.iter.next().map(|ptr| GenericEntryRef::new(ptr)) } #[inline] @@ -54,7 +54,7 @@ where { #[inline] fn next_back(&mut self) -> Option { - self.iter.next_back().map(|ptr| EntryRef::new(ptr)) + self.iter.next_back().map(|ptr| GenericEntryRef::new(ptr)) } } @@ -86,11 +86,11 @@ where for<'b> K::Ref<'b>: KeyRef<'b, K>, Q: Ord + ?Sized + Comparable>, { - type Item = EntryRef<'a, K, V>; + type Item = GenericEntryRef<'a, K, V>; #[inline] fn next(&mut self) -> Option { - self.iter.next().map(|ptr| EntryRef::new(ptr)) + self.iter.next().map(|ptr| GenericEntryRef::new(ptr)) } } @@ -102,7 +102,7 @@ where { #[inline] fn next_back(&mut self) -> Option { - self.iter.next_back().map(|ptr| EntryRef::new(ptr)) + self.iter.next_back().map(|ptr| GenericEntryRef::new(ptr)) } } @@ -134,11 +134,11 @@ where for<'b> K::Ref<'b>: KeyRef<'b, K>, Q: Ord + ?Sized + Comparable> + Comparable, { - type Item = EntryRef<'a, K, V>; + type Item = GenericEntryRef<'a, K, V>; #[inline] fn next(&mut self) -> Option { - self.iter.next().map(|ptr| EntryRef::new(ptr)) + self.iter.next().map(|ptr| GenericEntryRef::new(ptr)) } } @@ -150,6 +150,6 @@ where { #[inline] fn next_back(&mut self) -> Option { - self.iter.next_back().map(|ptr| EntryRef::new(ptr)) + self.iter.next_back().map(|ptr| GenericEntryRef::new(ptr)) } } diff --git a/src/swmr/generic/reader.rs b/src/swmr/generic/reader.rs index 7801130..8e385bb 100644 --- a/src/swmr/generic/reader.rs +++ b/src/swmr/generic/reader.rs @@ -1,4 +1,12 @@ -use super::*; +use core::ops::Bound; +use std::sync::Arc; + +use dbutils::equivalent::Comparable; +use rarena_allocator::Allocator; + +use super::{ + GenericEntryRef, GenericOrderWalCore, Iter, KeyRef, Range, RefRange, Type, HEADER_SIZE, +}; /// A read-only view of a generic single-writer, multi-reader WAL. pub struct GenericWalReader(Arc>); @@ -22,7 +30,7 @@ impl GenericWalReader { /// Returns the reserved space in the WAL. /// - /// # Safety + /// ## Safety /// - The writer must ensure that the returned slice is not modified. /// - This method is not thread-safe, so be careful when using it. #[inline] @@ -54,13 +62,13 @@ where { /// Returns the first key-value pair in the map. The key in this pair is the minimum key in the wal. #[inline] - pub fn first(&self) -> Option> { + pub fn first(&self) -> Option> { self.0.first() } /// Returns the last key-value pair in the map. The key in this pair is the maximum key in the wal. #[inline] - pub fn last(&self) -> Option> { + pub fn last(&self) -> Option> { self.0.last() } @@ -123,7 +131,7 @@ where /// Returns `true` if the key exists in the WAL. /// - /// # Safety + /// ## Safety /// - The given `key` must be valid to construct to `K::Ref` without remaining. #[inline] pub unsafe fn contains_key_by_bytes(&self, key: &[u8]) -> bool { @@ -132,7 +140,7 @@ where /// Gets the value associated with the key. #[inline] - pub fn get<'a, Q>(&'a self, key: &'a Q) -> Option> + pub fn get<'a, Q>(&'a self, key: &'a Q) -> Option> where Q: ?Sized + Ord + Comparable> + Comparable, { @@ -141,7 +149,7 @@ where /// Gets the value associated with the key. #[inline] - pub fn get_by_ref<'a, Q>(&'a self, key: &'a Q) -> Option> + pub fn get_by_ref<'a, Q>(&'a self, key: &'a Q) -> Option> where Q: ?Sized + Ord + Comparable>, { @@ -150,10 +158,10 @@ where /// Gets the value associated with the key. /// - /// # Safety + /// ## Safety /// - The given `key` must be valid to construct to `K::Ref` without remaining. #[inline] - pub unsafe fn get_by_bytes(&self, key: &[u8]) -> Option> { + pub unsafe fn get_by_bytes(&self, key: &[u8]) -> Option> { self.0.get_by_bytes(key) } } diff --git a/src/swmr/generic/tests.rs b/src/swmr/generic/tests.rs index 3f3da25..d5ec429 100644 --- a/src/swmr/generic/tests.rs +++ b/src/swmr/generic/tests.rs @@ -20,27 +20,33 @@ mod iters; #[cfg(all(test, any(test_swmr_generic_get, all_tests)))] mod get; +#[doc(hidden)] #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Arbitrary)] -struct Person { - id: u64, - name: String, +pub struct Person { + #[doc(hidden)] + pub id: u64, + #[doc(hidden)] + pub name: String, } impl Person { - fn random() -> Self { + #[doc(hidden)] + pub fn random() -> Self { Self { id: rand::random(), name: names::Generator::default().next().unwrap(), } } - fn as_ref(&self) -> PersonRef<'_> { + #[doc(hidden)] + pub fn as_ref(&self) -> PersonRef<'_> { PersonRef { id: self.id, name: &self.name, } } + #[doc(hidden)] fn to_vec(&self) -> Vec { let mut buf = vec![0; self.encoded_len()]; self.encode(&mut buf).unwrap(); @@ -48,30 +54,13 @@ impl Person { } } +#[doc(hidden)] #[derive(Debug)] -struct PersonRef<'a> { +pub struct PersonRef<'a> { id: u64, name: &'a str, } -impl PersonRef<'_> { - fn encoded_len(&self) -> usize { - encoded_u64_varint_len(self.id) + self.name.len() - } - - fn encode(&self, buf: &mut [u8]) -> Result<(), dbutils::leb128::EncodeVarintError> { - let id_size = encode_u64_varint(self.id, buf)?; - buf[id_size..].copy_from_slice(self.name.as_bytes()); - Ok(()) - } - - fn to_vec(&self) -> Vec { - let mut buf = vec![0; self.encoded_len()]; - self.encode(&mut buf).unwrap(); - buf - } -} - impl PartialEq for PersonRef<'_> { fn eq(&self, other: &Self) -> bool { self.id == other.id && self.name == other.name @@ -167,3 +156,73 @@ impl<'a> TypeRef<'a> for PersonRef<'a> { PersonRef { id, name } } } + +fn insert_batch(wal: &mut GenericOrderWal) -> Vec<(Person, String)> { + const N: u32 = 100; + + let mut batch = vec![]; + let output = (0..N).map(|i| ({ let mut p = Person::random(); p.id = i as u64; p }, format!("My id is {i}")).clone()).collect::>(); + + for (person, val) in output.iter() { + if person.id % 3 == 0 { + batch.push(GenericEntry::new(person.clone(), val.clone())); + } else if person.id % 3 == 1 { + batch.push(GenericEntry::new(person, val)); + } else { + unsafe { batch.push(GenericEntry::new(person, Generic::from_slice(val.as_bytes()))); } + } + } + + wal.insert_batch(&mut batch).unwrap(); + + for (p, val) in output.iter() { + assert_eq!(wal.get(p).unwrap().value(), val); + } + + let wal = wal.reader(); + for (p, val) in output.iter() { + assert_eq!(wal.get(p).unwrap().value(), val); + } + + // output + vec![] +} + +#[test] +fn test_insert_batch_inmemory() { + insert_batch(&mut GenericBuilder::new().with_capacity(MB).alloc::().unwrap()); +} + +#[test] +fn test_insert_batch_map_anon() { + insert_batch(&mut GenericBuilder::new().with_capacity(MB).map_anon::().unwrap()); +} + +#[test] +#[cfg_attr(miri, ignore)] +fn test_insert_batch_map_file() { + let dir = ::tempfile::tempdir().unwrap(); + let path = dir.path().join(concat!( + "test_", + stringify!($prefix), + "_insert_batch_map_file" + )); + let mut map = unsafe { + GenericBuilder::new().map_mut::( + &path, + OpenOptions::new() + .create_new(Some(MB)) + .write(true) + .read(true), + ) + .unwrap() + }; + + insert_batch(&mut map); + + let map = unsafe { GenericBuilder::new().map::(&path).unwrap() }; + + for i in 0..100u32 { + assert_eq!(map.get(&i.to_be_bytes()).unwrap(), i.to_be_bytes()); + } +} diff --git a/src/swmr/generic/tests/constructor.rs b/src/swmr/generic/tests/constructor.rs index 6ddd1b1..25f9e6e 100644 --- a/src/swmr/generic/tests/constructor.rs +++ b/src/swmr/generic/tests/constructor.rs @@ -1,6 +1,7 @@ use super::*; #[test] +#[allow(clippy::needless_borrows_for_generic_args)] fn owned_comparable() { let p1 = Person { id: 3127022870678870148, @@ -14,8 +15,8 @@ fn owned_comparable() { let p1bytes = p1.to_vec(); let p2bytes = p2.to_vec(); - let ptr1 = Pointer::::new(p1bytes.len(), 0, p1bytes.as_ptr()); - let ptr2 = Pointer::::new(p2bytes.len(), 0, p2bytes.as_ptr()); + let ptr1 = GenericPointer::::new(p1bytes.len(), 0, p1bytes.as_ptr()); + let ptr2 = GenericPointer::::new(p2bytes.len(), 0, p2bytes.as_ptr()); let map = SkipSet::new(); map.insert(ptr1); @@ -27,7 +28,7 @@ fn owned_comparable() { assert!(map.contains(&Owned::new(&p2))); assert!(map.get(&Owned::new(&p2)).is_some()); - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); wal.insert(&p1, &"My name is Alice!".to_string()).unwrap(); wal.insert(&p2, &"My name is Bob!".to_string()).unwrap(); @@ -52,8 +53,8 @@ fn ref_comparable() { let p1bytes = p1.to_vec(); let p2bytes = p2.to_vec(); - let ptr1 = Pointer::::new(p1bytes.len(), 0, p1bytes.as_ptr()); - let ptr2 = Pointer::::new(p2bytes.len(), 0, p2bytes.as_ptr()); + let ptr1 = GenericPointer::::new(p1bytes.len(), 0, p1bytes.as_ptr()); + let ptr2 = GenericPointer::::new(p2bytes.len(), 0, p2bytes.as_ptr()); let map = SkipSet::new(); map.insert(ptr1); @@ -65,7 +66,7 @@ fn ref_comparable() { assert!(map.contains(&Owned::new(&p2))); assert!(map.get(&Owned::new(&p2)).is_some()); - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); unsafe { wal @@ -98,8 +99,9 @@ fn ref_comparable() { } #[test] +#[allow(clippy::needless_borrows_for_generic_args)] fn construct_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); let person = Person { id: 1, @@ -119,10 +121,9 @@ fn construct_inmemory() { } #[test] +#[allow(clippy::needless_borrows_for_generic_args)] fn construct_map_anon() { - let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); - + let mut wal = GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); let person = Person { id: 1, name: "Alice".to_string(), @@ -135,14 +136,14 @@ fn construct_map_anon() { #[test] #[cfg_attr(miri, ignore)] +#[allow(clippy::needless_borrows_for_generic_args)] fn construct_map_file() { let dir = tempdir().unwrap(); let path = dir.path().join("generic_wal_construct_map_file"); unsafe { - let mut wal = GenericOrderWal::::map_mut( + let mut wal = GenericBuilder::new().map_mut::( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -168,22 +169,21 @@ fn construct_map_file() { }; unsafe { - let wal = GenericOrderWal::::map_mut( + let wal = GenericBuilder::new().map_mut::( &path, - Options::new(), OpenOptions::new().create(Some(MB)).write(true).read(true), ) .unwrap(); assert_eq!(wal.get(&pr).unwrap().value(), "My name is Alice!"); } - let wal = unsafe { GenericOrderWal::::map(&path, Options::new()).unwrap() }; + let wal = unsafe { GenericBuilder::new().map::(&path).unwrap() }; assert_eq!(wal.get(&pr).unwrap().value(), "My name is Alice!"); } #[test] fn construct_with_small_capacity_inmemory() { - let wal = GenericOrderWal::::new(Options::new().with_capacity(1)); + let wal = GenericBuilder::new().with_capacity(1).alloc::(); assert!(wal.is_err()); match wal { @@ -194,7 +194,7 @@ fn construct_with_small_capacity_inmemory() { #[test] fn construct_with_small_capacity_map_anon() { - let wal = GenericOrderWal::::map_anon(Options::new().with_capacity(1)); + let wal = GenericBuilder::new().with_capacity(1).map_anon::(); assert!(wal.is_err()); match wal { @@ -211,9 +211,8 @@ fn construct_with_small_capacity_map_file() { .join("generic_wal_construct_with_small_capacity_map_file"); let wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut::( &path, - Options::new(), OpenOptions::new() .create_new(Some(1)) .write(true) @@ -240,14 +239,14 @@ fn zero_reserved(wal: &mut GenericOrderWal) { #[test] fn zero_reserved_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); zero_reserved(&mut wal); } #[test] fn zero_reserved_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); zero_reserved(&mut wal); } @@ -258,9 +257,8 @@ fn zero_reserved_map_file() { let path = dir.path().join("generic_wal_zero_reserved_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( - &path, - Options::new(), + GenericBuilder::new().map_mut::( + &path, OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -286,17 +284,15 @@ fn reserved(wal: &mut GenericOrderWal) { #[test] fn reserved_inmemory() { - let mut wal = - GenericOrderWal::::new(Options::new().with_capacity(MB).with_reserved(4)) - .unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).with_reserved(4).alloc() + .unwrap(); reserved(&mut wal); } #[test] fn reserved_map_anon() { - let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB).with_reserved(4)) - .unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).with_reserved(4).map_anon() + .unwrap(); reserved(&mut wal); } @@ -307,9 +303,8 @@ fn reserved_map_file() { let path = dir.path().join("generic_wal_reserved_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().with_reserved(4).map_mut::( &path, - Options::new().with_reserved(4), OpenOptions::new() .create_new(Some(MB)) .write(true) diff --git a/src/swmr/generic/tests/get.rs b/src/swmr/generic/tests/get.rs index c6df958..a4a3c9a 100644 --- a/src/swmr/generic/tests/get.rs +++ b/src/swmr/generic/tests/get.rs @@ -25,14 +25,14 @@ fn first(wal: &mut GenericOrderWal) { #[test] fn first_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); first(&mut wal); } #[test] fn first_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); first(&mut wal); } @@ -43,9 +43,8 @@ fn first_map_file() { let path = dir.path().join("generic_wal_first_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -81,14 +80,14 @@ fn last(wal: &mut GenericOrderWal) { #[test] fn last_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); last(&mut wal); } #[test] fn last_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); last(&mut wal); } @@ -99,9 +98,8 @@ fn last_map_file() { let path = dir.path().join("generic_wal_last_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -113,6 +111,7 @@ fn last_map_file() { last(&mut wal); } +#[allow(clippy::needless_borrows_for_generic_args)] fn get_or_insert(wal: &mut GenericOrderWal) { let people = (0..100) .map(|_| { @@ -128,6 +127,7 @@ fn get_or_insert(wal: &mut GenericOrderWal) { for (p, pv) in &people { assert!(wal.contains_key(p)); assert!(wal.contains_key_by_ref(&p.as_ref())); + assert_eq!( wal .get_or_insert(p, &format!("Hello! {}!", p.name)) @@ -145,14 +145,14 @@ fn get_or_insert(wal: &mut GenericOrderWal) { #[test] fn get_or_insert_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); get_or_insert(&mut wal); } #[test] fn get_or_insert_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); get_or_insert(&mut wal); } @@ -163,9 +163,8 @@ fn get_or_insert_map_file() { let path = dir.path().join("generic_wal_get_or_insert_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -183,7 +182,7 @@ fn get_or_insert_with(wal: &mut GenericOrderWal) { let p = Person::random(); let v = format!("My name is {}", p.name); wal - .get_or_insert_with(&p, || v.clone()) + .get_or_insert_with(&p, || v.clone().into()) .unwrap_right() .unwrap(); (p, v) @@ -197,7 +196,7 @@ fn get_or_insert_with(wal: &mut GenericOrderWal) { assert!(wal.contains_key_by_ref(&p.as_ref())); assert_eq!( wal - .get_or_insert_with(p, || format!("Hello! {}!", p.name)) + .get_or_insert_with(p, || format!("Hello! {}!", p.name).into()) .unwrap_left() .value(), pv @@ -212,14 +211,14 @@ fn get_or_insert_with(wal: &mut GenericOrderWal) { #[test] fn get_or_insert_with_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); get_or_insert_with(&mut wal); } #[test] fn get_or_insert_with_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); get_or_insert_with(&mut wal); } @@ -230,9 +229,8 @@ fn get_or_insert_with_map_file() { let path = dir.path().join("generic_wal_get_or_insert_with_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -244,6 +242,7 @@ fn get_or_insert_with_map_file() { get_or_insert_with(&mut wal); } +#[allow(clippy::needless_borrows_for_generic_args)] fn get_or_insert_key_with_value_bytes(wal: &mut GenericOrderWal) { let people = (0..100) .map(|_| { @@ -252,7 +251,7 @@ fn get_or_insert_key_with_value_bytes(wal: &mut GenericOrderWal) let v = format!("My name is {}", p.name); unsafe { wal - .get_by_bytes_or_insert(pvec.as_ref(), &v) + .get_or_insert(Generic::from_slice(pvec.as_ref()), &v) .unwrap_right() .unwrap(); } @@ -265,6 +264,7 @@ fn get_or_insert_key_with_value_bytes(wal: &mut GenericOrderWal) for (p, pv) in &people { assert!(wal.contains_key(p)); assert!(wal.contains_key_by_ref(&p.as_ref())); + assert_eq!( wal .get_or_insert(p, &format!("Hello! {}!", p.name)) @@ -282,14 +282,14 @@ fn get_or_insert_key_with_value_bytes(wal: &mut GenericOrderWal) #[test] fn get_or_insert_key_with_value_bytes_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); get_or_insert_key_with_value_bytes(&mut wal); } #[test] fn get_or_insert_key_with_value_bytes_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); get_or_insert_key_with_value_bytes(&mut wal); } @@ -302,9 +302,8 @@ fn get_or_insert_key_with_value_bytes_map_file() { .join("generic_wal_get_or_insert_key_with_value_bytes_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -323,7 +322,7 @@ fn get_or_insert_value_bytes(wal: &mut GenericOrderWal) { let v = format!("My name is {}", p.name); unsafe { wal - .get_or_insert_bytes(&p, v.as_bytes()) + .get_or_insert(&p, Generic::from_slice(v.as_bytes())) .unwrap_right() .unwrap(); } @@ -339,7 +338,7 @@ fn get_or_insert_value_bytes(wal: &mut GenericOrderWal) { unsafe { assert_eq!( wal - .get_or_insert_bytes(p, pv.as_bytes()) + .get_or_insert(p, Generic::from_slice(pv.as_bytes())) .unwrap_left() .value(), pv @@ -355,14 +354,14 @@ fn get_or_insert_value_bytes(wal: &mut GenericOrderWal) { #[test] fn get_or_insert_value_bytes_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); get_or_insert_value_bytes(&mut wal); } #[test] fn get_or_insert_value_bytes_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); get_or_insert_value_bytes(&mut wal); } @@ -375,9 +374,8 @@ fn get_or_insert_value_bytes_map_file() { .join("generic_wal_get_or_insert_value_bytes_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -397,7 +395,7 @@ fn get_by_bytes_or_insert_with(wal: &mut GenericOrderWal) { let v = format!("My name is {}", p.name); unsafe { wal - .get_by_bytes_or_insert_with(pvec.as_ref(), || v.clone()) + .get_or_insert_with(Generic::from_slice(pvec.as_ref()), || v.clone().into()) .unwrap_right() .unwrap(); } @@ -413,7 +411,7 @@ fn get_by_bytes_or_insert_with(wal: &mut GenericOrderWal) { unsafe { assert_eq!( wal - .get_by_bytes_or_insert_with(pvec, || format!("Hello! {}!", p.name)) + .get_or_insert_with(Generic::from_slice(pvec), || format!("Hello! {}!", p.name).into()) .unwrap_left() .value(), pv @@ -429,14 +427,14 @@ fn get_by_bytes_or_insert_with(wal: &mut GenericOrderWal) { #[test] fn get_by_bytes_or_insert_with_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); get_by_bytes_or_insert_with(&mut wal); } #[test] fn get_by_bytes_or_insert_with_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); get_by_bytes_or_insert_with(&mut wal); } @@ -449,9 +447,8 @@ fn get_by_bytes_or_insert_with_map_file() { .join("generic_wal_get_by_bytes_or_insert_with_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -471,7 +468,7 @@ fn get_by_bytes_or_insert_bytes(wal: &mut GenericOrderWal) { let v = format!("My name is {}", p.name); unsafe { wal - .get_by_bytes_or_insert_bytes(pvec.as_ref(), v.as_bytes()) + .get_or_insert(Generic::from_slice(pvec.as_ref()), Generic::from_slice(v.as_bytes())) .unwrap_right() .unwrap(); } @@ -487,7 +484,7 @@ fn get_by_bytes_or_insert_bytes(wal: &mut GenericOrderWal) { unsafe { assert_eq!( wal - .get_by_bytes_or_insert_bytes(pvec, pv.as_bytes()) + .get_or_insert(Generic::from_slice(pvec), Generic::from_slice(pv.as_bytes())) .unwrap_left() .value(), pv @@ -503,14 +500,14 @@ fn get_by_bytes_or_insert_bytes(wal: &mut GenericOrderWal) { #[test] fn get_by_bytes_or_insert_bytes_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); get_by_bytes_or_insert_bytes(&mut wal); } #[test] fn get_by_bytes_or_insert_bytes_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); get_by_bytes_or_insert_bytes(&mut wal); } @@ -523,9 +520,8 @@ fn get_by_bytes_or_insert_bytes_map_file() { .join("generic_wal_get_by_bytes_or_insert_bytes_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( - &path, - Options::new(), + GenericBuilder::new().map_mut( + &path, OpenOptions::new() .create_new(Some(MB)) .write(true) diff --git a/src/swmr/generic/tests/insert.rs b/src/swmr/generic/tests/insert.rs index 62713d3..79436bd 100644 --- a/src/swmr/generic/tests/insert.rs +++ b/src/swmr/generic/tests/insert.rs @@ -4,6 +4,7 @@ fn insert_to_full(wal: &mut GenericOrderWal) { let mut full = false; for _ in 0u32.. { let p = Person::random(); + #[allow(clippy::needless_borrows_for_generic_args)] match wal.insert(&p, &format!("My name is {}", p.name)) { Ok(_) => {} Err(e) => match e { @@ -20,14 +21,14 @@ fn insert_to_full(wal: &mut GenericOrderWal) { #[test] fn insert_to_full_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(100)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(100).alloc().unwrap(); insert_to_full(&mut wal); } #[test] fn insert_to_full_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(100)).unwrap(); + GenericBuilder::new().with_capacity(100).map_anon().unwrap(); insert_to_full(&mut wal); } @@ -38,9 +39,8 @@ fn insert_to_full_map_file() { let path = dir.path().join("generic_wal_insert_to_full_map_file"); unsafe { - let mut wal = GenericOrderWal::::map_mut( + let mut wal = GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(100)) .write(true) @@ -55,6 +55,7 @@ fn insert(wal: &mut GenericOrderWal) -> Vec { let people = (0..100) .map(|_| { let p = Person::random(); + #[allow(clippy::needless_borrows_for_generic_args)] wal.insert(&p, &format!("My name is {}", p.name)).unwrap(); p }) @@ -76,14 +77,14 @@ fn insert(wal: &mut GenericOrderWal) -> Vec { #[test] fn insert_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); insert(&mut wal); } #[test] fn insert_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); insert(&mut wal); } @@ -94,9 +95,8 @@ fn insert_map_file() { let path = dir.path().join("generic_wal_insert_map_file"); let people = unsafe { - let mut wal = GenericOrderWal::::map_mut( + let mut wal = GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -106,7 +106,7 @@ fn insert_map_file() { insert(&mut wal) }; - let wal = unsafe { GenericOrderWal::::map(&path, Options::new()).unwrap() }; + let wal = unsafe { GenericBuilder::new().map::(&path).unwrap() }; for p in people { assert!(wal.contains_key(&p)); @@ -157,14 +157,14 @@ fn insert_key_bytes_with_value( #[test] fn insert_key_bytes_with_value_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); insert_key_bytes_with_value(&mut wal); } #[test] fn insert_key_bytes_with_value_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); insert_key_bytes_with_value(&mut wal); } @@ -177,9 +177,8 @@ fn insert_key_bytes_with_value_map_file() { .join("generic_wal_insert_key_bytes_with_value_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -206,7 +205,7 @@ fn insert_key_bytes_with_value_map_file() { ); } - let wal = unsafe { GenericOrderWal::::map(&path, Options::new()).unwrap() }; + let wal = unsafe { GenericBuilder::new().map::(&path,).unwrap() }; for (pbytes, p) in people { assert!(wal.contains_key(&p)); @@ -230,7 +229,7 @@ fn insert_key_with_value_bytes(wal: &mut GenericOrderWal) -> Vec let p = Person::random(); unsafe { wal - .insert_key_with_value_bytes(&p, format!("My name is {}", p.name).as_bytes()) + .insert(&p, Generic::from_slice(format!("My name is {}", p.name).as_bytes())) .unwrap(); } p @@ -253,14 +252,14 @@ fn insert_key_with_value_bytes(wal: &mut GenericOrderWal) -> Vec #[test] fn insert_key_with_value_bytes_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); insert_key_with_value_bytes(&mut wal); } #[test] fn insert_key_with_value_bytes_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); insert_key_with_value_bytes(&mut wal); } @@ -273,9 +272,9 @@ fn insert_key_with_value_bytes_map_file() { .join("generic_wal_insert_key_with_value_bytes_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), + OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -304,7 +303,7 @@ fn insert_bytes(wal: &mut GenericOrderWal) -> Vec { let pbytes = p.to_vec(); unsafe { wal - .insert_bytes(&pbytes, format!("My name is {}", p.name).as_bytes()) + .insert(Generic::from_slice(&pbytes), Generic::from_slice(format!("My name is {}", p.name).as_bytes())) .unwrap(); } p @@ -329,14 +328,14 @@ fn insert_bytes(wal: &mut GenericOrderWal) -> Vec { #[test] fn insert_bytes_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); insert_bytes(&mut wal); } #[test] fn insert_bytes_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); insert_bytes(&mut wal); } @@ -347,9 +346,9 @@ fn insert_bytes_map_file() { let path = dir.path().join("generic_wal_insert_bytes_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), + OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -389,6 +388,7 @@ fn concurrent_basic(mut w: GenericOrderWal) { spawn(move || { for i in 0..100u32 { + #[allow(clippy::needless_borrows_for_generic_args)] w.insert(&i, &i.to_le_bytes()).unwrap(); } }); @@ -400,7 +400,7 @@ fn concurrent_basic(mut w: GenericOrderWal) { #[test] fn concurrent_basic_inmemory() { - let wal = GenericOrderWal::::new(Options::new().with_capacity(MB).with_reserved(4)) + let wal = GenericBuilder::new().with_capacity(MB).alloc() .unwrap(); concurrent_basic(wal); } @@ -408,7 +408,7 @@ fn concurrent_basic_inmemory() { #[test] fn concurrent_basic_map_anon() { let wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB).with_reserved(4)) + GenericBuilder::new().with_capacity(MB).map_anon() .unwrap(); concurrent_basic(wal); } @@ -420,9 +420,8 @@ fn concurrent_basic_map_file() { let path = dir.path().join("generic_wal_concurrent_basic_map_file"); let wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new().with_reserved(4), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -434,7 +433,7 @@ fn concurrent_basic_map_file() { concurrent_basic(wal); let wal = - unsafe { GenericOrderWal::::map(path, Options::new().with_reserved(4)).unwrap() }; + unsafe { GenericBuilder::new().map::(path).unwrap() }; for i in 0..100u32 { assert!(wal.contains_key(&i)); @@ -453,7 +452,7 @@ fn concurrent_one_key(mut w: GenericOrderWal) { }) }); - w.insert(&1, &1u32.to_le_bytes()).unwrap(); + w.insert(1, 1u32.to_le_bytes()).unwrap(); for handle in handles { handle.join().unwrap(); @@ -462,7 +461,7 @@ fn concurrent_one_key(mut w: GenericOrderWal) { #[test] fn concurrent_one_key_inmemory() { - let wal = GenericOrderWal::::new(Options::new().with_capacity(MB).with_reserved(4)) + let wal = GenericBuilder::new().alloc() .unwrap(); concurrent_one_key(wal); } @@ -470,7 +469,7 @@ fn concurrent_one_key_inmemory() { #[test] fn concurrent_one_key_map_anon() { let wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB).with_reserved(4)) + GenericBuilder::new().map_anon() .unwrap(); concurrent_one_key(wal); } @@ -482,9 +481,8 @@ fn concurrent_one_key_map_file() { let path = dir.path().join("generic_wal_concurrent_basic_map_file"); let wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new().with_reserved(4), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -496,7 +494,7 @@ fn concurrent_one_key_map_file() { concurrent_one_key(wal); let wal = - unsafe { GenericOrderWal::::map(path, Options::new().with_reserved(4)).unwrap() }; + unsafe { GenericBuilder::new().map::(path).unwrap() }; assert!(wal.contains_key(&1)); } diff --git a/src/swmr/generic/tests/iters.rs b/src/swmr/generic/tests/iters.rs index 3370236..ddf106d 100644 --- a/src/swmr/generic/tests/iters.rs +++ b/src/swmr/generic/tests/iters.rs @@ -31,14 +31,14 @@ fn iter(wal: &mut GenericOrderWal) -> Vec<(Person, String)> { #[test] fn iter_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); iter(&mut wal); } #[test] fn iter_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); iter(&mut wal); } @@ -49,9 +49,8 @@ fn iter_map_file() { let path = dir.path().join("generic_wal_iter_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut::( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -113,14 +112,14 @@ fn range(wal: &mut GenericOrderWal) { #[test] fn range_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); range(&mut wal); } #[test] fn range_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); range(&mut wal); } @@ -131,9 +130,8 @@ fn range_map_file() { let path = dir.path().join("generic_wal_range_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut::( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) @@ -190,14 +188,14 @@ fn range_ref(wal: &mut GenericOrderWal) { #[test] fn range_ref_inmemory() { - let mut wal = GenericOrderWal::::new(Options::new().with_capacity(MB)).unwrap(); + let mut wal = GenericBuilder::new().with_capacity(MB).alloc::().unwrap(); range(&mut wal); } #[test] fn range_ref_map_anon() { let mut wal = - GenericOrderWal::::map_anon(Options::new().with_capacity(MB)).unwrap(); + GenericBuilder::new().with_capacity(MB).map_anon::().unwrap(); range_ref(&mut wal); } @@ -208,9 +206,8 @@ fn range_ref_map_file() { let path = dir.path().join("generic_wal_range_map_file"); let mut wal = unsafe { - GenericOrderWal::::map_mut( + GenericBuilder::new().map_mut( &path, - Options::new(), OpenOptions::new() .create_new(Some(MB)) .write(true) diff --git a/src/swmr/wal.rs b/src/swmr/wal.rs index 55fc88b..1a54757 100644 --- a/src/swmr/wal.rs +++ b/src/swmr/wal.rs @@ -1,12 +1,25 @@ -use super::super::*; +use crossbeam_skiplist::SkipSet; -use either::Either; -use error::Error; -use wal::{ - sealed::{Constructor, Sealed, WalCore}, - ImmutableWal, +use crate::{ + error::Error, + pointer::Pointer, + wal::sealed::{Constructor, Sealed, WalCore}, + Options, }; +use dbutils::{ + checksum::{BuildChecksumer, Crc32}, + Ascend, +}; +use rarena_allocator::{either::Either, Allocator}; + +pub use crate::{ + builder::Builder, + wal::{Batch, BatchWithBuilders, BatchWithKeyBuilder, BatchWithValueBuilder, ImmutableWal, Wal}, + Comparator, KeyBuilder, VacantBuffer, ValueBuilder, +}; +pub use dbutils::CheapClone; +use core::{borrow::Borrow, marker::PhantomData}; use rarena_allocator::sync::Arena; use std::sync::Arc; @@ -147,7 +160,7 @@ where impl OrderWal { /// Returns the path of the WAL if it is backed by a file. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::{swmr::OrderWal, Wal, Builder}; diff --git a/src/unsync.rs b/src/unsync.rs index 8bdd5c7..cecd314 100644 --- a/src/unsync.rs +++ b/src/unsync.rs @@ -3,12 +3,17 @@ use std::{collections::BTreeSet, rc::Rc}; use super::*; +use checksum::BuildChecksumer; use either::Either; use error::Error; +use pointer::Pointer; use rarena_allocator::unsync::Arena; -use wal::{ - sealed::{Constructor, Sealed}, - ImmutableWal, +use wal::sealed::{Constructor, Sealed}; + +pub use super::{ + builder::Builder, + wal::{Batch, BatchWithBuilders, BatchWithKeyBuilder, BatchWithValueBuilder, ImmutableWal, Wal}, + Comparator, KeyBuilder, VacantBuffer, ValueBuilder, }; /// Iterators for the `OrderWal`. @@ -79,7 +84,7 @@ where impl OrderWal { /// Returns the path of the WAL if it is backed by a file. /// - /// # Example + /// ## Example /// /// ```rust /// use orderwal::{unsync::OrderWal, Wal, Builder}; diff --git a/src/utils.rs b/src/utils.rs index dd141cf..b08c761 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -86,3 +86,21 @@ pub(crate) const fn check( Ok(()) } + +#[inline] +pub(crate) fn check_batch_entry( + klen: usize, + vlen: usize, + max_key_size: u32, + max_value_size: u32, +) -> Result<(), Error> { + if klen > max_key_size as usize { + return Err(Error::key_too_large(klen as u64, max_key_size)); + } + + if vlen > max_value_size as usize { + return Err(Error::value_too_large(vlen as u64, max_value_size)); + } + + Ok(()) +} diff --git a/src/wal.rs b/src/wal.rs index 9b74f45..d49f774 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -1,194 +1,13 @@ +use checksum::BuildChecksumer; use core::ops::RangeBounds; use super::*; -mod builder; -pub use builder::*; - -mod generic_builder; -pub use generic_builder::*; - pub(crate) mod sealed; +pub(crate) mod r#type; -/// A batch of keys and values that can be inserted into the [`Wal`]. -pub trait Batch { - /// The key type. - type Key: Borrow<[u8]>; - - /// The value type. - type Value: Borrow<[u8]>; - - /// The [`Comparator`] type. - type Comparator: Comparator; - - /// The iterator type. - type IterMut<'a>: Iterator> - where - Self: 'a; - - /// Returns an iterator over the keys and values. - fn iter_mut(&mut self) -> Self::IterMut<'_>; -} - -impl Batch for T -where - K: Borrow<[u8]>, - V: Borrow<[u8]>, - C: Comparator, - for<'a> &'a mut T: IntoIterator>, -{ - type Key = K; - type Value = V; - type Comparator = C; - - type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; - - fn iter_mut(&mut self) -> Self::IterMut<'_> { - IntoIterator::into_iter(self) - } -} - -/// A batch of keys and values that can be inserted into the [`Wal`]. -/// Comparing to [`Batch`], this trait is used to build -/// the key in place. -pub trait BatchWithKeyBuilder { - /// The key builder type. - type KeyBuilder: Fn(&mut VacantBuffer<'_>) -> Result<(), Self::Error>; - - /// The error for the key builder. - type Error; - - /// The value type. - type Value: Borrow<[u8]>; - - /// The [`Comparator`] type. - type Comparator: Comparator; - - /// The iterator type. - type IterMut<'a>: Iterator< - Item = &'a mut EntryWithKeyBuilder, - > - where - Self: 'a; - - /// Returns an iterator over the keys and values. - fn iter_mut(&mut self) -> Self::IterMut<'_>; -} - -impl BatchWithKeyBuilder for T -where - KB: Fn(&mut VacantBuffer<'_>) -> Result<(), E>, - V: Borrow<[u8]>, - C: Comparator, - for<'a> &'a mut T: IntoIterator>, -{ - type KeyBuilder = KB; - type Error = E; - type Value = V; - type Comparator = C; - - type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; - - fn iter_mut(&mut self) -> Self::IterMut<'_> { - IntoIterator::into_iter(self) - } -} - -/// A batch of keys and values that can be inserted into the [`Wal`]. -/// Comparing to [`Batch`], this trait is used to build -/// the value in place. -pub trait BatchWithValueBuilder { - /// The value builder type. - type ValueBuilder: Fn(&mut VacantBuffer<'_>) -> Result<(), Self::Error>; - - /// The error for the value builder. - type Error; - - /// The key type. - type Key: Borrow<[u8]>; - - /// The [`Comparator`] type. - type Comparator: Comparator; - - /// The iterator type. - type IterMut<'a>: Iterator< - Item = &'a mut EntryWithValueBuilder, - > - where - Self: 'a; - - /// Returns an iterator over the keys and values. - fn iter_mut(&mut self) -> Self::IterMut<'_>; -} - -impl BatchWithValueBuilder for T -where - VB: Fn(&mut VacantBuffer<'_>) -> Result<(), E>, - K: Borrow<[u8]>, - C: Comparator, - for<'a> &'a mut T: IntoIterator>, -{ - type Key = K; - type Error = E; - type ValueBuilder = VB; - type Comparator = C; - - type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; - - fn iter_mut(&mut self) -> Self::IterMut<'_> { - IntoIterator::into_iter(self) - } -} - -/// A batch of keys and values that can be inserted into the [`Wal`]. -/// Comparing to [`Batch`], this trait is used to build -/// the key and value in place. -pub trait BatchWithBuilders { - /// The value builder type. - type ValueBuilder: Fn(&mut VacantBuffer<'_>) -> Result<(), Self::ValueError>; - - /// The error for the value builder. - type ValueError; - - /// The value builder type. - type KeyBuilder: Fn(&mut VacantBuffer<'_>) -> Result<(), Self::KeyError>; - - /// The error for the value builder. - type KeyError; - - /// The [`Comparator`] type. - type Comparator: Comparator; - - /// The iterator type. - type IterMut<'a>: Iterator< - Item = &'a mut EntryWithBuilders, - > - where - Self: 'a; - - /// Returns an iterator over the keys and values. - fn iter_mut(&mut self) -> Self::IterMut<'_>; -} - -impl BatchWithBuilders for T -where - VB: Fn(&mut VacantBuffer<'_>) -> Result<(), VE>, - KB: Fn(&mut VacantBuffer<'_>) -> Result<(), KE>, - C: Comparator, - for<'a> &'a mut T: IntoIterator>, -{ - type KeyBuilder = KB; - type KeyError = KE; - type ValueBuilder = VB; - type ValueError = VE; - type Comparator = C; - - type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; - - fn iter_mut(&mut self) -> Self::IterMut<'_> { - IntoIterator::into_iter(self) - } -} +mod batch; +pub use batch::*; /// An abstract layer for the immutable write-ahead log. pub trait ImmutableWal: sealed::Constructor { @@ -239,7 +58,7 @@ pub trait ImmutableWal: sealed::Constructor { /// Returns the reserved space in the WAL. /// - /// # Safety + /// ## Safety /// - The writer must ensure that the returned slice is not modified. /// - This method is not thread-safe, so be careful when using it. unsafe fn reserved_slice<'a>(&'a self) -> &'a [u8] @@ -361,7 +180,7 @@ pub trait ImmutableWal: sealed::Constructor { /// An abstract layer for the write-ahead log. pub trait Wal: - sealed::Sealed> + ImmutableWal + sealed::Sealed> + ImmutableWal { /// The read only reader type for this wal. type Reader: ImmutableWal; @@ -373,7 +192,7 @@ pub trait Wal: /// Returns the mutable reference to the reserved slice. /// - /// # Safety + /// ## Safety /// - The caller must ensure that the there is no others accessing reserved slice for either read or write. /// - This method is not thread-safe, so be careful when using it. unsafe fn reserved_slice_mut<'a>(&'a mut self) -> &'a mut [u8] diff --git a/src/wal/batch.rs b/src/wal/batch.rs new file mode 100644 index 0000000..179b777 --- /dev/null +++ b/src/wal/batch.rs @@ -0,0 +1,260 @@ +use core::borrow::Borrow; + +use dbutils::{buffer::VacantBuffer, Comparator}; + +use super::{entry::{ + Entry, EntryWithBuilders, EntryWithKeyBuilder, EntryWithValueBuilder, GenericEntry, +}, GenericEntryRefMut}; + +/// A batch of keys and values that can be inserted into the [`Wal`]. +pub trait Batch { + /// The key type. + type Key: Borrow<[u8]>; + + /// The value type. + type Value: Borrow<[u8]>; + + /// The [`Comparator`] type. + type Comparator: Comparator; + + /// The iterator type. + type IterMut<'a>: Iterator> + where + Self: 'a; + + /// Returns an iterator over the keys and values. + fn iter_mut(&mut self) -> Self::IterMut<'_>; +} + +impl Batch for T +where + K: Borrow<[u8]>, + V: Borrow<[u8]>, + C: Comparator, + for<'a> &'a mut T: IntoIterator>, +{ + type Key = K; + type Value = V; + type Comparator = C; + + type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; + + fn iter_mut(&mut self) -> Self::IterMut<'_> { + IntoIterator::into_iter(self) + } +} + +/// A batch of keys and values that can be inserted into the [`Wal`]. +/// Comparing to [`Batch`], this trait is used to build +/// the key in place. +pub trait BatchWithKeyBuilder { + /// The key builder type. + type KeyBuilder: Fn(&mut VacantBuffer<'_>) -> Result<(), Self::Error>; + + /// The error for the key builder. + type Error; + + /// The value type. + type Value: Borrow<[u8]>; + + /// The [`Comparator`] type. + type Comparator: Comparator; + + /// The iterator type. + type IterMut<'a>: Iterator< + Item = &'a mut EntryWithKeyBuilder, + > + where + Self: 'a; + + /// Returns an iterator over the keys and values. + fn iter_mut(&mut self) -> Self::IterMut<'_>; +} + +impl BatchWithKeyBuilder for T +where + KB: Fn(&mut VacantBuffer<'_>) -> Result<(), E>, + V: Borrow<[u8]>, + C: Comparator, + for<'a> &'a mut T: IntoIterator>, +{ + type KeyBuilder = KB; + type Error = E; + type Value = V; + type Comparator = C; + + type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; + + fn iter_mut(&mut self) -> Self::IterMut<'_> { + IntoIterator::into_iter(self) + } +} + +/// A batch of keys and values that can be inserted into the [`Wal`]. +/// Comparing to [`Batch`], this trait is used to build +/// the value in place. +pub trait BatchWithValueBuilder { + /// The value builder type. + type ValueBuilder: Fn(&mut VacantBuffer<'_>) -> Result<(), Self::Error>; + + /// The error for the value builder. + type Error; + + /// The key type. + type Key: Borrow<[u8]>; + + /// The [`Comparator`] type. + type Comparator: Comparator; + + /// The iterator type. + type IterMut<'a>: Iterator< + Item = &'a mut EntryWithValueBuilder, + > + where + Self: 'a; + + /// Returns an iterator over the keys and values. + fn iter_mut(&mut self) -> Self::IterMut<'_>; +} + +impl BatchWithValueBuilder for T +where + VB: Fn(&mut VacantBuffer<'_>) -> Result<(), E>, + K: Borrow<[u8]>, + C: Comparator, + for<'a> &'a mut T: IntoIterator>, +{ + type Key = K; + type Error = E; + type ValueBuilder = VB; + type Comparator = C; + + type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; + + fn iter_mut(&mut self) -> Self::IterMut<'_> { + IntoIterator::into_iter(self) + } +} + +/// A batch of keys and values that can be inserted into the [`Wal`]. +/// Comparing to [`Batch`], this trait is used to build +/// the key and value in place. +pub trait BatchWithBuilders { + /// The value builder type. + type ValueBuilder: Fn(&mut VacantBuffer<'_>) -> Result<(), Self::ValueError>; + + /// The error for the value builder. + type ValueError; + + /// The value builder type. + type KeyBuilder: Fn(&mut VacantBuffer<'_>) -> Result<(), Self::KeyError>; + + /// The error for the value builder. + type KeyError; + + /// The [`Comparator`] type. + type Comparator: Comparator; + + /// The iterator type. + type IterMut<'a>: Iterator< + Item = &'a mut EntryWithBuilders, + > + where + Self: 'a; + + /// Returns an iterator over the keys and values. + fn iter_mut(&mut self) -> Self::IterMut<'_>; +} + +impl BatchWithBuilders for T +where + VB: Fn(&mut VacantBuffer<'_>) -> Result<(), VE>, + KB: Fn(&mut VacantBuffer<'_>) -> Result<(), KE>, + C: Comparator, + for<'a> &'a mut T: IntoIterator>, +{ + type KeyBuilder = KB; + type KeyError = KE; + type ValueBuilder = VB; + type ValueError = VE; + type Comparator = C; + + type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; + + fn iter_mut(&mut self) -> Self::IterMut<'_> { + IntoIterator::into_iter(self) + } +} + +/// An iterator wrapper for any `&mut T: IntoIterator>`. +pub struct GenericBatchIterMut<'a, K, V, T> { + iter: T, + _m: core::marker::PhantomData<&'a (K, V)>, +} + +impl GenericBatchIterMut<'_, K, V, T> { + /// Creates a new iterator wrapper. + #[inline] + const fn new(iter: T) -> Self { + Self { + iter, + _m: core::marker::PhantomData, + } + } +} + +impl<'a, K, V, T> Iterator for GenericBatchIterMut<'a, K, V, T> +where + T: Iterator>, +{ + type Item = GenericEntryRefMut<'a, K, V>; + + fn next(&mut self) -> Option { + self.iter.next().map(|entry| entry.as_ref_mut()) + } +} + +/// The container for entries in the [`GenericBatch`]. +pub trait GenericBatch { + /// The key type. + type Key; + + /// The value type. + type Value; + + /// The iterator type. + type IterMut<'e>: Iterator> + where + Self: 'e; + + /// Returns an iterator over the keys and values. + fn iter_mut(&mut self) -> Self::IterMut<'_>; +} + +impl GenericBatch for T +where + for<'a> &'a mut T: IntoIterator>, +{ + type Key = K; + type Value = V; + + type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; + + fn iter_mut(&mut self) -> Self::IterMut<'_> { + IntoIterator::into_iter(self) + } +} + +// impl<'a, K, V> GenericBatch for Vec> +// { +// type Key = K; +// type Value = V; + +// type IterMut<'b> = GenericBatchIterMut<'b, K, V, core::slice::IterMut<'a, GenericEntry<'a, K, V>>> +// where +// Self: 'b; + +// fn iter_mut(&mut self) -> Self::IterMut<'_> { +// GenericBatchIterMut::new(IntoIterator::into_iter(self)) +// } +// } diff --git a/src/wal/generic_builder.rs b/src/wal/generic_builder.rs deleted file mode 100644 index 29376fb..0000000 --- a/src/wal/generic_builder.rs +++ /dev/null @@ -1,330 +0,0 @@ -use super::*; - -/// A write-ahead log builder. -pub struct GenericBuilder { - pub(super) opts: Options, - pub(super) cks: S, -} - -impl Default for GenericBuilder { - #[inline] - fn default() -> Self { - Self::new() - } -} - -impl GenericBuilder { - /// Returns a new write-ahead log builder with the given options. - #[inline] - pub fn new() -> Self { - Self { - opts: Options::default(), - cks: Crc32::default(), - } - } -} - -impl GenericBuilder { - /// Returns a new write-ahead log builder with the new checksumer - /// - /// # Example - /// - /// ```rust - /// use orderwal::{GenericBuilder, Crc32}; - /// - /// let opts = GenericBuilder::new().with_checksumer(Crc32::new()); - /// ``` - #[inline] - pub fn with_checksumer(self, cks: NS) -> GenericBuilder { - GenericBuilder { - opts: self.opts, - cks, - } - } - - /// Returns a new write-ahead log builder with the new options - /// - /// # Example - /// - /// ```rust - /// use orderwal::{GenericBuilder, Options}; - /// - /// let opts = GenericBuilder::new().with_options(Options::default()); - /// ``` - #[inline] - pub fn with_options(self, opts: Options) -> Self { - Self { - opts, - cks: self.cks, - } - } - - /// Set the reserved bytes of the WAL. - /// - /// The `reserved` is used to configure the start position of the WAL. This is useful - /// when you want to add some bytes as your own WAL's header. - /// - /// The default reserved is `0`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let opts = GenericBuilder::new().with_reserved(8); - /// ``` - #[inline] - pub const fn with_reserved(mut self, reserved: u32) -> Self { - self.opts = self.opts.with_reserved(reserved); - self - } - - /// Get the reserved of the WAL. - /// - /// The `reserved` is used to configure the start position of the WAL. This is useful - /// when you want to add some bytes as your own WAL's header. - /// - /// The default reserved is `0`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let opts = GenericBuilder::new().with_reserved(8); - /// - /// assert_eq!(opts.reserved(), 8); - /// ``` - #[inline] - pub const fn reserved(&self) -> u32 { - self.opts.reserved() - } - - /// Returns the magic version. - /// - /// The default value is `0`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_magic_version(1); - /// assert_eq!(options.magic_version(), 1); - /// ``` - #[inline] - pub const fn magic_version(&self) -> u16 { - self.opts.magic_version() - } - - /// Returns the capacity of the WAL. - /// - /// The default value is `0`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_capacity(1000); - /// assert_eq!(options.capacity(), 1000); - /// ``` - #[inline] - pub const fn capacity(&self) -> u32 { - self.opts.capacity() - } - - /// Returns the maximum key length. - /// - /// The default value is `u16::MAX`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_maximum_key_size(1024); - /// assert_eq!(options.maximum_key_size(), 1024); - /// ``` - #[inline] - pub const fn maximum_key_size(&self) -> u32 { - self.opts.maximum_key_size() - } - - /// Returns the maximum value length. - /// - /// The default value is `u32::MAX`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_maximum_value_size(1024); - /// assert_eq!(options.maximum_value_size(), 1024); - /// ``` - #[inline] - pub const fn maximum_value_size(&self) -> u32 { - self.opts.maximum_value_size() - } - - /// Returns `true` if the WAL syncs on write. - /// - /// The default value is `true`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new(); - /// assert_eq!(options.sync_on_write(), true); - /// ``` - #[inline] - pub const fn sync_on_write(&self) -> bool { - self.opts.sync_on_write() - } - - /// Returns the bits of the page size. - /// - /// Configures the anonymous memory map to be allocated using huge pages. - /// - /// This option corresponds to the `MAP_HUGETLB` flag on Linux. It has no effect on Windows. - /// - /// The size of the requested page can be specified in page bits. - /// If not provided, the system default is requested. - /// The requested length should be a multiple of this, or the mapping will fail. - /// - /// This option has no effect on file-backed memory maps. - /// - /// The default value is `None`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_huge(64); - /// assert_eq!(options.huge(), Some(64)); - /// ``` - #[inline] - pub const fn huge(&self) -> Option { - self.opts.huge() - } - - /// Sets the capacity of the WAL. - /// - /// This configuration will be ignored when using file-backed memory maps. - /// - /// The default value is `0`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_capacity(100); - /// assert_eq!(options.capacity(), 100); - /// ``` - #[inline] - pub const fn with_capacity(mut self, cap: u32) -> Self { - self.opts = self.opts.with_capacity(cap); - self - } - - /// Sets the maximum key length. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_maximum_key_size(1024); - /// assert_eq!(options.maximum_key_size(), 1024); - /// ``` - #[inline] - pub const fn with_maximum_key_size(mut self, size: u32) -> Self { - self.opts = self.opts.with_maximum_key_size(size); - self - } - - /// Sets the maximum value length. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_maximum_value_size(1024); - /// assert_eq!(options.maximum_value_size(), 1024); - /// ``` - #[inline] - pub const fn with_maximum_value_size(mut self, size: u32) -> Self { - self.opts = self.opts.with_maximum_value_size(size); - self - } - - /// Returns the bits of the page size. - /// - /// Configures the anonymous memory map to be allocated using huge pages. - /// - /// This option corresponds to the `MAP_HUGETLB` flag on Linux. It has no effect on Windows. - /// - /// The size of the requested page can be specified in page bits. - /// If not provided, the system default is requested. - /// The requested length should be a multiple of this, or the mapping will fail. - /// - /// This option has no effect on file-backed memory maps. - /// - /// The default value is `None`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_huge(64); - /// assert_eq!(options.huge(), Some(64)); - /// ``` - #[inline] - pub const fn with_huge(mut self, page_bits: u8) -> Self { - self.opts = self.opts.with_huge(page_bits); - self - } - - /// Sets the WAL to sync on write. - /// - /// The default value is `true`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_sync_on_write(false); - /// assert_eq!(options.sync_on_write(), false); - /// ``` - #[inline] - pub const fn with_sync_on_write(mut self, sync: bool) -> Self { - self.opts = self.opts.with_sync_on_write(sync); - self - } - - /// Sets the magic version. - /// - /// The default value is `0`. - /// - /// # Example - /// - /// ```rust - /// use orderwal::GenericBuilder; - /// - /// let options = GenericBuilder::new().with_magic_version(1); - /// assert_eq!(options.magic_version(), 1); - /// ``` - #[inline] - pub const fn with_magic_version(mut self, version: u16) -> Self { - self.opts = self.opts.with_magic_version(version); - self - } -} diff --git a/src/wal/sealed.rs b/src/wal/sealed.rs index 1ca4158..251ae0f 100644 --- a/src/wal/sealed.rs +++ b/src/wal/sealed.rs @@ -1,5 +1,6 @@ use core::ptr::NonNull; +use checksum::{BuildChecksumer, Checksumer}; use rarena_allocator::ArenaPosition; use super::*; @@ -10,32 +11,23 @@ pub trait Pointer { fn new(klen: usize, vlen: usize, ptr: *const u8, cmp: Self::Comparator) -> Self; } -impl Pointer for crate::Pointer { - type Comparator = C; - - #[inline] - fn new(klen: usize, vlen: usize, ptr: *const u8, cmp: C) -> Self { - crate::Pointer::::new(klen, vlen, ptr, cmp) - } -} - pub trait Base: Default { type Pointer: Pointer; fn insert(&mut self, ele: Self::Pointer) where - Self::Pointer: Ord; + Self::Pointer: Ord + 'static; } impl

Base for SkipSet

where - P: Pointer + Send + 'static, + P: Pointer + Send, { type Pointer = P; fn insert(&mut self, ele: Self::Pointer) where - P: Ord, + P: Ord + 'static, { SkipSet::insert(self, ele); } @@ -68,15 +60,7 @@ pub trait Sealed: Constructor { let max_key_size = opts.maximum_key_size(); let max_value_size = opts.maximum_value_size(); - if klen > max_key_size as usize { - return Err(Error::key_too_large(klen as u64, max_key_size)); - } - - if vlen > max_value_size as usize { - return Err(Error::value_too_large(vlen as u64, max_value_size)); - } - - Ok(()) + crate::utils::check_batch_entry(klen, vlen, max_key_size, max_value_size) } fn hasher(&self) -> &S; @@ -599,7 +583,7 @@ pub trait Constructor: Sized { where C: CheapClone, S: BuildChecksumer, - Self::Pointer: Ord, + Self::Pointer: Ord + 'static, { let slice = arena.reserved_slice(); let magic_text = &slice[0..6]; diff --git a/src/swmr/generic/traits.rs b/src/wal/type.rs similarity index 77% rename from src/swmr/generic/traits.rs rename to src/wal/type.rs index 5012c98..a9c8620 100644 --- a/src/swmr/generic/traits.rs +++ b/src/wal/type.rs @@ -1,30 +1,11 @@ -use core::{cmp, hash::Hash}; +use core::cmp; use among::Among; -use crossbeam_skiplist::{Comparable, Equivalent}; +use dbutils::equivalent::Comparable; mod impls; pub use impls::*; -use super::GenericEntry; - -/// The container for entries in the [`GenericBatch`]. -pub trait GenericBatch { - /// The key type. - type Key; - - /// The value type. - type Value; - - /// The iterator type. - type IterMut<'a>: Iterator> - where - Self: 'a; - - /// Returns an iterator over the keys and values. - fn iter_mut(&mut self) -> Self::IterMut<'_>; -} - /// The type trait for limiting the types that can be used as keys and values in the [`GenericOrderWal`]. /// /// This trait and its implementors can only be used with the [`GenericOrderWal`] type, otherwise @@ -39,8 +20,16 @@ pub trait Type { /// Returns the length of the encoded type size. fn encoded_len(&self) -> usize; - /// Encodes the type into a binary slice, you can assume that the buf length is equal to the value returned by [`encoded_len`](Type::encoded_len). + /// Encodes the type into a bytes slice, you can assume that the buf length is equal to the value returned by [`encoded_len`](Type::encoded_len). fn encode(&self, buf: &mut [u8]) -> Result<(), Self::Error>; + + /// Encodes the type into a [`Vec`]. + #[inline] + fn encode_into_vec(&self) -> Result, Self::Error> { + let mut buf = vec![0; self.encoded_len()]; + self.encode(&mut buf)?; + Ok(buf) + } } impl Type for &T { @@ -58,7 +47,7 @@ impl Type for &T { } } -pub(super) trait InsertAmongExt { +pub(crate) trait InsertAmongExt { fn encoded_len(&self) -> usize; fn encode(&self, buf: &mut [u8]) -> Result<(), T::Error>; } @@ -91,7 +80,7 @@ pub trait TypeRef<'a> { /// Creates a reference type from a binary slice, when using it with [`GenericOrderWal`], /// you can assume that the slice is the same as the one returned by [`encode`](Type::encode). /// - /// # Safety + /// ## Safety /// - the `src` must the same as the one returned by [`encode`](Type::encode). unsafe fn from_slice(src: &'a [u8]) -> Self; } diff --git a/src/swmr/generic/traits/impls.rs b/src/wal/type/impls.rs similarity index 92% rename from src/swmr/generic/traits/impls.rs rename to src/wal/type/impls.rs index 01a20a6..3781314 100644 --- a/src/swmr/generic/traits/impls.rs +++ b/src/wal/type/impls.rs @@ -7,20 +7,6 @@ pub use string::Str; mod net; -impl GenericBatch for T -where - for<'a> &'a mut T: IntoIterator>, -{ - type Key = K; - type Value = V; - - type IterMut<'a> = <&'a mut T as IntoIterator>::IntoIter where Self: 'a; - - fn iter_mut(&mut self) -> Self::IterMut<'_> { - IntoIterator::into_iter(self) - } -} - impl Type for () { type Ref<'a> = (); type Error = (); diff --git a/src/swmr/generic/traits/impls/bytes.rs b/src/wal/type/impls/bytes.rs similarity index 100% rename from src/swmr/generic/traits/impls/bytes.rs rename to src/wal/type/impls/bytes.rs diff --git a/src/swmr/generic/traits/impls/net.rs b/src/wal/type/impls/net.rs similarity index 100% rename from src/swmr/generic/traits/impls/net.rs rename to src/wal/type/impls/net.rs diff --git a/src/swmr/generic/traits/impls/string.rs b/src/wal/type/impls/string.rs similarity index 100% rename from src/swmr/generic/traits/impls/string.rs rename to src/wal/type/impls/string.rs