diff --git a/packages/cw-storey/src/lib.rs b/packages/cw-storey/src/lib.rs index 3dd7878..69297d0 100644 --- a/packages/cw-storey/src/lib.rs +++ b/packages/cw-storey/src/lib.rs @@ -3,9 +3,9 @@ //! This crate provides //! - a [*CosmWasm*] storage backend for use with [`storey`] collections, //! - a [*MessagePack*] encoding integration to be used for serializing and deserializing -//! values, and +//! values, and //! - a set of container re-exports that remove the need to manually specify the -//! encoding, instead relying on the default [*MessagePack*] encoding. +//! encoding, instead relying on the default [*MessagePack*] encoding. //! //! [*CosmWasm*]: https://github.com/CosmWasm/cosmwasm //! [*MessagePack*]: https://msgpack.org/ diff --git a/packages/storey/src/containers/column.rs b/packages/storey/src/containers/column.rs index fcd7a2f..64b06ed 100644 --- a/packages/storey/src/containers/column.rs +++ b/packages/storey/src/containers/column.rs @@ -7,7 +7,7 @@ use crate::encoding::{DecodableWith, EncodableWith}; use crate::storage::{IterableStorage, StorageBranch}; use crate::storage::{Storage, StorageMut}; -use super::{BoundFor, BoundedIterableAccessor, IterableAccessor, Storable}; +use super::{BoundFor, BoundedIterableAccessor, IterableAccessor, NonTerminal, Storable}; const META_LAST_IX: &[u8] = &[0]; const META_LEN: &[u8] = &[1]; @@ -85,6 +85,7 @@ where E: Encoding, T: EncodableWith + DecodableWith, { + type Kind = NonTerminal; type Accessor = ColumnAccess; type Key = u32; type KeyDecodeError = ColumnKeyDecodeError; diff --git a/packages/storey/src/containers/item.rs b/packages/storey/src/containers/item.rs index 2d3a0f4..25d63e9 100644 --- a/packages/storey/src/containers/item.rs +++ b/packages/storey/src/containers/item.rs @@ -4,7 +4,7 @@ use crate::encoding::{DecodableWith, EncodableWith, Encoding}; use crate::storage::StorageBranch; use crate::storage::{Storage, StorageMut}; -use super::Storable; +use super::{Storable, Terminal}; /// A single item in the storage. /// @@ -70,6 +70,7 @@ where E: Encoding, T: EncodableWith + DecodableWith, { + type Kind = Terminal; type Accessor = ItemAccess; type Key = (); type KeyDecodeError = ItemKeyDecodeError; diff --git a/packages/storey/src/containers/map/key.rs b/packages/storey/src/containers/map/key.rs new file mode 100644 index 0000000..6f01279 --- /dev/null +++ b/packages/storey/src/containers/map/key.rs @@ -0,0 +1,213 @@ +/// A key that can be used with a [`Map`](crate::Map). +pub trait Key { + /// The kind of key, meaning either fixed size or dynamic size. + type Kind: KeyKind; + + /// Encode the key into a byte vector. + fn encode(&self) -> Vec; +} + +/// An owned key that can be used with a [`Map`](crate::Map). +pub trait OwnedKey: Key { + /// The error type that can occur when decoding the key. + type Error; + + /// Decode the key from a byte slice. + fn from_bytes(bytes: &[u8]) -> Result + where + Self: Sized; +} + +impl Key for String { + type Kind = DynamicKey; + + fn encode(&self) -> Vec { + self.as_bytes().to_vec() + } +} + +impl Key for Box { + type Kind = DynamicKey; + + fn encode(&self) -> Vec { + self.as_bytes().to_vec() + } +} + +impl Key for str { + type Kind = DynamicKey; + + fn encode(&self) -> Vec { + self.as_bytes().to_vec() + } +} + +/// An error type representing a failure to decode a UTF-8 string. +#[derive(Debug, PartialEq, Eq, Clone, Copy, thiserror::Error)] +#[error("invalid UTF8")] +pub struct InvalidUtf8; + +impl OwnedKey for String { + type Error = InvalidUtf8; + + fn from_bytes(bytes: &[u8]) -> Result + where + Self: Sized, + { + std::str::from_utf8(bytes) + .map(String::from) + .map_err(|_| InvalidUtf8) + } +} + +impl OwnedKey for Box { + type Error = InvalidUtf8; + + fn from_bytes(bytes: &[u8]) -> Result + where + Self: Sized, + { + std::str::from_utf8(bytes) + .map(Box::from) + .map_err(|_| InvalidUtf8) + } +} + +impl Key for Vec { + type Kind = DynamicKey; + + fn encode(&self) -> Vec { + self.clone() + } +} + +impl Key for Box<[u8]> { + type Kind = DynamicKey; + + fn encode(&self) -> Vec { + self.to_vec() + } +} + +impl Key for [u8] { + type Kind = DynamicKey; + + fn encode(&self) -> Vec { + self.to_vec() + } +} + +impl Key for [u8; N] { + type Kind = FixedSizeKey; + + fn encode(&self) -> Vec { + self.to_vec() + } +} + +impl OwnedKey for Vec { + type Error = (); + + fn from_bytes(bytes: &[u8]) -> Result + where + Self: Sized, + { + Ok(bytes.to_vec()) + } +} + +impl OwnedKey for Box<[u8]> { + type Error = (); + + fn from_bytes(bytes: &[u8]) -> Result + where + Self: Sized, + { + Ok(bytes.to_vec().into_boxed_slice()) + } +} + +/// An error type for decoding arrays. +pub enum ArrayDecodeError { + InvalidLength, +} + +impl OwnedKey for [u8; N] { + type Error = ArrayDecodeError; + + fn from_bytes(bytes: &[u8]) -> Result + where + Self: Sized, + { + if bytes.len() != N { + return Err(ArrayDecodeError::InvalidLength); + } + + let mut buf = [0; N]; + buf.copy_from_slice(bytes); + Ok(buf) + } +} + +/// A trait specifying the kind of key. +/// +/// There are two kinds of keys: fixed-size keys and dynamic keys, which are +/// represented by the [`FixedSizeKey`] and [`DynamicKey`] types, respectively. +/// +/// This trait is [sealed](https://rust-lang.github.io/api-guidelines/future-proofing.html#sealed-traits) +/// and cannot be implemented outside of this crate. +pub trait KeyKind: sealed::KeyKindSeal {} + +/// A marker type representing a fixed-size key. +pub struct FixedSizeKey; + +/// A marker type representing a dynamic-size key. +pub struct DynamicKey; + +impl KeyKind for FixedSizeKey {} +impl KeyKind for DynamicKey {} + +mod sealed { + pub trait KeyKindSeal {} + + impl KeyKindSeal for super::FixedSizeKey {} + impl KeyKindSeal for super::DynamicKey {} +} + +/// An error type for decoding numeric keys. +pub enum NumericKeyDecodeError { + InvalidLength, +} + +macro_rules! impl_key_for_numeric { + ($($t:ty),*) => { + $( + impl Key for $t { + type Kind = FixedSizeKey<{(Self::BITS / 8) as usize}>; + + fn encode(&self) -> Vec { + self.to_be_bytes().to_vec() + } + } + + impl OwnedKey for $t { + type Error = NumericKeyDecodeError; + + fn from_bytes(bytes: &[u8]) -> Result + where + Self: Sized, + { + if bytes.len() != std::mem::size_of::() { + return Err(NumericKeyDecodeError::InvalidLength); + } + + let mut buf = [0; std::mem::size_of::()]; + buf.copy_from_slice(bytes); + Ok(Self::from_be_bytes(buf)) + } + } + )* + }; +} + +impl_key_for_numeric!(u8, u16, u32, u64, u128, i8, i16, i32, i64, i128); diff --git a/packages/storey/src/containers/map/key_encoding.rs b/packages/storey/src/containers/map/key_encoding.rs new file mode 100644 index 0000000..6c26fee --- /dev/null +++ b/packages/storey/src/containers/map/key_encoding.rs @@ -0,0 +1,45 @@ +use crate::containers::{NonTerminal, Terminal}; + +use super::key::{DynamicKey, FixedSizeKey}; + +/// A trait that specifies the encoding strategy for a key. +/// +/// This trait is implemented on tuples of the form `(K, C)` where `K` is the key type (dynamic/fixed) +/// and `C` is the container type (terminal/nonterminal). Once we know these two properties, we can +/// determine the encoding strategy for the key. +/// +/// Scenarios: +/// - If the key is dynamic and the container is nonterminal, then the key needs to be +/// length prefixed - otherwise, we would not know where the key ends and the key for the inner +/// container starts. +/// - If the key is dynamic and the container is terminal, then the key is the rest of the string. +/// - If the key is fixed size, then we statically provide the number of bytes to read/write. +pub trait KeyEncodingT { + const BEHAVIOR: KeyEncoding; +} + +impl KeyEncodingT for (DynamicKey, NonTerminal) { + const BEHAVIOR: KeyEncoding = KeyEncoding::LenPrefix; +} + +impl KeyEncodingT for (FixedSizeKey, Terminal) { + const BEHAVIOR: KeyEncoding = KeyEncoding::UseRest; +} + +impl KeyEncodingT for (DynamicKey, Terminal) { + const BEHAVIOR: KeyEncoding = KeyEncoding::UseRest; +} + +impl KeyEncodingT for (FixedSizeKey, NonTerminal) { + const BEHAVIOR: KeyEncoding = KeyEncoding::UseN(L); +} + +/// The encoding strategy for a given key. +pub enum KeyEncoding { + /// The key needs to be length prefixed. + LenPrefix, + /// The key doesn't need to be length prefixed. The rest of the string is the key. + UseRest, + /// The key is of fixed size. + UseN(usize), +} diff --git a/packages/storey/src/containers/map.rs b/packages/storey/src/containers/map/mod.rs similarity index 61% rename from packages/storey/src/containers/map.rs rename to packages/storey/src/containers/map/mod.rs index a94391f..8f5a606 100644 --- a/packages/storey/src/containers/map.rs +++ b/packages/storey/src/containers/map/mod.rs @@ -1,10 +1,24 @@ +pub mod key; +mod key_encoding; + +pub use key::{Key, OwnedKey}; +use key_encoding::KeyEncoding; +use key_encoding::KeyEncodingT; + use std::{borrow::Borrow, marker::PhantomData}; use crate::storage::IterableStorage; use crate::storage::StorageBranch; +use self::key::DynamicKey; +use self::key::FixedSizeKey; + +use super::BoundFor; +use super::BoundedIterableAccessor; use super::IterableAccessor; +use super::NonTerminal; use super::Storable; +use super::Terminal; /// A map that stores values of type `V` under keys of type `K`. /// @@ -52,6 +66,7 @@ where K: OwnedKey, V: Storable, ::KeyDecodeError: std::fmt::Display, + (K::Kind, V::Kind): KeyEncodingT, { /// Creates a new map with the given prefix. /// @@ -94,7 +109,9 @@ where K: OwnedKey, V: Storable, ::KeyDecodeError: std::fmt::Display, + (K::Kind, V::Kind): KeyEncodingT, { + type Kind = NonTerminal; type Accessor = MapAccess; type Key = (K, V::Key); type KeyDecodeError = MapKeyDecodeError; @@ -109,17 +126,36 @@ where } fn decode_key(key: &[u8]) -> Result> { - let len = *key.first().ok_or(MapKeyDecodeError::EmptyKey)? as usize; - - if key.len() < len + 1 { - return Err(MapKeyDecodeError::KeyTooShort(len)); + let behavior = <(K::Kind, V::Kind)>::BEHAVIOR; + + match behavior { + KeyEncoding::LenPrefix => { + let len = *key.first().ok_or(MapKeyDecodeError::EmptyKey)? as usize; + + if key.len() < len + 1 { + return Err(MapKeyDecodeError::KeyTooShort(len)); + } + + let map_key = + K::from_bytes(&key[1..len + 1]).map_err(|_| MapKeyDecodeError::InvalidUtf8)?; + let rest = V::decode_key(&key[len + 1..]).map_err(MapKeyDecodeError::Inner)?; + + Ok((map_key, rest)) + } + KeyEncoding::UseRest => { + let map_key = K::from_bytes(key).map_err(|_| MapKeyDecodeError::InvalidUtf8)?; + let rest = V::decode_key(&[]).map_err(MapKeyDecodeError::Inner)?; + + Ok((map_key, rest)) + } + KeyEncoding::UseN(n) => { + let map_key = + K::from_bytes(&key[..n]).map_err(|_| MapKeyDecodeError::InvalidUtf8)?; + let rest = V::decode_key(&key[n..]).map_err(MapKeyDecodeError::Inner)?; + + Ok((map_key, rest)) + } } - - let map_key = - K::from_bytes(&key[1..len + 1]).map_err(|_| MapKeyDecodeError::InvalidUtf8)?; - let rest = V::decode_key(&key[len + 1..]).map_err(MapKeyDecodeError::Inner)?; - - Ok((map_key, rest)) } fn decode_value(value: &[u8]) -> Result { @@ -155,6 +191,7 @@ impl MapAccess where K: Key, V: Storable, + (K::Kind, V::Kind): KeyEncodingT, { /// Returns an immutable accessor for the inner container of this map. /// @@ -186,9 +223,14 @@ where pub fn entry(&self, key: &Q) -> V::Accessor> where K: Borrow, - Q: Key + ?Sized, + Q: Key + ?Sized, { - let key = length_prefixed_key(key); + let behavior = <(K::Kind, V::Kind)>::BEHAVIOR; + + let key = match behavior { + KeyEncoding::LenPrefix => len_prefix(key.encode()), + _ => key.encode(), + }; V::access_impl(StorageBranch::new(&self.storage, key)) } @@ -225,23 +267,25 @@ where pub fn entry_mut(&mut self, key: &Q) -> V::Accessor> where K: Borrow, - Q: Key + ?Sized, + Q: Key + ?Sized, { - let key = length_prefixed_key(key); + let behavior = <(K::Kind, V::Kind)>::BEHAVIOR; + + let key = match behavior { + KeyEncoding::LenPrefix => len_prefix(key.encode()), + _ => key.encode(), + }; V::access_impl(StorageBranch::new(&mut self.storage, key)) } } -fn length_prefixed_key(key: &K) -> Vec { - let len = key.bytes().len(); - let bytes = key.bytes(); - let mut key = Vec::with_capacity(len + 1); - - key.push(len as u8); - key.extend_from_slice(bytes); - - key +fn len_prefix>(bytes: T) -> Vec { + let len = bytes.as_ref().len(); + let mut result = Vec::with_capacity(len + 1); + result.extend_from_slice(&(len as u8).to_be_bytes()); + result.extend_from_slice(bytes.as_ref()); + result } impl IterableAccessor for MapAccess @@ -250,6 +294,7 @@ where V: Storable, ::KeyDecodeError: std::fmt::Display, S: IterableStorage, + (K::Kind, V::Kind): KeyEncodingT, { type Storable = Map; type Storage = S; @@ -259,44 +304,43 @@ where } } -pub trait Key { - fn bytes(&self) -> &[u8]; -} - -pub trait OwnedKey: Key { - type Error; - - fn from_bytes(bytes: &[u8]) -> Result - where - Self: Sized; -} +// The following dance is necessary to make bounded iteration unavailable for maps +// that have both dynamic keys and "non-terminal" values (i.e. maps of maps, maps of columns, etc). +// +// This is because in cases where the key is dynamically size **and** there's another key +// after it, we have to length-prefix the key. This makes bounded iteration behave differently +// than in other cases (and rather unintuitively). -impl Key for String { - fn bytes(&self) -> &[u8] { - self.as_bytes() - } +impl BoundedIterableAccessor for MapAccess +where + K: OwnedKey, + V: Storable, + ::KeyDecodeError: std::fmt::Display, + S: IterableStorage, + (K::Kind, V::Kind): BoundedIterationAllowed + KeyEncodingT, +{ } -impl Key for str { - fn bytes(&self) -> &[u8] { - self.as_bytes() - } -} +trait BoundedIterationAllowed {} -#[derive(Debug, PartialEq, Eq, Clone, Copy, thiserror::Error)] -#[error("invalid UTF8")] -pub struct InvalidUtf8; +impl BoundedIterationAllowed for (FixedSizeKey, Terminal) {} +impl BoundedIterationAllowed for (FixedSizeKey, NonTerminal) {} +impl BoundedIterationAllowed for (DynamicKey, Terminal) {} -impl OwnedKey for String { - type Error = InvalidUtf8; +impl BoundFor> for &Q +where + K: Borrow + OwnedKey, + V: Storable, + Q: Key + ?Sized, + (K::Kind, V::Kind): KeyEncodingT, +{ + fn into_bytes(self) -> Vec { + let behavior = <(K::Kind, V::Kind)>::BEHAVIOR; - fn from_bytes(bytes: &[u8]) -> Result - where - Self: Sized, - { - std::str::from_utf8(bytes) - .map(String::from) - .map_err(|_| InvalidUtf8) + match behavior { + KeyEncoding::LenPrefix => len_prefix(self.encode()), + _ => self.encode(), + } } } @@ -323,7 +367,7 @@ mod tests { assert_eq!(map.access(&storage).entry("foo").get().unwrap(), Some(1337)); assert_eq!( - storage.get(&[0, 3, 102, 111, 111]), + storage.get(&[0, 102, 111, 111]), Some(1337u64.to_le_bytes().to_vec()) ); map.access(&mut storage).entry_mut("foo").remove(); @@ -332,6 +376,74 @@ mod tests { assert_eq!(map.access(&storage).entry("bar").get().unwrap(), None); } + #[test] + fn bounded_iter_dyn_map_of_item() { + let mut storage = TestStorage::new(); + + let map = Map::>::new(0); + let mut access = map.access(&mut storage); + + access.entry_mut("foo").set(&1337).unwrap(); + access.entry_mut("bar").set(&42).unwrap(); + access.entry_mut("baz").set(&69).unwrap(); + + let items = access + .bounded_pairs(Some("bar"), Some("bazz")) + .collect::, _>>() + .unwrap(); + assert_eq!( + items, + vec![(("bar".to_string(), ()), 42), (("baz".to_string(), ()), 69)] + ); + } + + #[test] + fn iter_static_map_of_item() { + let mut storage = TestStorage::new(); + + let map = Map::>::new(0); + let mut access = map.access(&mut storage); + + access.entry_mut("foo").set(&1337).unwrap(); + access.entry_mut("bar").set(&42).unwrap(); + access.entry_mut("baz").set(&69).unwrap(); + + let items = access.pairs().collect::, _>>().unwrap(); + assert_eq!( + items, + vec![ + (("bar".to_string(), ()), 42), + (("baz".to_string(), ()), 69), + (("foo".to_string(), ()), 1337) + ] + ); + } + + #[test] + fn bounded_iter_static_map_of_map() { + let mut storage = TestStorage::new(); + + let map = Map::>>::new(0); + let mut access = map.access(&mut storage); + + access.entry_mut(&2).entry_mut("bar").set(&1337).unwrap(); + access.entry_mut(&3).entry_mut("baz").set(&42).unwrap(); + access.entry_mut(&4).entry_mut("quux").set(&69).unwrap(); + + let items = access + .bounded_pairs(Some(&2), Some(&4)) + .collect::, _>>() + .unwrap(); + + assert_eq!( + items, + vec![ + ((2, ("bar".to_string(), ())), 1337), + ((3, ("baz".to_string(), ())), 42) + ] + ); + } + #[test] fn pairs() { let mut storage = TestStorage::new(); diff --git a/packages/storey/src/containers/mod.rs b/packages/storey/src/containers/mod.rs index 3d723d2..6c5b3ad 100644 --- a/packages/storey/src/containers/mod.rs +++ b/packages/storey/src/containers/mod.rs @@ -3,7 +3,7 @@ mod column; mod item; -mod map; +pub mod map; use std::marker::PhantomData; @@ -15,6 +15,8 @@ use crate::storage::IterableStorage; /// The fundamental trait every collection/container should implement. pub trait Storable { + type Kind: StorableKind; + /// The accessor type for this collection/container. An accessor is a type that provides /// methods for reading and writing to the collection/container and encapsulates the /// specific [`Storage`] type used (the `S` type parameter here). @@ -66,7 +68,7 @@ pub enum KVDecodeError { Value(V), } -/// A trait for collection accessors (see [`Storable::AccessorT`]) that provide iteration over +/// A trait for collection accessors (see [`Storable::Accessor`]) that provide iteration over /// their contents. pub trait IterableAccessor: Sized { /// The [`Storable`] type this accessor is associated with. @@ -245,3 +247,33 @@ where self.inner.next().map(|v| S::decode_value(&v)) } } + +/// The kind of a storable. +/// +/// This is used to differentiate between terminal and non-terminal storables. +/// See also: [`Terminal`] and [`NonTerminal`]. +/// +/// This trait is [sealed](https://rust-lang.github.io/api-guidelines/future-proofing.html#sealed-traits-protect-against-downstream-implementations-c-sealed) +/// and cannot be implemented outside of this crate. +pub trait StorableKind: sealed::StorableKindSeal {} + +/// A terminal [`Storable`] kind. A terminal storable doesn't manage any subkeys, +/// and is the end of the line in a composable collection. +/// +/// An example of a terminal storable is [`Item`], but not [`Column`] or [`Map`]. +pub struct Terminal; + +/// A non-terminal [`Storable`] kind. A non-terminal storable manages subkeys. +/// +/// Some examples of non-terminal storables are [`Column`] and [`Map`]. +pub struct NonTerminal; + +impl StorableKind for Terminal {} +impl StorableKind for NonTerminal {} + +mod sealed { + pub trait StorableKindSeal {} + + impl StorableKindSeal for super::Terminal {} + impl StorableKindSeal for super::NonTerminal {} +} diff --git a/packages/storey/tests/composition.rs b/packages/storey/tests/composition.rs index 59dd251..25f9fb9 100644 --- a/packages/storey/tests/composition.rs +++ b/packages/storey/tests/composition.rs @@ -25,7 +25,7 @@ fn map_of_map() { Some(1337) ); assert_eq!( - storage.get(&[0, 3, 102, 111, 111, 3, 98, 97, 114]), + storage.get(&[0, 3, 102, 111, 111, 98, 97, 114]), Some(1337u64.to_le_bytes().to_vec()) ); assert_eq!( diff --git a/release-plz.toml b/release-plz.toml index 9495192..9587414 100644 --- a/release-plz.toml +++ b/release-plz.toml @@ -18,10 +18,9 @@ commit_parsers = [ { message = "^changed", group = "Changed" }, { message = "^deprecated", group = "Deprecated" }, { message = "^fix", group = "Fixed" }, - { message = "^security", group = "Security" }, + { message = "^sec", group = "Security" }, { message = "^docs", group = "Documentation" }, { message = "^test", group = "Tests" }, - { message = "^.*", group = "Other" }, ] [workspace]