diff --git a/costs/src/lib.rs b/costs/src/lib.rs index 83d29f64..a6867028 100644 --- a/costs/src/lib.rs +++ b/costs/src/lib.rs @@ -93,6 +93,11 @@ pub struct OperationCost { } impl OperationCost { + /// Is Nothing + pub fn is_nothing(&self) -> bool { + self == &Self::default() + } + /// Helper function to build default `OperationCost` with different /// `seek_count`. pub fn with_seek_count(seek_count: u16) -> Self { diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 96f2364e..992cb19e 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -17,7 +17,7 @@ tempfile = { version = "3.10.1", optional = true } bincode = { version = "2.0.0-rc.3" } grovedb-storage = { version = "1.0.0-rc.2", path = "../storage", optional = true } grovedb-visualize = { version = "1.0.0-rc.2", path = "../visualize", optional = true } -hex = { version = "0.4.3", optional = true } +hex = { version = "0.4.3"} itertools = { version = "0.12.1", optional = true } integer-encoding = { version = "4.0.0", optional = true } grovedb-costs = { version = "1.0.0-rc.2", path = "../costs", optional = true } @@ -51,7 +51,6 @@ full = [ "tempfile", "grovedb-storage/rocksdb_storage", "visualize", - "hex", "itertools", "integer-encoding", "grovedb-costs", diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 0a8d573d..7f4521a7 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -274,7 +274,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { let base_path = KeyInfoPath(vec![]); if let Some(estimated_layer_info) = self.paths.get(&base_path) { // Then we have to get the tree - if self.cached_merks.get(&base_path).is_none() { + if !self.cached_merks.contains_key(&base_path) { GroveDb::add_average_case_get_merk_at_path::( &mut cost, &base_path, diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 5bb59dfa..f45bbff7 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -214,7 +214,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { ); // Then we have to get the tree - if self.cached_merks.get(path).is_none() { + if !self.cached_merks.contains(path) { GroveDb::add_worst_case_get_merk_at_path::(&mut cost, path, false); self.cached_merks.insert(path.clone()); } @@ -239,7 +239,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { let base_path = KeyInfoPath(vec![]); if let Some(_estimated_layer_info) = self.paths.get(&base_path) { // Then we have to get the tree - if self.cached_merks.get(&base_path).is_none() { + if !self.cached_merks.contains(&base_path) { GroveDb::add_worst_case_get_merk_at_path::( &mut cost, &base_path, false, ); diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index f987060c..23acf447 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -144,6 +144,16 @@ fn node_to_update( n_keep: n_keep.into(), path_append, }, + crate::Element::Reference( + ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + n_keep, + path_append, + ), + .., + ) => grovedbg_types::Element::UpstreamRootHeightWithParentPathAdditionReference { + n_keep: n_keep.into(), + path_append, + }, crate::Element::Reference( ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), .., diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index b6f75b10..957618d0 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -59,8 +59,12 @@ impl Element { let value = result?; value.ok_or_else(|| { Error::PathKeyNotFound(format!( - "key not found in Merk for get: {}", - hex::encode(key) + "get: key \"{}\" not found in Merk that has a root key [{}] and is of type {}", + hex::encode(key), + merk.root_key() + .map(hex::encode) + .unwrap_or("None".to_string()), + merk.merk_type )) }) }) diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 91d57fe3..e7cb9df1 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -43,12 +43,13 @@ use grovedb_merk::{ #[cfg(feature = "full")] use integer_encoding::VarInt; +#[cfg(any(feature = "full", feature = "verify"))] +use crate::reference_path::{path_from_reference_path_type, ReferencePathType}; #[cfg(any(feature = "full", feature = "verify"))] use crate::{element::SUM_ITEM_COST_SIZE, Element, Error}; #[cfg(feature = "full")] use crate::{ element::{SUM_TREE_COST_SIZE, TREE_COST_SIZE}, - reference_path::{path_from_reference_path_type, ReferencePathType}, ElementFlags, }; @@ -64,8 +65,7 @@ impl Element { } #[cfg(any(feature = "full", feature = "verify"))] - /// Decoded the integer value in the SumItem element type, returns 0 for - /// everything else + /// Decoded the integer value in the SumItem element type pub fn as_sum_item_value(&self) -> Result { match self { Element::SumItem(value, _) => Ok(*value), @@ -73,6 +73,33 @@ impl Element { } } + #[cfg(any(feature = "full", feature = "verify"))] + /// Decoded the integer value in the SumItem element type + pub fn into_sum_item_value(self) -> Result { + match self { + Element::SumItem(value, _) => Ok(value), + _ => Err(Error::WrongElementType("expected a sum item")), + } + } + + #[cfg(any(feature = "full", feature = "verify"))] + /// Decoded the integer value in the SumTree element type + pub fn as_sum_tree_value(&self) -> Result { + match self { + Element::SumTree(_, value, _) => Ok(*value), + _ => Err(Error::WrongElementType("expected a sum tree")), + } + } + + #[cfg(any(feature = "full", feature = "verify"))] + /// Decoded the integer value in the SumTree element type + pub fn into_sum_tree_value(self) -> Result { + match self { + Element::SumTree(_, value, _) => Ok(value), + _ => Err(Error::WrongElementType("expected a sum tree")), + } + } + #[cfg(any(feature = "full", feature = "verify"))] /// Gives the item value in the Item element type pub fn as_item_bytes(&self) -> Result<&[u8], Error> { @@ -91,6 +118,15 @@ impl Element { } } + #[cfg(any(feature = "full", feature = "verify"))] + /// Gives the reference path type in the Reference element type + pub fn into_reference_path_type(self) -> Result { + match self { + Element::Reference(value, ..) => Ok(value), + _ => Err(Error::WrongElementType("expected a reference")), + } + } + #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is a sum tree pub fn is_sum_tree(&self) -> bool { @@ -103,6 +139,12 @@ impl Element { matches!(self, Element::SumTree(..) | Element::Tree(..)) } + #[cfg(any(feature = "full", feature = "verify"))] + /// Check if the element is a reference + pub fn is_reference(&self) -> bool { + matches!(self, Element::Reference(..)) + } + #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is an item pub fn is_item(&self) -> bool { diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index c71bb52f..4c29c400 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -101,7 +101,7 @@ pub enum Element { Item(Vec, Option), /// A reference to an object by its path Reference(ReferencePathType, MaxReferenceHop, Option), - /// A subtree, contains the a prefixed key representing the root of the + /// A subtree, contains the prefixed key representing the root of the /// subtree. Tree(Option>, Option), /// Signed integer value that can be totaled in a sum tree @@ -111,6 +111,18 @@ pub enum Element { SumTree(Option>, SumValue, Option), } +impl Element { + pub fn type_str(&self) -> &str { + match self { + Element::Item(..) => "item", + Element::Reference(..) => "reference", + Element::Tree(..) => "tree", + Element::SumItem(..) => "sum item", + Element::SumTree(..) => "sum tree", + } + } +} + #[cfg(any(feature = "full", feature = "visualize"))] impl fmt::Debug for Element { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index eba5ae1f..c992ba26 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -53,7 +53,7 @@ use crate::{ QueryPathKeyElementTrioResultType, }, }, - util::{merk_optional_tx, storage_context_optional_tx}, + util::{merk_optional_tx, merk_optional_tx_internal_error, storage_context_optional_tx}, Error, PathQuery, TransactionArg, }; #[cfg(any(feature = "full", feature = "verify"))] @@ -563,7 +563,7 @@ impl Element { if !item.is_range() { // this is a query on a key if let QueryItem::Key(key) = item { - let element_res = merk_optional_tx!( + let element_res = merk_optional_tx_internal_error!( &mut cost, storage, subtree_path, diff --git a/grovedb/src/error.rs b/grovedb/src/error.rs index d7f476af..956b5343 100644 --- a/grovedb/src/error.rs +++ b/grovedb/src/error.rs @@ -19,7 +19,7 @@ pub enum Error { InternalError(&'static str), #[error("invalid proof: {0}")] /// Invalid proof - InvalidProof(&'static str), + InvalidProof(String), #[error("invalid input: {0}")] /// Invalid input InvalidInput(&'static str), diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 5cf4fe84..9a0068eb 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -164,7 +164,6 @@ pub mod replication; mod tests; #[cfg(feature = "full")] mod util; -#[cfg(any(feature = "full", feature = "verify"))] mod versioning; #[cfg(feature = "full")] mod visualize; @@ -336,11 +335,11 @@ impl GroveDb { /// Opens a Merk at given path for with direct write access. Intended for /// replication purposes. - fn open_merk_for_replication<'db, 'b, B>( + fn open_merk_for_replication<'tx, 'db: 'tx, 'b, B>( &'db self, path: SubtreePath<'b, B>, - tx: &'db Transaction, - ) -> Result>, Error> + tx: &'tx Transaction<'db>, + ) -> Result>, Error> where B: AsRef<[u8]> + 'b, { diff --git a/grovedb/src/operations.rs b/grovedb/src/operations.rs index af637f42..9864b0bc 100644 --- a/grovedb/src/operations.rs +++ b/grovedb/src/operations.rs @@ -40,3 +40,6 @@ pub mod insert; pub(crate) mod is_empty_tree; #[cfg(any(feature = "full", feature = "verify"))] pub mod proof; + +#[cfg(feature = "full")] +pub use get::{QueryItemOrSumReturnType, MAX_REFERENCE_HOPS}; diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index 69512567..12700106 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -32,6 +32,8 @@ mod average_case; #[cfg(feature = "full")] mod query; +#[cfg(feature = "full")] +pub use query::QueryItemOrSumReturnType; #[cfg(feature = "estimated_costs")] mod worst_case; diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 29a581d9..7e29b233 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -36,6 +36,8 @@ use grovedb_costs::{ #[cfg(feature = "full")] use integer_encoding::VarInt; +#[cfg(feature = "full")] +use crate::element::SumValue; use crate::{element::QueryOptions, query_result_type::PathKeyOptionalElementTrio}; #[cfg(feature = "full")] use crate::{ @@ -44,6 +46,16 @@ use crate::{ Element, Error, GroveDb, PathQuery, TransactionArg, }; +#[cfg(feature = "full")] +#[derive(Debug, Eq, PartialEq, Clone)] +/// A return type for query_item_value_or_sum +pub enum QueryItemOrSumReturnType { + /// an Item in serialized form + ItemData(Vec), + /// A sum item or a sum tree value + SumValue(SumValue), +} + #[cfg(feature = "full")] impl GroveDb { /// Encoded query for multiple path queries @@ -190,10 +202,8 @@ where { )), } } - Element::Item(..) | Element::SumItem(..) => Ok(element), - Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidQuery( - "path_queries can only refer to items and references", - )), + Element::Item(..) | Element::SumItem(..) | Element::SumTree(..) => Ok(element), + Element::Tree(..) => Err(Error::InvalidQuery("path_queries can not refer to trees")), } } @@ -309,6 +319,94 @@ where { Ok((results, skipped)).wrap_with_cost(cost) } + /// Queries the backing store and returns element items by their value, + /// Sum Items are returned + pub fn query_item_value_or_sum( + &self, + path_query: &PathQuery, + allow_cache: bool, + decrease_limit_on_range_with_no_sub_elements: bool, + error_if_intermediate_path_tree_not_present: bool, + transaction: TransactionArg, + ) -> CostResult<(Vec, u16), Error> { + let mut cost = OperationCost::default(); + + let (elements, skipped) = cost_return_on_error!( + &mut cost, + self.query_raw( + path_query, + allow_cache, + decrease_limit_on_range_with_no_sub_elements, + error_if_intermediate_path_tree_not_present, + QueryResultType::QueryElementResultType, + transaction + ) + ); + + let results_wrapped = elements + .into_iterator() + .map(|result_item| match result_item { + QueryResultElement::ElementResultItem(element) => { + match element { + Element::Reference(reference_path, ..) => { + match reference_path { + ReferencePathType::AbsolutePathReference(absolute_path) => { + // While `map` on iterator is lazy, we should accumulate costs + // even if `collect` will + // end in `Err`, so we'll use + // external costs accumulator instead of + // returning costs from `map` call. + let maybe_item = self + .follow_reference( + absolute_path.as_slice().into(), + allow_cache, + transaction, + ) + .unwrap_add_cost(&mut cost)?; + + match maybe_item { + Element::Item(item, _) => { + Ok(QueryItemOrSumReturnType::ItemData(item)) + } + Element::SumItem(sum_value, _) => { + Ok(QueryItemOrSumReturnType::SumValue(sum_value)) + } + Element::SumTree(_, sum_value, _) => { + Ok(QueryItemOrSumReturnType::SumValue(sum_value)) + } + _ => Err(Error::InvalidQuery( + "the reference must result in an item", + )), + } + } + _ => Err(Error::CorruptedCodeExecution( + "reference after query must have absolute paths", + )), + } + } + Element::Item(item, _) => Ok(QueryItemOrSumReturnType::ItemData(item)), + Element::SumItem(sum_value, _) => { + Ok(QueryItemOrSumReturnType::SumValue(sum_value)) + } + Element::SumTree(_, sum_value, _) => { + Ok(QueryItemOrSumReturnType::SumValue(sum_value)) + } + Element::Tree(..) => Err(Error::InvalidQuery( + "path_queries can only refer to items, sum items, references and sum \ + trees", + )), + } + } + _ => Err(Error::CorruptedCodeExecution( + "query returned incorrect result type", + )), + }) + .collect::, Error>>(); + + let results = cost_return_on_error_no_add!(&cost, results_wrapped); + Ok((results, skipped)).wrap_with_cost(cost) + } + /// Retrieves only SumItem elements that match a path query pub fn query_sums( &self, diff --git a/grovedb/src/operations/proof/util.rs b/grovedb/src/operations/proof/util.rs index 05e868a3..82e8c585 100644 --- a/grovedb/src/operations/proof/util.rs +++ b/grovedb/src/operations/proof/util.rs @@ -26,6 +26,7 @@ // IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +use std::fmt; #[cfg(any(feature = "full", feature = "verify"))] use std::io::Read; #[cfg(feature = "full")] @@ -38,9 +39,9 @@ use grovedb_merk::{ #[cfg(any(feature = "full", feature = "verify"))] use integer_encoding::{VarInt, VarIntReader}; -use crate::operations::proof::verify::ProvedKeyValues; #[cfg(any(feature = "full", feature = "verify"))] use crate::Error; +use crate::{operations::proof::verify::ProvedKeyValues, reference_path::ReferencePathType}; #[cfg(any(feature = "full", feature = "verify"))] pub const EMPTY_TREE_HASH: [u8; 32] = [0; 32]; @@ -60,6 +61,21 @@ pub enum ProofTokenType { Invalid, } +#[cfg(any(feature = "full", feature = "verify"))] +impl fmt::Display for ProofTokenType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let variant_str = match self { + ProofTokenType::Merk => "Merk", + ProofTokenType::SizedMerk => "SizedMerk", + ProofTokenType::EmptyTree => "EmptyTree", + ProofTokenType::AbsentPath => "AbsentPath", + ProofTokenType::PathInfo => "PathInfo", + ProofTokenType::Invalid => "Invalid", + }; + write!(f, "{}", variant_str) + } +} + #[cfg(any(feature = "full", feature = "verify"))] impl From for u8 { fn from(proof_token_type: ProofTokenType) -> Self { @@ -88,6 +104,20 @@ impl From for ProofTokenType { } } +#[cfg(any(feature = "full", feature = "verify"))] +impl ProofTokenType { + pub fn u8_to_display(val: u8) -> String { + match val { + 0x01 => "merk".to_string(), + 0x02 => "sized merk".to_string(), + 0x04 => "empty tree".to_string(), + 0x05 => "absent path".to_string(), + 0x06 => "path info".to_string(), + v => format!("invalid proof token {}", v), + } + } +} + #[cfg(any(feature = "full", feature = "verify"))] #[derive(Debug)] // TODO: possibility for a proof writer?? @@ -151,7 +181,7 @@ impl<'a> ProofReader<'a> { fn read_length_data(&mut self) -> Result { self.proof_data .read_varint() - .map_err(|_| Error::InvalidProof("expected length data")) + .map_err(|_| Error::InvalidProof("expected length data".to_string())) } /// Read proof with optional type @@ -175,7 +205,7 @@ impl<'a> ProofReader<'a> { proof_token_type, proof, Some(key.ok_or(Error::InvalidProof( - "key must exist for verbose merk proofs", + "key must exist for verbose merk proofs".to_string(), ))?), )) } @@ -207,8 +237,11 @@ impl<'a> ProofReader<'a> { self.read_into_slice(&mut data_type)?; if let Some(expected_data_type) = expected_data_type_option { - if data_type != [expected_data_type] { - return Err(Error::InvalidProof("wrong data_type")); + if data_type[0] != expected_data_type { + return Err(Error::InvalidProof(format!( + "wrong data_type, expected {}, got {}", + expected_data_type, data_type[0] + ))); } } @@ -242,7 +275,9 @@ impl<'a> ProofReader<'a> { (proof, key) } else { - return Err(Error::InvalidProof("expected merk or sized merk proof")); + return Err(Error::InvalidProof( + "expected merk or sized merk proof".to_string(), + )); }; Ok((proof_token_type, proof, key)) @@ -254,7 +289,10 @@ impl<'a> ProofReader<'a> { self.read_into_slice(&mut data_type)?; if data_type != [Into::::into(ProofTokenType::PathInfo)] { - return Err(Error::InvalidProof("wrong data_type, expected path_info")); + return Err(Error::InvalidProof(format!( + "wrong data_type, expected path_info, got {}", + ProofTokenType::u8_to_display(data_type[0]) + ))); } let mut path = vec![]; diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index baea8735..7a347c15 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -223,9 +223,11 @@ impl GroveDb { )?; let (new_root_hash, new_elements) = Self::verify_subset_query(proof, &new_path_query)?; if new_root_hash != last_root_hash { - return Err(Error::InvalidProof( - "root hash for different path queries do no match", - )); + return Err(Error::InvalidProof(format!( + "root hash for different path queries do no match, first is {}, this one is {}", + hex::encode(last_root_hash), + hex::encode(new_root_hash) + ))); } results.push(new_elements); } @@ -276,7 +278,8 @@ impl ProofVerifier { } else if original_path.len() > path_slices.len() { // TODO: can we relax this constraint return Err(Error::InvalidProof( - "original path query path must not be greater than the subset path len", + "original path query path must not be greater than the subset path len" + .to_string(), )); } else { let original_path_in_new_path = original_path @@ -285,7 +288,7 @@ impl ProofVerifier { if !original_path_in_new_path { return Err(Error::InvalidProof( - "the original path should be a subset of the subset path", + "the original path should be a subset of the subset path".to_string(), )); } else { // We construct a new path query @@ -373,7 +376,7 @@ impl ProofVerifier { last_root_hash = proof_root_hash; let children = children.ok_or(Error::InvalidProof( - "MERK_PROOF always returns a result set", + "MERK_PROOF always returns a result set".to_string(), ))?; for proved_path_key_value in children { @@ -474,7 +477,8 @@ impl ProofVerifier { // which is invalid as there exists a subquery value return Err(Error::InvalidProof( "expected unsized proof for subquery path as subquery \ - value exists", + value exists" + .to_string(), )); } let subquery_path_result_set = @@ -517,9 +521,11 @@ impl ProofVerifier { .to_owned(); if combined_child_hash != expected_combined_child_hash { - return Err(Error::InvalidProof( - "child hash doesn't match the expected hash", - )); + return Err(Error::InvalidProof(format!( + "child hash {} doesn't match the expected hash {}", + hex::encode(combined_child_hash), + hex::encode(expected_combined_child_hash) + ))); } } _ => { @@ -552,10 +558,13 @@ impl ProofVerifier { ProofTokenType::EmptyTree => { last_root_hash = EMPTY_TREE_HASH; } - _ => { + t => { // execute_subquery_proof only expects proofs for merk trees // root proof is handled separately - return Err(Error::InvalidProof("wrong proof type")); + return Err(Error::InvalidProof(format!( + "wrong proof type, expected sized merk, merk or empty tree but got {}", + t + ))); } } Ok(last_root_hash) @@ -576,13 +585,14 @@ impl ProofVerifier { *expected_child_hash = subquery_path_result_set[0].proof; *current_value_bytes = subquery_path_result_set[0].value.to_owned(); } - _ => { + e => { // the means that the subquery path pointed to a non tree // element, this is not valid as you cannot apply the // the subquery value to non tree items - return Err(Error::InvalidProof( - "subquery path cannot point to non tree element", - )); + return Err(Error::InvalidProof(format!( + "subquery path cannot point to non tree element, got {}", + e.type_str() + ))); } } Ok(()) @@ -607,9 +617,10 @@ impl ProofVerifier { proof_reader.read_next_proof(current_path.last().unwrap_or(&Default::default()))?; // intermediate proofs are all going to be unsized merk proofs if proof_token_type != ProofTokenType::Merk { - return Err(Error::InvalidProof( - "expected MERK proof type for intermediate subquery path keys", - )); + return Err(Error::InvalidProof(format!( + "expected MERK proof type for intermediate subquery path keys, got {}", + proof_token_type + ))); } match proof_token_type { ProofTokenType::Merk => { @@ -645,9 +656,11 @@ impl ProofVerifier { .to_owned(); if combined_child_hash != *expected_root_hash { - return Err(Error::InvalidProof( - "child hash doesn't match the expected hash", - )); + return Err(Error::InvalidProof(format!( + "child hash {} doesn't match the expected hash {}", + hex::encode(combined_child_hash), + hex::encode(expected_root_hash) + ))); } // after confirming they are linked use the latest hash values for subsequent @@ -658,10 +671,11 @@ impl ProofVerifier { &result_set.expect("confirmed is some"), )?; } - _ => { - return Err(Error::InvalidProof( - "expected merk of sized merk proof type for subquery path", - )); + t => { + return Err(Error::InvalidProof(format!( + "expected merk of sized merk proof type for subquery path, got {}", + t + ))); } } } @@ -669,9 +683,10 @@ impl ProofVerifier { let (proof_token_type, subkey_proof) = proof_reader.read_next_proof(current_path.last().unwrap_or(&Default::default()))?; if proof_token_type != expected_proof_token_type { - return Err(Error::InvalidProof( - "unexpected proof type for subquery path", - )); + return Err(Error::InvalidProof(format!( + "unexpected proof type for subquery path, expected {}, got {}", + expected_proof_token_type, proof_token_type + ))); } match proof_token_type { @@ -691,9 +706,10 @@ impl ProofVerifier { Ok((verification_result.0, verification_result.1, false)) } - _ => Err(Error::InvalidProof( - "expected merk or sized merk proof type for subquery path", - )), + t => Err(Error::InvalidProof(format!( + "expected merk or sized merk proof type for subquery path, got {}", + t + ))), } } @@ -720,14 +736,18 @@ impl ProofVerifier { if Some(combined_hash) != expected_child_hash { return Err(Error::InvalidProof( "proof invalid: could not verify empty subtree while generating absent \ - path proof", + path proof" + .to_string(), )); } else { last_result_set = vec![]; break; } } else if proof_token_type != ProofTokenType::Merk { - return Err(Error::InvalidProof("expected a merk proof for absent path")); + return Err(Error::InvalidProof(format!( + "expected a merk proof for absent path, got {}", + proof_token_type + ))); } let mut child_query = Query::new(); @@ -743,18 +763,22 @@ impl ProofVerifier { Vec::new(), )?; - if expected_child_hash.is_none() { - root_key_hash = Some(proof_result.0); - } else { + if let Some(expected_child_hash) = expected_child_hash { let combined_hash = combine_hash( value_hash_fn(last_result_set[0].value.as_slice()).value(), &proof_result.0, ) .value() .to_owned(); - if Some(combined_hash) != expected_child_hash { - return Err(Error::InvalidProof("proof invalid: invalid parent")); + if combined_hash != expected_child_hash { + return Err(Error::InvalidProof(format!( + "proof invalid: invalid parent, expected {}, got {}", + hex::encode(expected_child_hash), + hex::encode(combined_hash) + ))); } + } else { + root_key_hash = Some(proof_result.0); } last_result_set = proof_result @@ -768,9 +792,10 @@ impl ProofVerifier { let elem = Element::deserialize(last_result_set[0].value.as_slice())?; let child_hash = match elem { Element::Tree(..) | Element::SumTree(..) => Ok(Some(last_result_set[0].proof)), - _ => Err(Error::InvalidProof( - "intermediate proofs should be for trees", - )), + e => Err(Error::InvalidProof(format!( + "intermediate proofs should be for trees, got {}", + e.type_str() + ))), }?; expected_child_hash = child_hash; } @@ -779,10 +804,14 @@ impl ProofVerifier { if let Some(hash) = root_key_hash { Ok(hash) } else { - Err(Error::InvalidProof("proof invalid: no non root tree found")) + Err(Error::InvalidProof( + "proof invalid: no non root tree found".to_string(), + )) } } else { - Err(Error::InvalidProof("proof invalid: path not absent")) + Err(Error::InvalidProof( + "proof invalid: path not absent".to_string(), + )) } } @@ -802,7 +831,10 @@ impl ProofVerifier { let (proof_token_type, parent_merk_proof) = proof_reader.read_next_proof(path_slice.last().unwrap_or(&Default::default()))?; if proof_token_type != ProofTokenType::Merk { - return Err(Error::InvalidProof("wrong data_type expected merk proof")); + return Err(Error::InvalidProof(format!( + "wrong data_type expected Merk Proof, got {}", + proof_token_type + ))); } let mut parent_query = Query::new(); @@ -821,15 +853,18 @@ impl ProofVerifier { .1 .expect("MERK_PROOF always returns a result set"); if result_set.is_empty() || &result_set[0].key != key { - return Err(Error::InvalidProof("proof invalid: invalid parent")); + return Err(Error::InvalidProof( + "proof invalid: invalid parent".to_string(), + )); } let elem = Element::deserialize(result_set[0].value.as_slice())?; let child_hash = match elem { Element::Tree(..) | Element::SumTree(..) => Ok(result_set[0].proof), - _ => Err(Error::InvalidProof( - "intermediate proofs should be for trees", - )), + t => Err(Error::InvalidProof(format!( + "intermediate proofs should be for trees, got {}", + t.type_str() + ))), }?; let combined_root_hash = combine_hash( @@ -839,9 +874,11 @@ impl ProofVerifier { .value() .to_owned(); if child_hash != combined_root_hash { - return Err(Error::InvalidProof( - "Bad path: tree hash does not have expected hash", - )); + return Err(Error::InvalidProof(format!( + "Bad path: tree hash does not have expected hash, got {}, expected {}", + hex::encode(child_hash), + hex::encode(combined_root_hash) + ))); } *expected_root_hash = proof_result.0; @@ -876,7 +913,7 @@ impl ProofVerifier { .unwrap() .map_err(|e| { eprintln!("{e}"); - Error::InvalidProof("invalid proof verification parameters") + Error::InvalidProof("invalid proof verification parameters".to_string()) })?; // convert the result set to proved_path_key_values diff --git a/grovedb/src/query_result_type.rs b/grovedb/src/query_result_type.rs index 37de6c0d..289ffb26 100644 --- a/grovedb/src/query_result_type.rs +++ b/grovedb/src/query_result_type.rs @@ -49,6 +49,7 @@ pub enum QueryResultType { } /// Query result elements +#[derive(Debug, Clone)] pub struct QueryResultElements { /// Elements pub elements: Vec, @@ -187,6 +188,85 @@ impl QueryResultElements { }) .collect() } + + /// To last path to keys btree map + /// This is useful if for example the element is a sum item and isn't + /// important Used in Platform Drive for getting voters for multiple + /// contenders + pub fn to_last_path_to_keys_btree_map(self) -> BTreeMap> { + let mut map: BTreeMap, Vec> = BTreeMap::new(); + + for result_item in self.elements.into_iter() { + if let QueryResultElement::PathKeyElementTrioResultItem((mut path, key, _)) = + result_item + { + if let Some(last) = path.pop() { + map.entry(last).or_insert_with(Vec::new).push(key); + } + } + } + + map + } + + /// To last path to key, elements btree map + pub fn to_last_path_to_key_elements_btree_map(self) -> BTreeMap> { + let mut map: BTreeMap, BTreeMap> = BTreeMap::new(); + + for result_item in self.elements.into_iter() { + if let QueryResultElement::PathKeyElementTrioResultItem((mut path, key, element)) = + result_item + { + if let Some(last) = path.pop() { + map.entry(last) + .or_insert_with(BTreeMap::new) + .insert(key, element); + } + } + } + + map + } + + /// To last path to elements btree map + /// This is useful if the key is not import + pub fn to_last_path_to_elements_btree_map(self) -> BTreeMap> { + let mut map: BTreeMap, Vec> = BTreeMap::new(); + + for result_item in self.elements.into_iter() { + if let QueryResultElement::PathKeyElementTrioResultItem((mut path, _, element)) = + result_item + { + if let Some(last) = path.pop() { + map.entry(last).or_insert_with(Vec::new).push(element); + } + } + } + + map + } + + /// To last path to keys btree map + /// This is useful if for example the element is a sum item and isn't + /// important Used in Platform Drive for getting voters for multiple + /// contenders + pub fn to_previous_of_last_path_to_keys_btree_map(self) -> BTreeMap> { + let mut map: BTreeMap, Vec> = BTreeMap::new(); + + for result_item in self.elements.into_iter() { + if let QueryResultElement::PathKeyElementTrioResultItem((mut path, key, _)) = + result_item + { + if let Some(_) = path.pop() { + if let Some(last) = path.pop() { + map.entry(last).or_insert_with(Vec::new).push(key); + } + } + } + } + + map + } } impl Default for QueryResultElements { @@ -196,6 +276,7 @@ impl Default for QueryResultElements { } /// Query result element +#[derive(Debug, Clone)] pub enum QueryResultElement { /// Element result item ElementResultItem(Element), diff --git a/grovedb/src/reference_path.rs b/grovedb/src/reference_path.rs index 52f07eb8..38c3f147 100644 --- a/grovedb/src/reference_path.rs +++ b/grovedb/src/reference_path.rs @@ -28,7 +28,7 @@ //! Space efficient methods for referencing other elements in GroveDB -#[cfg(feature = "full")] +#[cfg(any(feature = "full", feature = "verify"))] use std::fmt; use bincode::{Decode, Encode}; @@ -37,7 +37,7 @@ use grovedb_visualize::visualize_to_vec; #[cfg(feature = "full")] use integer_encoding::VarInt; -#[cfg(feature = "full")] +#[cfg(any(feature = "full", feature = "verify"))] use crate::Error; #[cfg(any(feature = "full", feature = "verify"))] @@ -54,6 +54,16 @@ pub enum ReferencePathType { /// path [p, q] result = [a, b, p, q] UpstreamRootHeightReference(u8, Vec>), + /// This is very similar to the UpstreamRootHeightReference, however + /// it appends to the absolute path when resolving the parent of the + /// reference. If the reference is stored at 15/9/80/7 then 80 will be + /// appended to what we are referring to. For example if we have the + /// reference at [a, b, c, d, e, f] (e is the parent path here) and we + /// have in the UpstreamRootHeightWithParentPathAdditionReference the + /// height set to 2 and the addon path set to [x, y], we would get as a + /// result [a, b, x, y, e] + UpstreamRootHeightWithParentPathAdditionReference(u8, Vec>), + /// This discards the last n elements from the current path and appends a /// new path to the subpath. If current path is [a, b, c, d] and we /// discard the last element, subpath = [a, b, c] we can then append @@ -76,7 +86,31 @@ pub enum ReferencePathType { SiblingReference(Vec), } -#[cfg(feature = "full")] +#[cfg(any(feature = "full", feature = "verify"))] +impl ReferencePathType { + /// Given the reference path type and the current qualified path (path+key), + /// this computes the absolute path of the item the reference is pointing + /// to. + pub fn absolute_path_using_current_qualified_path>( + self, + current_qualified_path: &[B], + ) -> Result>, Error> { + path_from_reference_qualified_path_type(self, current_qualified_path) + } + + /// Given the reference path type, the current path and the terminal key, + /// this computes the absolute path of the item the reference is + /// pointing to. + pub fn absolute_path>( + self, + current_path: &[B], + current_key: Option<&[u8]>, + ) -> Result>, Error> { + path_from_reference_path_type(self, current_path, current_key) + } +} + +#[cfg(any(feature = "full", feature = "visualize"))] impl fmt::Debug for ReferencePathType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut v = Vec::new(); @@ -86,7 +120,7 @@ impl fmt::Debug for ReferencePathType { } } -#[cfg(feature = "full")] +#[cfg(any(feature = "full", feature = "verify"))] /// Given the reference path type and the current qualified path (path+key), /// this computes the absolute path of the item the reference is pointing to. pub fn path_from_reference_qualified_path_type>( @@ -103,7 +137,7 @@ pub fn path_from_reference_qualified_path_type>( } } -#[cfg(feature = "full")] +#[cfg(any(feature = "full", feature = "verify"))] /// Given the reference path type, the current path and the terminal key, this /// computes the absolute path of the item the reference is pointing to. pub fn path_from_reference_path_type>( @@ -130,6 +164,25 @@ pub fn path_from_reference_path_type>( subpath_as_vec.append(&mut path); Ok(subpath_as_vec) } + ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + no_of_elements_to_keep, + mut path, + ) => { + if usize::from(no_of_elements_to_keep) > current_path.len() || current_path.len() == 0 { + return Err(Error::InvalidInput( + "reference stored path cannot satisfy reference constraints", + )); + } + let last = current_path.last().unwrap().as_ref().to_vec(); + let current_path_iter = current_path.iter(); + let mut subpath_as_vec = current_path_iter + .take(no_of_elements_to_keep as usize) + .map(|x| x.as_ref().to_vec()) + .collect::>(); + subpath_as_vec.append(&mut path); + subpath_as_vec.push(last); + Ok(subpath_as_vec) + } // Discard the last n elements from current path, append new path to subpath ReferencePathType::UpstreamFromElementHeightReference( @@ -224,6 +277,7 @@ impl ReferencePathType { .sum::() } ReferencePathType::UpstreamRootHeightReference(_, path) + | ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference(_, path) | ReferencePathType::UpstreamFromElementHeightReference(_, path) => { 1 + 1 + path @@ -266,6 +320,27 @@ mod tests { ); } + #[test] + fn test_upstream_root_height_with_parent_addition_reference() { + let stored_path = vec![b"a".as_ref(), b"b".as_ref(), b"m".as_ref()]; + // selects the first 2 elements from the stored path and appends the new path. + let ref1 = ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + 2, + vec![b"c".to_vec(), b"d".to_vec()], + ); + let final_path = path_from_reference_path_type(ref1, &stored_path, None).unwrap(); + assert_eq!( + final_path, + vec![ + b"a".to_vec(), + b"b".to_vec(), + b"c".to_vec(), + b"d".to_vec(), + b"m".to_vec() + ] + ); + } + #[test] fn test_upstream_from_element_height_reference() { let stored_path = vec![b"a".as_ref(), b"b".as_ref(), b"m".as_ref()]; diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index cbedc9af..d05f2396 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -149,6 +149,117 @@ macro_rules! storage_context_with_parent_optional_tx { }; } +/// Macro to execute same piece of code on different storage contexts +/// (transactional or not) using path argument. +macro_rules! storage_context_with_parent_optional_tx_internal_error { + ( + &mut $cost:ident, + $db:expr, + $path:expr, + $batch:expr, + $transaction:ident, + $storage:ident, + $root_key:ident, + $is_sum_tree:ident, + { $($body:tt)* } + ) => { + { + use ::grovedb_storage::Storage; + if let Some(tx) = $transaction { + let $storage = $db + .get_transactional_storage_context($path.clone(), $batch, tx) + .unwrap_add_cost(&mut $cost); + if let Some((parent_path, parent_key)) = $path.derive_parent() { + let parent_storage = $db + .get_transactional_storage_context(parent_path, $batch, tx) + .unwrap_add_cost(&mut $cost); + let result = Element::get_from_storage(&parent_storage, parent_key) + .map_err(|e| { + Error::PathParentLayerNotFound( + format!( + "could not get key for parent of subtree optional on tx: {}", + e + ) + ) + }).unwrap_add_cost(&mut $cost); + match result { + Ok(element) => { + match element { + Element::Tree(root_key, _) => { + let $root_key = root_key; + let $is_sum_tree = false; + $($body)* + } + Element::SumTree(root_key, ..) => { + let $root_key = root_key; + let $is_sum_tree = true; + $($body)* + } + _ => { + return Err(Error::CorruptedData( + "parent is not a tree" + .to_owned(), + )).wrap_with_cost($cost); + } + } + }, + Err(e) => Err(e), + } + } else { + return Err(Error::CorruptedData( + "path is empty".to_owned(), + )).wrap_with_cost($cost); + } + } else { + let $storage = $db + .get_storage_context($path.clone(), $batch).unwrap_add_cost(&mut $cost); + if let Some((parent_path, parent_key)) = $path.derive_parent() { + let parent_storage = $db.get_storage_context( + parent_path, + $batch + ).unwrap_add_cost(&mut $cost); + let result = Element::get_from_storage(&parent_storage, parent_key) + .map_err(|e| { + Error::PathParentLayerNotFound( + format!( + "could not get key for parent of subtree optional no tx: {}", + e + ) + ) + }).unwrap_add_cost(&mut $cost); + match result { + Ok(element) => { + match element { + Element::Tree(root_key, _) => { + let $root_key = root_key; + let $is_sum_tree = false; + $($body)* + } + Element::SumTree(root_key, ..) => { + let $root_key = root_key; + let $is_sum_tree = true; + $($body)* + } + _ => { + return Err(Error::CorruptedData( + "parent is not a tree" + .to_owned(), + )).wrap_with_cost($cost); + } + } + }, + Err(e) => Err(e), + } + } else { + return Err(Error::CorruptedData( + "path is empty".to_owned(), + )).wrap_with_cost($cost); + } + } + } + }; +} + /// Macro to execute same piece of code on different storage contexts with /// empty prefix. macro_rules! meta_storage_context_optional_tx { @@ -245,6 +356,76 @@ macro_rules! merk_optional_tx { }; } +/// Macro to execute same piece of code on Merk with varying storage +/// contexts. +macro_rules! merk_optional_tx_internal_error { + ( + &mut $cost:ident, + $db:expr, + $path:expr, + $batch:expr, + $transaction:ident, + $subtree:ident, + { $($body:tt)* } + ) => { + if $path.is_root() { + use crate::util::storage_context_optional_tx; + storage_context_optional_tx!( + $db, + ::grovedb_path::SubtreePath::empty(), + $batch, + $transaction, + storage, + { + let $subtree = cost_return_on_error!( + &mut $cost, + ::grovedb_merk::Merk::open_base( + storage.unwrap_add_cost(&mut $cost), + false, + Some(&Element::value_defined_cost_for_serialized_value) + ).map(|merk_res| + merk_res + .map_err(|_| crate::Error::CorruptedData( + "cannot open a subtree".to_owned() + )) + ) + ); + $($body)* + }) + } else { + use crate::util::storage_context_with_parent_optional_tx_internal_error; + storage_context_with_parent_optional_tx_internal_error!( + &mut $cost, + $db, + $path, + $batch, + $transaction, + storage, + root_key, + is_sum_tree, + { + #[allow(unused_mut)] + let mut $subtree = cost_return_on_error!( + &mut $cost, + ::grovedb_merk::Merk::open_layered_with_root_key( + storage, + root_key, + is_sum_tree, + Some(&Element::value_defined_cost_for_serialized_value), + ).map(|merk_res| + merk_res + .map_err(|_| crate::Error::CorruptedData( + "cannot open a subtree".to_owned() + )) + ) + ); + $($body)* + } + ) + } + }; +} + /// Macro to execute same piece of code on Merk with varying storage /// contexts. macro_rules! merk_optional_tx_path_not_empty { @@ -331,8 +512,10 @@ macro_rules! root_merk_optional_tx { } pub(crate) use merk_optional_tx; +pub(crate) use merk_optional_tx_internal_error; pub(crate) use merk_optional_tx_path_not_empty; pub(crate) use meta_storage_context_optional_tx; pub(crate) use root_merk_optional_tx; pub(crate) use storage_context_optional_tx; pub(crate) use storage_context_with_parent_optional_tx; +pub(crate) use storage_context_with_parent_optional_tx_internal_error; diff --git a/grovedb/src/visualize.rs b/grovedb/src/visualize.rs index 6f1f1c0d..9eb1c00b 100644 --- a/grovedb/src/visualize.rs +++ b/grovedb/src/visualize.rs @@ -130,6 +130,21 @@ impl Visualize for ReferencePathType { .as_bytes(), )?; } + ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + height, + end_path, + ) => { + drawer.write(b"upstream root height with parent path addition reference: ")?; + drawer.write(format!("[height: {height}").as_bytes())?; + drawer.write( + end_path + .iter() + .map(hex::encode) + .collect::>() + .join("/") + .as_bytes(), + )?; + } ReferencePathType::UpstreamFromElementHeightReference(up, end_path) => { drawer.write(b"upstream from element reference: ")?; drawer.write(format!("[up: {up}").as_bytes())?; diff --git a/grovedbg-types/src/lib.rs b/grovedbg-types/src/lib.rs index ff7a4127..dacc4255 100644 --- a/grovedbg-types/src/lib.rs +++ b/grovedbg-types/src/lib.rs @@ -44,6 +44,10 @@ pub enum Element { n_keep: u32, path_append: Vec, }, + UpstreamRootHeightWithParentPathAdditionReference { + n_keep: u32, + path_append: Vec, + }, UpstreamFromElementHeightReference { n_remove: u32, path_append: Vec, diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index b92222ac..1453d708 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -55,7 +55,7 @@ pub type AverageFlagsSize = u32; pub type Weight = u8; #[cfg(feature = "full")] -#[derive(Clone, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] /// Estimated number of sum trees #[derive(Default)] pub enum EstimatedSumTrees { @@ -91,7 +91,7 @@ impl EstimatedSumTrees { } #[cfg(feature = "full")] -#[derive(Clone, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] /// Estimated layer sizes pub enum EstimatedLayerSizes { /// All subtrees @@ -259,7 +259,7 @@ pub type EstimatedLevelNumber = u32; pub type EstimatedToBeEmpty = bool; #[cfg(feature = "full")] -#[derive(Clone, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] /// Information on an estimated layer pub struct EstimatedLayerInformation { /// Is sum tree? @@ -274,7 +274,7 @@ pub struct EstimatedLayerInformation { impl EstimatedLayerInformation {} #[cfg(feature = "full")] -#[derive(Clone, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] /// Estimated elements and level number of a layer pub enum EstimatedLayerCount { /// Potentially at max elements diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 0a0b805e..28ce3f43 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -220,6 +220,17 @@ pub enum MerkType { LayeredMerk, } +impl fmt::Display for MerkType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let description = match self { + MerkType::StandaloneMerk => "StandaloneMerk", + MerkType::BaseMerk => "BaseMerk", + MerkType::LayeredMerk => "LayeredMerk", + }; + write!(f, "{}", description) + } +} + impl MerkType { /// Returns bool pub(crate) fn requires_root_storage_update(&self) -> bool { diff --git a/merk/src/proofs/encoding.rs b/merk/src/proofs/encoding.rs index b0e31484..d0395fe7 100644 --- a/merk/src/proofs/encoding.rs +++ b/merk/src/proofs/encoding.rs @@ -998,7 +998,7 @@ mod test { #[test] fn decode_multiple_child() { let bytes = [0x11, 0x11, 0x11, 0x10]; - let mut decoder = Decoder { + let decoder = Decoder { bytes: &bytes, offset: 0, }; diff --git a/merk/src/proofs/query/mod.rs b/merk/src/proofs/query/mod.rs index 9d485564..29296efc 100644 --- a/merk/src/proofs/query/mod.rs +++ b/merk/src/proofs/query/mod.rs @@ -44,7 +44,7 @@ mod verify; #[cfg(any(feature = "full", feature = "verify"))] use std::cmp::Ordering; -use std::collections::HashSet; +use std::{collections::HashSet, ops::RangeFull}; #[cfg(any(feature = "full", feature = "verify"))] use grovedb_costs::{cost_return_on_error, CostContext, CostResult, CostsExt, OperationCost};