From 04319be5910567a492cb08db024088b019cec5fb Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Wed, 11 Oct 2023 15:10:39 +0100 Subject: [PATCH 1/9] update op ordering --- grovedb/src/batch/mod.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 9724ea7b..a833a4da 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -175,10 +175,22 @@ impl PartialOrd for Op { impl Ord for Op { fn cmp(&self, other: &Self) -> Ordering { match (self, other) { - (Op::Delete, Op::Insert { .. }) => Ordering::Less, - (Op::Delete, Op::Replace { .. }) => Ordering::Less, - (Op::Insert { .. }, Op::Delete) => Ordering::Greater, - (Op::Replace { .. }, Op::Delete) => Ordering::Greater, + // deal with relationship between delete ops + (Op::DeleteTree, Op::DeleteSumTree) | (Op::DeleteSumTree, Op::DeleteTree) => { + Ordering::Equal + } + (Op::DeleteTree, Op::Delete) => Ordering::Less, + (Op::Delete, Op::DeleteTree) => Ordering::Greater, + (Op::DeleteSumTree, Op::Delete) => Ordering::Less, + (Op::Delete, Op::DeleteSumTree) => Ordering::Greater, + + // deal with the relationship between delete and other ops + // delete should always happen first + (Op::Delete, _) => Ordering::Less, + (Op::DeleteSumTree, _) => Ordering::Less, + (Op::DeleteTree, _) => Ordering::Less, + + // all insert operations are considered equal _ => Ordering::Equal, } } From e46e98005f04860f82d1023d0b84c7118dea720e Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Thu, 26 Oct 2023 13:55:12 +0100 Subject: [PATCH 2/9] prevent panic in merk proof construction --- grovedb/src/operations/proof/generate.rs | 11 +++++++---- grovedb/src/tests/query_tests.rs | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 82852359..fad48d1d 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -510,10 +510,13 @@ impl GroveDb { let mut cost = OperationCost::default(); - let mut proof_result = subtree - .prove_without_encoding(query.clone(), limit_offset.0, limit_offset.1) - .unwrap() - .expect("should generate proof"); + let mut proof_result = cost_return_on_error_no_add!( + &cost, + subtree + .prove_without_encoding(query.clone(), limit_offset.0, limit_offset.1) + .unwrap() + .map_err(|e| Error::InternalError("failed to generate proof")) + ); cost_return_on_error!(&mut cost, self.post_process_proof(path, &mut proof_result)); diff --git a/grovedb/src/tests/query_tests.rs b/grovedb/src/tests/query_tests.rs index 0bb6a1f0..bb71dac8 100644 --- a/grovedb/src/tests/query_tests.rs +++ b/grovedb/src/tests/query_tests.rs @@ -2658,3 +2658,19 @@ fn test_query_b_depends_on_query_a() { assert_eq!(age_result[0].2, Some(Element::new_item(vec![12]))); assert_eq!(age_result[1].2, Some(Element::new_item(vec![46]))); } + +#[test] +fn test_prove_absent_path_with_intermediate_emtpy_tree() { + // root + // test_leaf (empty) + let mut grovedb = make_test_grovedb(); + + // prove the absence of key "book" in ["test_leaf", "invalid"] + let mut query = Query::new(); + query.insert_key(b"book".to_vec()); + let mut pathquery = + PathQuery::new_unsized(vec![TEST_LEAF.to_vec(), b"invalid".to_vec()], query); + + let proof = grovedb.prove_query(&pathquery).unwrap(); + dbg!(proof); +} From 4eb98e88da134d0174eb303b6feaa5e9584a0e65 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Thu, 26 Oct 2023 13:59:23 +0100 Subject: [PATCH 3/9] cleanup --- grovedb/src/operations/proof/generate.rs | 8 ++++---- grovedb/src/tests/query_tests.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index fad48d1d..97c48060 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -573,12 +573,12 @@ impl GroveDb { .open_non_transactional_merk_at_path(current_path.as_slice().into(), None) .unwrap_add_cost(&mut cost); - if subtree.is_err() { + let Ok(subtree) = subtree else { break; - } + }; let has_item = Element::get( - subtree.as_ref().expect("confirmed not error above"), + &subtree, key, true, ) @@ -590,7 +590,7 @@ impl GroveDb { &mut cost, self.generate_and_store_merk_proof( ¤t_path.as_slice().into(), - &subtree.expect("confirmed not error above"), + &subtree, &next_key_query, (None, None), ProofTokenType::Merk, diff --git a/grovedb/src/tests/query_tests.rs b/grovedb/src/tests/query_tests.rs index bb71dac8..a2432c82 100644 --- a/grovedb/src/tests/query_tests.rs +++ b/grovedb/src/tests/query_tests.rs @@ -2672,5 +2672,5 @@ fn test_prove_absent_path_with_intermediate_emtpy_tree() { PathQuery::new_unsized(vec![TEST_LEAF.to_vec(), b"invalid".to_vec()], query); let proof = grovedb.prove_query(&pathquery).unwrap(); - dbg!(proof); + assert_eq!(proof.is_err(), false); } From cf80190de0e0bdc0cb9c06fe9df137916f2a9ae9 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Thu, 26 Oct 2023 14:14:21 +0100 Subject: [PATCH 4/9] fix proof construction --- grovedb/src/operations/proof/generate.rs | 6 ++++++ grovedb/src/tests/query_tests.rs | 10 +++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 97c48060..b166a47b 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -510,6 +510,12 @@ impl GroveDb { let mut cost = OperationCost::default(); + // if the subtree is empty, return the EmptyTree proof op + if subtree.root_hash().unwrap() == EMPTY_TREE_HASH { + write_to_vec(proofs, &[ProofTokenType::EmptyTree.into()]); + return Ok(limit_offset).wrap_with_cost(cost); + } + let mut proof_result = cost_return_on_error_no_add!( &cost, subtree diff --git a/grovedb/src/tests/query_tests.rs b/grovedb/src/tests/query_tests.rs index a2432c82..44dedb62 100644 --- a/grovedb/src/tests/query_tests.rs +++ b/grovedb/src/tests/query_tests.rs @@ -42,6 +42,7 @@ use crate::{ }, Element, GroveDb, PathQuery, SizedQuery, }; +use crate::operations::proof::util::EMPTY_TREE_HASH; fn populate_tree_for_non_unique_range_subquery(db: &TempGroveDb) { // Insert a couple of subtrees first @@ -2668,9 +2669,12 @@ fn test_prove_absent_path_with_intermediate_emtpy_tree() { // prove the absence of key "book" in ["test_leaf", "invalid"] let mut query = Query::new(); query.insert_key(b"book".to_vec()); - let mut pathquery = + let mut path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec(), b"invalid".to_vec()], query); - let proof = grovedb.prove_query(&pathquery).unwrap(); - assert_eq!(proof.is_err(), false); + let proof = grovedb.prove_query(&path_query).unwrap().expect("should generate proofs"); + + let (root_hash, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query).expect("should verify proof"); + assert_eq!(result_set.len(), 0); + assert_eq!(root_hash, EMPTY_TREE_HASH); } From b2b96f78982e99c8599bc87a41451a156ec2f5e0 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Thu, 26 Oct 2023 14:43:51 +0100 Subject: [PATCH 5/9] fix verification --- grovedb/src/operations/proof/generate.rs | 7 +------ grovedb/src/operations/proof/verify.rs | 24 +++++++++++++++++++++++- grovedb/src/tests/query_tests.rs | 12 ++++++++---- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index b166a47b..76af9c5f 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -583,12 +583,7 @@ impl GroveDb { break; }; - let has_item = Element::get( - &subtree, - key, - true, - ) - .unwrap_add_cost(&mut cost); + let has_item = Element::get(&subtree, key, true).unwrap_add_cost(&mut cost); let mut next_key_query = Query::new(); next_key_query.insert_key(key.to_vec()); diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 9e8c6e44..9f358e48 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -706,9 +706,31 @@ impl ProofVerifier { let mut expected_child_hash = None; let mut last_result_set: ProvedPathKeyValues = vec![]; + // we get the key of the first element in the path + // we verify that it is in the root + // if no expected_child_hash we set the root child hash + // then we get the result set, which should be the element at the current key + // we deserialize that element, and extra it's child hash, the child hash is in + // the proof (whatever that means) child_hash = Hash(value, root_hash) + for key in path_slices { let (proof_token_type, merk_proof, _) = proof_reader.read_proof()?; - if proof_token_type != ProofTokenType::Merk { + if proof_token_type == ProofTokenType::EmptyTree { + let combined_hash = combine_hash( + value_hash_fn(last_result_set[0].value.as_slice()).value(), + &[0; 32], + ) + .unwrap(); + if Some(combined_hash) != expected_child_hash { + return Err(Error::InvalidProof( + "proof invalid: could not verify empty subtree while generating absent \ + path proof", + )); + } else { + last_result_set = vec![]; + break; + } + } else if proof_token_type != ProofTokenType::Merk { return Err(Error::InvalidProof("expected a merk proof for absent path")); } diff --git a/grovedb/src/tests/query_tests.rs b/grovedb/src/tests/query_tests.rs index 44dedb62..826d8596 100644 --- a/grovedb/src/tests/query_tests.rs +++ b/grovedb/src/tests/query_tests.rs @@ -34,6 +34,7 @@ use tempfile::TempDir; use crate::{ batch::GroveDbOp, + operations::proof::util::EMPTY_TREE_HASH, query_result_type::{PathKeyOptionalElementTrio, QueryResultType}, reference_path::ReferencePathType, tests::{ @@ -42,7 +43,6 @@ use crate::{ }, Element, GroveDb, PathQuery, SizedQuery, }; -use crate::operations::proof::util::EMPTY_TREE_HASH; fn populate_tree_for_non_unique_range_subquery(db: &TempGroveDb) { // Insert a couple of subtrees first @@ -2672,9 +2672,13 @@ fn test_prove_absent_path_with_intermediate_emtpy_tree() { let mut path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec(), b"invalid".to_vec()], query); - let proof = grovedb.prove_query(&path_query).unwrap().expect("should generate proofs"); + let proof = grovedb + .prove_query(&path_query) + .unwrap() + .expect("should generate proofs"); - let (root_hash, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query).expect("should verify proof"); + let (root_hash, result_set) = + GroveDb::verify_query(proof.as_slice(), &path_query).expect("should verify proof"); assert_eq!(result_set.len(), 0); - assert_eq!(root_hash, EMPTY_TREE_HASH); + assert_eq!(root_hash, grovedb.root_hash(None).unwrap().unwrap()); } From 7113738d7bef0ec032f52a979baf71896ce1ae64 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Thu, 26 Oct 2023 14:50:04 +0100 Subject: [PATCH 6/9] add documentation --- grovedb/src/operations/proof/verify.rs | 10 +++------- grovedb/src/tests/query_tests.rs | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 9f358e48..a69935bf 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -706,16 +706,12 @@ impl ProofVerifier { let mut expected_child_hash = None; let mut last_result_set: ProvedPathKeyValues = vec![]; - // we get the key of the first element in the path - // we verify that it is in the root - // if no expected_child_hash we set the root child hash - // then we get the result set, which should be the element at the current key - // we deserialize that element, and extra it's child hash, the child hash is in - // the proof (whatever that means) child_hash = Hash(value, root_hash) - for key in path_slices { let (proof_token_type, merk_proof, _) = proof_reader.read_proof()?; if proof_token_type == ProofTokenType::EmptyTree { + // when we encounter the empty tree op, we need to ensure + // that the expected tree hash is the combination of the + // Element_value_hash and the empty root hash [0; 32] let combined_hash = combine_hash( value_hash_fn(last_result_set[0].value.as_slice()).value(), &[0; 32], diff --git a/grovedb/src/tests/query_tests.rs b/grovedb/src/tests/query_tests.rs index 826d8596..c8e53ff8 100644 --- a/grovedb/src/tests/query_tests.rs +++ b/grovedb/src/tests/query_tests.rs @@ -34,7 +34,6 @@ use tempfile::TempDir; use crate::{ batch::GroveDbOp, - operations::proof::util::EMPTY_TREE_HASH, query_result_type::{PathKeyOptionalElementTrio, QueryResultType}, reference_path::ReferencePathType, tests::{ From 5f203074ddfb32a2e149a361497faf8460eb599e Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Thu, 26 Oct 2023 15:04:20 +0100 Subject: [PATCH 7/9] clippy fixes --- grovedb/src/operations/proof/generate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 76af9c5f..48b7c7ea 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -521,7 +521,7 @@ impl GroveDb { subtree .prove_without_encoding(query.clone(), limit_offset.0, limit_offset.1) .unwrap() - .map_err(|e| Error::InternalError("failed to generate proof")) + .map_err(|_e| Error::InternalError("failed to generate proof")) ); cost_return_on_error!(&mut cost, self.post_process_proof(path, &mut proof_result)); From 4bb48e648b9933060259eba48f474f30af13e325 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 26 Oct 2023 23:15:05 +0700 Subject: [PATCH 8/9] fixed non used error --- grovedb/src/operations/proof/generate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 48b7c7ea..0e05e720 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -512,7 +512,7 @@ impl GroveDb { // if the subtree is empty, return the EmptyTree proof op if subtree.root_hash().unwrap() == EMPTY_TREE_HASH { - write_to_vec(proofs, &[ProofTokenType::EmptyTree.into()]); + cost_return_on_error_no_add!(&cost, write_to_vec(proofs, &[ProofTokenType::EmptyTree.into()])); return Ok(limit_offset).wrap_with_cost(cost); } From 33487061d0ee0226f13f8cdcb7b7af08c4b51f98 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 26 Oct 2023 23:15:36 +0700 Subject: [PATCH 9/9] fmt --- grovedb/src/operations/proof/generate.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 0e05e720..fad64c84 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -512,7 +512,10 @@ impl GroveDb { // if the subtree is empty, return the EmptyTree proof op if subtree.root_hash().unwrap() == EMPTY_TREE_HASH { - cost_return_on_error_no_add!(&cost, write_to_vec(proofs, &[ProofTokenType::EmptyTree.into()])); + cost_return_on_error_no_add!( + &cost, + write_to_vec(proofs, &[ProofTokenType::EmptyTree.into()]) + ); return Ok(limit_offset).wrap_with_cost(cost); }