From 8d973b8838eb1976c9f1a5f09ab3c8140ab532ad Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Thu, 20 Jan 2022 20:56:45 +0700 Subject: [PATCH 01/31] failing state --- grovedb/src/operations/get.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 661a75d0f..6ee15672a 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -80,12 +80,18 @@ impl GroveDb { Some(_) => &self.temp_subtrees, }; - let merk = subtrees - .get(&Self::compress_subtree_key(path, None)) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; + let merk = self.get_subtree(path).unwrap(); + + // let merk = subtrees + // .get(&Self::compress_subtree_key(path, None)) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; Element::get(&merk, key) } + fn get_subtree(&self, path: &[&[u8]]) -> Result, Error> { + todo!() + } + pub fn get_path_queries( &mut self, path_queries: &[&PathQuery], From f8725887d3db2c7e89c097187bad125c20a547d6 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Thu, 20 Jan 2022 22:50:37 +0700 Subject: [PATCH 02/31] refactor: is_empty_tree --- grovedb/src/lib.rs | 5 +- grovedb/src/operations/delete.rs | 1 + grovedb/src/operations/get.rs | 61 +++++++++++++++---------- grovedb/src/operations/is_empty_tree.rs | 9 +--- grovedb/src/tests.rs | 35 ++++++++++---- 5 files changed, 70 insertions(+), 41 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 485941644..8eaffc8e0 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -487,7 +487,10 @@ impl GroveDb { .map_err(PrefixedRocksDbStorageError::RocksDbError)?) } - pub fn get_subtrees_for_transaction(&mut self, transaction: Option<&OptimisticTransactionDBTransaction>) -> &HashMap, Merk> { + pub fn get_subtrees_for_transaction( + &mut self, + transaction: Option<&OptimisticTransactionDBTransaction>, + ) -> &HashMap, Merk> { match transaction { None => &self.subtrees, Some(_) => &self.temp_subtrees, diff --git a/grovedb/src/operations/delete.rs b/grovedb/src/operations/delete.rs index 4cccafcc0..59f665662 100644 --- a/grovedb/src/operations/delete.rs +++ b/grovedb/src/operations/delete.rs @@ -30,6 +30,7 @@ impl GroveDb { let mut merk = subtrees .get_mut(&Self::compress_subtree_key(path, None)) .ok_or(Error::InvalidPath("no subtree found under that path"))?; + Element::delete(&mut merk, key.clone(), transaction)?; } diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 6ee15672a..0da2c564e 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -1,6 +1,8 @@ use std::{ + arch::x86_64::_mm_extract_si64, collections::{HashMap, HashSet}, ops::Range, + rc::Rc, }; use merk::{ @@ -75,21 +77,28 @@ impl GroveDb { key: &[u8], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let subtrees = match transaction { - None => &self.subtrees, - Some(_) => &self.temp_subtrees, - }; - - let merk = self.get_subtree(path).unwrap(); - - // let merk = subtrees - // .get(&Self::compress_subtree_key(path, None)) - // .ok_or(Error::InvalidPath("no subtree found under that path"))?; - Element::get(&merk, key) + Element::get(&self.get_subtree(path, transaction)?, key) } - fn get_subtree(&self, path: &[&[u8]]) -> Result, Error> { - todo!() + pub fn get_subtree( + &self, + path: &[&[u8]], + transaction: Option<&OptimisticTransactionDBTransaction>, + ) -> Result, Error> { + let subtree_prefix = GroveDb::compress_subtree_key(path, None); + match transaction { + None => Ok( + Merk::open(PrefixedRocksDbStorage::new(self.storage(), subtree_prefix)?) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?, + ), + Some(_) => { + if let Some(merk) = self.temp_subtrees.get(&subtree_prefix) { + Ok(merk.clone()) + } else { + Err(Error::InvalidPath("no subtree found under that path")) + } + } + } } pub fn get_path_queries( @@ -98,8 +107,9 @@ impl GroveDb { transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result>, Error> { let elements = self.get_path_queries_raw(path_queries, transaction)?; - let results = elements.into_iter().map(|element| { - match element { + let results = elements + .into_iter() + .map(|element| match element { Element::Reference(reference_path) => { let maybe_item = self.follow_reference(reference_path, transaction)?; if let Element::Item(item) = maybe_item { @@ -108,9 +118,11 @@ impl GroveDb { Err(Error::InvalidQuery("the reference must result in an item")) } } - other => Err(Error::InvalidQuery("path_queries can only refer to references")), - } - }).collect::>, Error>>()?; + other => Err(Error::InvalidQuery( + "path_queries can only refer to references", + )), + }) + .collect::>, Error>>()?; Ok(results) } @@ -133,8 +145,9 @@ impl GroveDb { transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result<(Vec>, u16), Error> { let (elements, skipped) = self.get_path_query_raw(path_query, transaction)?; - let results = elements.into_iter().map(|element| { - match element { + let results = elements + .into_iter() + .map(|element| match element { Element::Reference(reference_path) => { let maybe_item = self.follow_reference(reference_path, transaction)?; if let Element::Item(item) = maybe_item { @@ -144,9 +157,11 @@ impl GroveDb { } } Element::Item(item) => Ok(item), - Element::Tree(_) => Err(Error::InvalidQuery("path_queries can only refer to items and references")), - } - }).collect::>, Error>>()?; + Element::Tree(_) => Err(Error::InvalidQuery( + "path_queries can only refer to items and references", + )), + }) + .collect::>, Error>>()?; Ok((results, skipped)) } diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index fa00f5057..959679566 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -8,14 +8,7 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let subtrees = match transaction { - None => &self.subtrees, - Some(_) => &self.temp_subtrees, - }; - - let merk = subtrees - .get(&Self::compress_subtree_key(path, None)) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; + let merk = self.get_subtree(path, transaction)?; let mut iter = merk.raw_iter(); iter.seek_to_first(); diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 46f5e0ec2..19f9f09b8 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -786,11 +786,14 @@ fn transaction_insert_item_with_transaction_should_use_transaction() { // The key was inserted inside the transaction, so it shouldn't be possible // to get it back without committing or using transaction let result = db.get(&[TEST_LEAF], &item_key, None); + dbg!(&result); assert!(matches!(result, Err(Error::InvalidPath(_)))); // Check that the element can be retrieved when transaction is passed let result_with_transaction = db .get(&[TEST_LEAF], &item_key, Some(&transaction)) .expect("Expected to work"); + dbg!("We got here"); + dbg!(&result_with_transaction); assert_eq!(result_with_transaction, Element::Item(b"ayy".to_vec())); // Test that commit works @@ -1035,12 +1038,16 @@ fn test_element_deletion() { db.insert(&[TEST_LEAF], b"key".to_vec(), element.clone(), None) .expect("successful insert"); let root_hash = db.root_tree.root().unwrap(); + dbg!("starting delete"); assert!(db.delete(&[TEST_LEAF], b"key".to_vec(), None).is_ok()); + dbg!("successful first delete"); assert!(matches!( db.get(&[TEST_LEAF], b"key", None), Err(Error::InvalidPath(_)) )); + dbg!("successful get"); assert_ne!(root_hash, db.root_tree.root().unwrap()); + dbg!("failed here"); } #[test] @@ -1313,8 +1320,13 @@ fn populate_tree_by_reference_for_non_unique_range_subquery(db: &mut TempGroveDb // Insert a couple of subtrees first for i in 1985u32..2000 { let i_vec = (i as u32).to_be_bytes().to_vec(); - db.insert(&[TEST_LEAF, b"1"], i_vec.clone(), Element::empty_tree(), None) - .expect("successful subtree insert"); + db.insert( + &[TEST_LEAF, b"1"], + i_vec.clone(), + Element::empty_tree(), + None, + ) + .expect("successful subtree insert"); // Insert element 0 // Insert some elements into subtree db.insert( @@ -1323,7 +1335,7 @@ fn populate_tree_by_reference_for_non_unique_range_subquery(db: &mut TempGroveDb Element::empty_tree(), None, ) - .expect("successful subtree insert"); + .expect("successful subtree insert"); for j in 100u32..150 { let random_key = rand::thread_rng().gen::<[u8; 32]>(); @@ -1337,7 +1349,7 @@ fn populate_tree_by_reference_for_non_unique_range_subquery(db: &mut TempGroveDb Element::Item(j_vec.clone()), None, ) - .expect("successful value insert"); + .expect("successful value insert"); db.insert( &[TEST_LEAF, b"1", i_vec.clone().as_slice(), b"0"], @@ -1345,7 +1357,7 @@ fn populate_tree_by_reference_for_non_unique_range_subquery(db: &mut TempGroveDb Element::Reference(vec![TEST_LEAF.to_vec(), b"0".to_vec(), random_key.to_vec()]), None, ) - .expect("successful value insert"); + .expect("successful value insert"); } } } @@ -1378,8 +1390,13 @@ fn populate_tree_by_reference_for_unique_range_subquery(db: &mut TempGroveDb) { for i in 1985u32..2000 { let i_vec = (i as u32).to_be_bytes().to_vec(); - db.insert(&[TEST_LEAF, b"1"], i_vec.clone(), Element::empty_tree(), None) - .expect("successful subtree insert"); + db.insert( + &[TEST_LEAF, b"1"], + i_vec.clone(), + Element::empty_tree(), + None, + ) + .expect("successful subtree insert"); // We should insert every item to the tree holding items db.insert( @@ -1388,7 +1405,7 @@ fn populate_tree_by_reference_for_unique_range_subquery(db: &mut TempGroveDb) { Element::Item(i_vec.clone()), None, ) - .expect("successful value insert"); + .expect("successful value insert"); // We should insert a reference to the item db.insert( @@ -1397,7 +1414,7 @@ fn populate_tree_by_reference_for_unique_range_subquery(db: &mut TempGroveDb) { Element::Reference(vec![TEST_LEAF.to_vec(), b"0".to_vec(), i_vec.clone()]), None, ) - .expect("successful value insert"); + .expect("successful value insert"); } } From ac913c7f11e64023d108efbbcb1de207f7df0131 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 21 Jan 2022 01:28:41 +0700 Subject: [PATCH 03/31] added basic test for get subtree --- grovedb/src/tests.rs | 71 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 19f9f09b8..364e7a35c 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1088,6 +1088,77 @@ fn test_find_subtrees() { ); } +#[test] +fn test_get_subtree() { + let mut db = make_grovedb(); + let element = Element::Item(b"ayy".to_vec()); + + // Returns error is subtree is not valid + let subtree = db.get_subtree(&[TEST_LEAF, b"invalid_tree"], None); + assert_eq!(subtree.is_err(), true); + + // Doesn't return an error for subtree that exists but empty + // let subtree = db.get_subtree(&[TEST_LEAF], None); + // assert_eq!(subtree.is_err(), false); + + // Insert some nested subtrees + db.insert(&[TEST_LEAF], b"key1".to_vec(), Element::empty_tree(), None) + .expect("successful subtree 1 insert"); + // let subtree = db.get_subtree(&[TEST_LEAF, b"key1"], None); + // assert_eq!(subtree.is_err(), false); + + db.insert( + &[TEST_LEAF, b"key1"], + b"key2".to_vec(), + Element::empty_tree(), + None, + ) + .expect("successful subtree 2 insert"); + + // Insert an element into subtree + db.insert( + &[TEST_LEAF, b"key1", b"key2"], + b"key3".to_vec(), + element.clone(), + None, + ) + .expect("successful value insert"); + db.insert(&[TEST_LEAF], b"key4".to_vec(), Element::empty_tree(), None) + .expect("successful subtree 3 insert"); + + // Retrieve subtree instance + // Check if it returns the same instance that was inserted + // let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"key2"], None).unwrap(); + // let result_element = Element::get(&subtree, b"key3").unwrap(); + // assert_eq!(result_element, Element::Item(b"ayy".to_vec())); + + // Insert a new tree with transaction + // db.start_transaction().unwrap(); + // let storage = db.storage(); + // let transaction = storage.transaction(); + + // db.insert( + // &[TEST_LEAF, b"key1"], + // b"innertree".to_vec(), + // Element::empty_tree(), + // // Some(&transaction), + // None, + // ).expect("successful subtree insert"); + + // db.insert( + // &[TEST_LEAF, b"key1", b"innertree"], + // b"key4".to_vec(), + // element.clone(), + // Some(&transaction), + // ).expect("successful value insert"); + + // Retrieve subtree instance without transaction + let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"innertree"], None); + assert_eq!(subtree.is_err(), false); + // let result_element = Element::get(&subtree, b"key4").unwrap(); + // assert_eq!(result_element, Element::Item(b"ayy".to_vec())); +} + #[test] fn test_subtree_deletion() { let element = Element::Item(b"ayy".to_vec()); From f58a2aeaadf268d2bebc32881f20331308430bb6 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 21 Jan 2022 09:36:22 +0700 Subject: [PATCH 04/31] improved get_subtree to differentiate between empty and non existent tree --- grovedb/src/operations/get.rs | 42 +++++++++++++++++++++++++++++++---- grovedb/src/tests.rs | 33 ++++++++++++++------------- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 0da2c564e..6b9a0fb0b 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -87,10 +87,29 @@ impl GroveDb { ) -> Result, Error> { let subtree_prefix = GroveDb::compress_subtree_key(path, None); match transaction { - None => Ok( - Merk::open(PrefixedRocksDbStorage::new(self.storage(), subtree_prefix)?) - .map_err(|_| Error::InvalidPath("no subtree found under that path"))?, - ), + None => { + let (subtree, has_keys) = self.get_subtree_with_key_info(path)?; + if !has_keys { + // if the subtree has not keys then it's either empty or invalid + // we can confirm that it's an empty tree by checking if it was inserted into + // the parent tree + let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; + let (parent_tree, has_keys) = self.get_subtree_with_key_info(parent_path)?; + if !has_keys { + // parent tree can't be empty, hence invalid path + Err(Error::InvalidPath("no subtree found under that path")) + } else { + // Check that it contains the child as an empty tree + let elem = Element::get(&parent_tree, key).map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + match elem { + Element::Tree(_) => Ok(subtree), + _ => Err(Error::InvalidPath("no subtree found under that path")) + } + } + } else { + Ok(subtree) + } + } Some(_) => { if let Some(merk) = self.temp_subtrees.get(&subtree_prefix) { Ok(merk.clone()) @@ -101,6 +120,21 @@ impl GroveDb { } } + fn get_subtree_with_key_info(&self, path: &[&[u8]]) -> Result<(Merk, bool), Error> { + let subtree_prefix = GroveDb::compress_subtree_key(path, None); + let merk = Merk::open(PrefixedRocksDbStorage::new(self.storage(), subtree_prefix)?) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + let mut has_keys = false; + { + let mut iter = merk.raw_iter(); + iter.seek_to_first(); + if iter.valid() { + has_keys = true; + } + } + Ok((merk, has_keys)) + } + pub fn get_path_queries( &mut self, path_queries: &[&PathQuery], diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 364e7a35c..7fe98e37f 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1113,7 +1113,7 @@ fn test_get_subtree() { Element::empty_tree(), None, ) - .expect("successful subtree 2 insert"); + .expect("successful subtree 2 insert"); // Insert an element into subtree db.insert( @@ -1122,7 +1122,7 @@ fn test_get_subtree() { element.clone(), None, ) - .expect("successful value insert"); + .expect("successful value insert"); db.insert(&[TEST_LEAF], b"key4".to_vec(), Element::empty_tree(), None) .expect("successful subtree 3 insert"); @@ -1137,20 +1137,21 @@ fn test_get_subtree() { // let storage = db.storage(); // let transaction = storage.transaction(); - // db.insert( - // &[TEST_LEAF, b"key1"], - // b"innertree".to_vec(), - // Element::empty_tree(), - // // Some(&transaction), - // None, - // ).expect("successful subtree insert"); - - // db.insert( - // &[TEST_LEAF, b"key1", b"innertree"], - // b"key4".to_vec(), - // element.clone(), - // Some(&transaction), - // ).expect("successful value insert"); + db.insert( + &[TEST_LEAF, b"key1"], + b"innertree".to_vec(), + Element::empty_tree(), + // Some(&transaction), + None, + ).expect("successful subtree insert"); + + db.insert( + &[TEST_LEAF, b"key1", b"innertree"], + b"key4".to_vec(), + element.clone(), + // Some(&transaction), + None, + ).expect("successful value insert"); // Retrieve subtree instance without transaction let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"innertree"], None); From 00766e4810d8f1c294b0f1c5ffaa1210a329c456 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 21 Jan 2022 10:04:23 +0700 Subject: [PATCH 05/31] improved get_subtree, works for root leaf nodes --- grovedb/src/operations/get.rs | 34 ++++++++++++++++++++++++++-------- grovedb/src/tests.rs | 10 ++++++---- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 6b9a0fb0b..d77900a76 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -88,22 +88,36 @@ impl GroveDb { let subtree_prefix = GroveDb::compress_subtree_key(path, None); match transaction { None => { - let (subtree, has_keys) = self.get_subtree_with_key_info(path)?; + let (subtree, has_keys) = self.get_subtree_with_key_info(path, None)?; if !has_keys { - // if the subtree has not keys then it's either empty or invalid + // if the subtree has no keys, it's either empty or invalid // we can confirm that it's an empty tree by checking if it was inserted into // the parent tree - let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; - let (parent_tree, has_keys) = self.get_subtree_with_key_info(parent_path)?; + let (key, parent_path) = + path.split_last().ok_or(Error::InvalidPath("empty path"))?; + + // if parent path is empty, we are dealing with root leaf node + // we can confirm validity of a root leaf node by checking root_leaf_keys + if parent_path.is_empty(){ + if self.root_leaf_keys.contains_key(&subtree_prefix){ + return Ok(subtree); + } else { + return Err(Error::InvalidPath("no subtree found under that path")); + } + } + + // Non root lead nodes, get parent tree and confirm child validity + let (parent_tree, has_keys) = self.get_subtree_with_key_info(parent_path, Some(key))?; if !has_keys { // parent tree can't be empty, hence invalid path Err(Error::InvalidPath("no subtree found under that path")) } else { // Check that it contains the child as an empty tree - let elem = Element::get(&parent_tree, key).map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + let elem = Element::get(&parent_tree, key) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; match elem { Element::Tree(_) => Ok(subtree), - _ => Err(Error::InvalidPath("no subtree found under that path")) + _ => Err(Error::InvalidPath("no subtree found under that path")), } } } else { @@ -120,8 +134,12 @@ impl GroveDb { } } - fn get_subtree_with_key_info(&self, path: &[&[u8]]) -> Result<(Merk, bool), Error> { - let subtree_prefix = GroveDb::compress_subtree_key(path, None); + fn get_subtree_with_key_info( + &self, + path: &[&[u8]], + key: Option<&[u8]> + ) -> Result<(Merk, bool), Error> { + let subtree_prefix = GroveDb::compress_subtree_key(path, key); let merk = Merk::open(PrefixedRocksDbStorage::new(self.storage(), subtree_prefix)?) .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; let mut has_keys = false; diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 7fe98e37f..2ab5340d4 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1098,8 +1098,8 @@ fn test_get_subtree() { assert_eq!(subtree.is_err(), true); // Doesn't return an error for subtree that exists but empty - // let subtree = db.get_subtree(&[TEST_LEAF], None); - // assert_eq!(subtree.is_err(), false); + let subtree = db.get_subtree(&[TEST_LEAF], None); + assert_eq!(subtree.is_err(), false); // Insert some nested subtrees db.insert(&[TEST_LEAF], b"key1".to_vec(), Element::empty_tree(), None) @@ -1143,7 +1143,8 @@ fn test_get_subtree() { Element::empty_tree(), // Some(&transaction), None, - ).expect("successful subtree insert"); + ) + .expect("successful subtree insert"); db.insert( &[TEST_LEAF, b"key1", b"innertree"], @@ -1151,7 +1152,8 @@ fn test_get_subtree() { element.clone(), // Some(&transaction), None, - ).expect("successful value insert"); + ) + .expect("successful value insert"); // Retrieve subtree instance without transaction let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"innertree"], None); From 59fda238b4a723c27eabf9976a8be3bff9f011da Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 21 Jan 2022 12:50:28 +0700 Subject: [PATCH 06/31] fixed get parent bug in get_subtree --- grovedb/src/operations/get.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index d77900a76..cf8d64657 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -106,8 +106,8 @@ impl GroveDb { } } - // Non root lead nodes, get parent tree and confirm child validity - let (parent_tree, has_keys) = self.get_subtree_with_key_info(parent_path, Some(key))?; + // Non root leaf nodes, get parent tree and confirm child validity + let (parent_tree, has_keys) = self.get_subtree_with_key_info(parent_path, None)?; if !has_keys { // parent tree can't be empty, hence invalid path Err(Error::InvalidPath("no subtree found under that path")) From 98888524ca3a9a720e9790635be8688a9a8c04bd Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 21 Jan 2022 12:57:31 +0700 Subject: [PATCH 07/31] refactor: split get_subtree into multiple functions --- grovedb/src/operations/get.rs | 82 +++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index cf8d64657..37cebef99 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -87,50 +87,56 @@ impl GroveDb { ) -> Result, Error> { let subtree_prefix = GroveDb::compress_subtree_key(path, None); match transaction { - None => { - let (subtree, has_keys) = self.get_subtree_with_key_info(path, None)?; - if !has_keys { - // if the subtree has no keys, it's either empty or invalid - // we can confirm that it's an empty tree by checking if it was inserted into - // the parent tree - let (key, parent_path) = - path.split_last().ok_or(Error::InvalidPath("empty path"))?; - - // if parent path is empty, we are dealing with root leaf node - // we can confirm validity of a root leaf node by checking root_leaf_keys - if parent_path.is_empty(){ - if self.root_leaf_keys.contains_key(&subtree_prefix){ - return Ok(subtree); - } else { - return Err(Error::InvalidPath("no subtree found under that path")); - } - } + None => self.get_subtree_without_transaction(path), + Some(_) => self.get_subtree_with_transaction(path), + } + } - // Non root leaf nodes, get parent tree and confirm child validity - let (parent_tree, has_keys) = self.get_subtree_with_key_info(parent_path, None)?; - if !has_keys { - // parent tree can't be empty, hence invalid path - Err(Error::InvalidPath("no subtree found under that path")) - } else { - // Check that it contains the child as an empty tree - let elem = Element::get(&parent_tree, key) - .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; - match elem { - Element::Tree(_) => Ok(subtree), - _ => Err(Error::InvalidPath("no subtree found under that path")), - } - } + fn get_subtree_without_transaction(&self, path: &[&[u8]]) -> Result, Error> { + let subtree_prefix = GroveDb::compress_subtree_key(path, None); + let (subtree, has_keys) = self.get_subtree_with_key_info(path, None)?; + if !has_keys { + // if the subtree has no keys, it's either empty or invalid + // we can confirm that it's an empty tree by checking if it was inserted into + // the parent tree + let (key, parent_path) = + path.split_last().ok_or(Error::InvalidPath("empty path"))?; + + // if parent path is empty, we are dealing with root leaf node + // we can confirm validity of a root leaf node by checking root_leaf_keys + if parent_path.is_empty(){ + if self.root_leaf_keys.contains_key(&subtree_prefix){ + return Ok(subtree); } else { - Ok(subtree) + return Err(Error::InvalidPath("no subtree found under that path")); } } - Some(_) => { - if let Some(merk) = self.temp_subtrees.get(&subtree_prefix) { - Ok(merk.clone()) - } else { - Err(Error::InvalidPath("no subtree found under that path")) + + // Non root leaf nodes, get parent tree and confirm child validity + let (parent_tree, has_keys) = self.get_subtree_with_key_info(parent_path, None)?; + if !has_keys { + // parent tree can't be empty, hence invalid path + Err(Error::InvalidPath("no subtree found under that path")) + } else { + // Check that it contains the child as an empty tree + let elem = Element::get(&parent_tree, key) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + match elem { + Element::Tree(_) => Ok(subtree), + _ => Err(Error::InvalidPath("no subtree found under that path")), } } + } else { + Ok(subtree) + } + } + + fn get_subtree_with_transaction(&self, path: &[&[u8]]) -> Result, Error> { + let subtree_prefix = GroveDb::compress_subtree_key(path, None); + if let Some(merk) = self.temp_subtrees.get(&subtree_prefix) { + Ok(merk.clone()) + } else { + Err(Error::InvalidPath("no subtree found under that path")) } } From e93f81979c9bd714fb47bc262240c1e853edd5d4 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 21 Jan 2022 12:59:48 +0700 Subject: [PATCH 08/31] cleanup --- grovedb/src/operations/get.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 37cebef99..a6ff8497d 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -85,31 +85,32 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result, Error> { - let subtree_prefix = GroveDb::compress_subtree_key(path, None); match transaction { None => self.get_subtree_without_transaction(path), Some(_) => self.get_subtree_with_transaction(path), } } - fn get_subtree_without_transaction(&self, path: &[&[u8]]) -> Result, Error> { + fn get_subtree_without_transaction( + &self, + path: &[&[u8]], + ) -> Result, Error> { let subtree_prefix = GroveDb::compress_subtree_key(path, None); let (subtree, has_keys) = self.get_subtree_with_key_info(path, None)?; if !has_keys { // if the subtree has no keys, it's either empty or invalid // we can confirm that it's an empty tree by checking if it was inserted into // the parent tree - let (key, parent_path) = - path.split_last().ok_or(Error::InvalidPath("empty path"))?; + let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; // if parent path is empty, we are dealing with root leaf node // we can confirm validity of a root leaf node by checking root_leaf_keys - if parent_path.is_empty(){ - if self.root_leaf_keys.contains_key(&subtree_prefix){ - return Ok(subtree); + if parent_path.is_empty() { + return if self.root_leaf_keys.contains_key(&subtree_prefix) { + Ok(subtree) } else { - return Err(Error::InvalidPath("no subtree found under that path")); - } + Err(Error::InvalidPath("no subtree found under that path")) + }; } // Non root leaf nodes, get parent tree and confirm child validity @@ -131,7 +132,10 @@ impl GroveDb { } } - fn get_subtree_with_transaction(&self, path: &[&[u8]]) -> Result, Error> { + fn get_subtree_with_transaction( + &self, + path: &[&[u8]], + ) -> Result, Error> { let subtree_prefix = GroveDb::compress_subtree_key(path, None); if let Some(merk) = self.temp_subtrees.get(&subtree_prefix) { Ok(merk.clone()) @@ -143,7 +147,7 @@ impl GroveDb { fn get_subtree_with_key_info( &self, path: &[&[u8]], - key: Option<&[u8]> + key: Option<&[u8]>, ) -> Result<(Merk, bool), Error> { let subtree_prefix = GroveDb::compress_subtree_key(path, key); let merk = Merk::open(PrefixedRocksDbStorage::new(self.storage(), subtree_prefix)?) From 7925ef4bd84923182cfd70415d30b27bf21de1f2 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 21 Jan 2022 13:02:38 +0700 Subject: [PATCH 09/31] updated tests for get_subtree --- grovedb/src/tests.rs | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 2ab5340d4..fbf91f50b 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1104,8 +1104,6 @@ fn test_get_subtree() { // Insert some nested subtrees db.insert(&[TEST_LEAF], b"key1".to_vec(), Element::empty_tree(), None) .expect("successful subtree 1 insert"); - // let subtree = db.get_subtree(&[TEST_LEAF, b"key1"], None); - // assert_eq!(subtree.is_err(), false); db.insert( &[TEST_LEAF, b"key1"], @@ -1128,21 +1126,20 @@ fn test_get_subtree() { // Retrieve subtree instance // Check if it returns the same instance that was inserted - // let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"key2"], None).unwrap(); - // let result_element = Element::get(&subtree, b"key3").unwrap(); - // assert_eq!(result_element, Element::Item(b"ayy".to_vec())); + let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"key2"], None).unwrap(); + let result_element = Element::get(&subtree, b"key3").unwrap(); + assert_eq!(result_element, Element::Item(b"ayy".to_vec())); // Insert a new tree with transaction - // db.start_transaction().unwrap(); - // let storage = db.storage(); - // let transaction = storage.transaction(); + db.start_transaction().unwrap(); + let storage = db.storage(); + let transaction = storage.transaction(); db.insert( &[TEST_LEAF, b"key1"], b"innertree".to_vec(), Element::empty_tree(), - // Some(&transaction), - None, + Some(&transaction), ) .expect("successful subtree insert"); @@ -1150,16 +1147,14 @@ fn test_get_subtree() { &[TEST_LEAF, b"key1", b"innertree"], b"key4".to_vec(), element.clone(), - // Some(&transaction), - None, + Some(&transaction), ) .expect("successful value insert"); // Retrieve subtree instance without transaction - let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"innertree"], None); - assert_eq!(subtree.is_err(), false); - // let result_element = Element::get(&subtree, b"key4").unwrap(); - // assert_eq!(result_element, Element::Item(b"ayy".to_vec())); + let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"innertree"], Some(&transaction)).unwrap(); + let result_element = Element::get(&subtree, b"key4").unwrap(); + assert_eq!(result_element, Element::Item(b"ayy".to_vec())); } #[test] From 42da1064a17ff5e359f9cbe3783a2f210536f6e1 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 21 Jan 2022 21:13:06 +0700 Subject: [PATCH 10/31] created new subtrees struct that prevents get_subtree from owning or referencing grovedb struct --- grovedb/src/lib.rs | 10 +++ grovedb/src/operations/get.rs | 85 +--------------------- grovedb/src/operations/is_empty_tree.rs | 2 +- grovedb/src/subtrees.rs | 96 +++++++++++++++++++++++++ grovedb/src/tests.rs | 8 +-- 5 files changed, 112 insertions(+), 89 deletions(-) create mode 100644 grovedb/src/subtrees.rs diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 8eaffc8e0..a305bca91 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -3,6 +3,7 @@ mod subtree; #[cfg(test)] mod tests; mod transaction; +mod subtrees; use std::{collections::HashMap, path::Path, rc::Rc}; @@ -16,6 +17,7 @@ use storage::{ Transaction, }; pub use subtree::Element; +use subtrees::Subtrees; // use crate::transaction::GroveDbTransaction; // pub use transaction::GroveDbTransaction; @@ -364,6 +366,14 @@ impl GroveDb { Ok(()) } + fn get_subtrees(&self) -> Subtrees { + Subtrees{ + root_leaf_keys: &self.root_leaf_keys, + temp_subtrees: &self.temp_subtrees, + storage: self.storage(), + } + } + /// A helper method to build a prefix to rocksdb keys or identify a subtree /// in `subtrees` map by tree path; fn compress_subtree_key(path: &[&[u8]], key: Option<&[u8]>) -> Vec { diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index a6ff8497d..0619a71be 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -77,90 +77,7 @@ impl GroveDb { key: &[u8], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - Element::get(&self.get_subtree(path, transaction)?, key) - } - - pub fn get_subtree( - &self, - path: &[&[u8]], - transaction: Option<&OptimisticTransactionDBTransaction>, - ) -> Result, Error> { - match transaction { - None => self.get_subtree_without_transaction(path), - Some(_) => self.get_subtree_with_transaction(path), - } - } - - fn get_subtree_without_transaction( - &self, - path: &[&[u8]], - ) -> Result, Error> { - let subtree_prefix = GroveDb::compress_subtree_key(path, None); - let (subtree, has_keys) = self.get_subtree_with_key_info(path, None)?; - if !has_keys { - // if the subtree has no keys, it's either empty or invalid - // we can confirm that it's an empty tree by checking if it was inserted into - // the parent tree - let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; - - // if parent path is empty, we are dealing with root leaf node - // we can confirm validity of a root leaf node by checking root_leaf_keys - if parent_path.is_empty() { - return if self.root_leaf_keys.contains_key(&subtree_prefix) { - Ok(subtree) - } else { - Err(Error::InvalidPath("no subtree found under that path")) - }; - } - - // Non root leaf nodes, get parent tree and confirm child validity - let (parent_tree, has_keys) = self.get_subtree_with_key_info(parent_path, None)?; - if !has_keys { - // parent tree can't be empty, hence invalid path - Err(Error::InvalidPath("no subtree found under that path")) - } else { - // Check that it contains the child as an empty tree - let elem = Element::get(&parent_tree, key) - .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; - match elem { - Element::Tree(_) => Ok(subtree), - _ => Err(Error::InvalidPath("no subtree found under that path")), - } - } - } else { - Ok(subtree) - } - } - - fn get_subtree_with_transaction( - &self, - path: &[&[u8]], - ) -> Result, Error> { - let subtree_prefix = GroveDb::compress_subtree_key(path, None); - if let Some(merk) = self.temp_subtrees.get(&subtree_prefix) { - Ok(merk.clone()) - } else { - Err(Error::InvalidPath("no subtree found under that path")) - } - } - - fn get_subtree_with_key_info( - &self, - path: &[&[u8]], - key: Option<&[u8]>, - ) -> Result<(Merk, bool), Error> { - let subtree_prefix = GroveDb::compress_subtree_key(path, key); - let merk = Merk::open(PrefixedRocksDbStorage::new(self.storage(), subtree_prefix)?) - .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; - let mut has_keys = false; - { - let mut iter = merk.raw_iter(); - iter.seek_to_first(); - if iter.valid() { - has_keys = true; - } - } - Ok((merk, has_keys)) + Element::get(&self.get_subtrees().get_subtree(path, transaction)?, key) } pub fn get_path_queries( diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index 959679566..e53c8bcf6 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -8,7 +8,7 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let merk = self.get_subtree(path, transaction)?; + let merk = self.get_subtrees().get_subtree(path, transaction)?; let mut iter = merk.raw_iter(); iter.seek_to_first(); diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs new file mode 100644 index 000000000..4d24f6725 --- /dev/null +++ b/grovedb/src/subtrees.rs @@ -0,0 +1,96 @@ +//! Module for retrieving subtrees +use std::{collections::HashMap}; +use merk::{Merk}; +use crate::{GroveDb, Error, Element}; +use std::{rc::Rc}; + +use storage::rocksdb_storage::{OptimisticTransactionDBTransaction, PrefixedRocksDbStorage}; +use storage::RawIterator; + +pub struct Subtrees<'a>{ + pub root_leaf_keys: &'a HashMap, usize>, + pub temp_subtrees: &'a HashMap, Merk>, + pub storage: Rc, +} + +impl Subtrees<'_> { + pub fn get_subtree(&self, path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>) -> Result, Error> { + match transaction { + None => self.get_subtree_without_transaction(path), + Some(_) => self.get_subtree_with_transaction(path), + } + } + + fn get_subtree_without_transaction( + &self, + path: &[&[u8]], + ) -> Result, Error> { + let subtree_prefix = GroveDb::compress_subtree_key(path, None); + let (subtree, has_keys) = self.get_subtree_with_key_info(path, None)?; + if !has_keys { + // if the subtree has no keys, it's either empty or invalid + // we can confirm that it's an empty tree by checking if it was inserted into + // the parent tree + let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; + + // if parent path is empty, we are dealing with root leaf node + // we can confirm validity of a root leaf node by checking root_leaf_keys + if parent_path.is_empty() { + return if self.root_leaf_keys.contains_key(&subtree_prefix) { + Ok(subtree) + } else { + Err(Error::InvalidPath("no subtree found under that path")) + }; + } + + // Non root leaf nodes, get parent tree and confirm child validity + let (parent_tree, has_keys) = self.get_subtree_with_key_info(parent_path, None)?; + if !has_keys { + // parent tree can't be empty, hence invalid path + Err(Error::InvalidPath("no subtree found under that path")) + } else { + // Check that it contains the child as an empty tree + let elem = Element::get(&parent_tree, key) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + match elem { + Element::Tree(_) => Ok(subtree), + _ => Err(Error::InvalidPath("no subtree found under that path")), + } + } + } else { + Ok(subtree) + } + } + + fn get_subtree_with_transaction( + &self, + path: &[&[u8]], + ) -> Result, Error> { + let subtree_prefix = GroveDb::compress_subtree_key(path, None); + if let Some(merk) = self.temp_subtrees.get(&subtree_prefix) { + Ok(merk.clone()) + } else { + Err(Error::InvalidPath("no subtree found under that path")) + } + } + + fn get_subtree_with_key_info( + &self, + path: &[&[u8]], + key: Option<&[u8]>, + ) -> Result<(Merk, bool), Error> { + let subtree_prefix = GroveDb::compress_subtree_key(path, key); + let merk = Merk::open(PrefixedRocksDbStorage::new(self.storage.clone(), subtree_prefix)?) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + let mut has_keys = false; + { + let mut iter = merk.raw_iter(); + iter.seek_to_first(); + if iter.valid() { + has_keys = true; + } + } + Ok((merk, has_keys)) + } + +} \ No newline at end of file diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index fbf91f50b..c7202d77b 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1094,11 +1094,11 @@ fn test_get_subtree() { let element = Element::Item(b"ayy".to_vec()); // Returns error is subtree is not valid - let subtree = db.get_subtree(&[TEST_LEAF, b"invalid_tree"], None); + let subtree = db.get_subtrees().get_subtree(&[TEST_LEAF, b"invalid_tree"], None); assert_eq!(subtree.is_err(), true); // Doesn't return an error for subtree that exists but empty - let subtree = db.get_subtree(&[TEST_LEAF], None); + let subtree = db.get_subtrees().get_subtree(&[TEST_LEAF], None); assert_eq!(subtree.is_err(), false); // Insert some nested subtrees @@ -1126,7 +1126,7 @@ fn test_get_subtree() { // Retrieve subtree instance // Check if it returns the same instance that was inserted - let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"key2"], None).unwrap(); + let subtree = db.get_subtrees().get_subtree(&[TEST_LEAF, b"key1", b"key2"], None).unwrap(); let result_element = Element::get(&subtree, b"key3").unwrap(); assert_eq!(result_element, Element::Item(b"ayy".to_vec())); @@ -1152,7 +1152,7 @@ fn test_get_subtree() { .expect("successful value insert"); // Retrieve subtree instance without transaction - let subtree = db.get_subtree(&[TEST_LEAF, b"key1", b"innertree"], Some(&transaction)).unwrap(); + let subtree = db.get_subtrees().get_subtree(&[TEST_LEAF, b"key1", b"innertree"], Some(&transaction)).unwrap(); let result_element = Element::get(&subtree, b"key4").unwrap(); assert_eq!(result_element, Element::Item(b"ayy".to_vec())); } From a43d0d6a63f5077c77dd2219c759f84fd4848810 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Fri, 21 Jan 2022 21:17:29 +0700 Subject: [PATCH 11/31] removed unnecessary get_subtrees_for_transaction function --- grovedb/src/lib.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a305bca91..18004c25e 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -497,16 +497,6 @@ impl GroveDb { .map_err(PrefixedRocksDbStorageError::RocksDbError)?) } - pub fn get_subtrees_for_transaction( - &mut self, - transaction: Option<&OptimisticTransactionDBTransaction>, - ) -> &HashMap, Merk> { - match transaction { - None => &self.subtrees, - Some(_) => &self.temp_subtrees, - } - } - /// Rollbacks previously started db transaction to initial state. /// For more details on the transaction usage, please check /// [`GroveDb::start_transaction`] From e7db197e9fd846024124eb15a340b717456f892e Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Sat, 22 Jan 2022 13:01:37 +0700 Subject: [PATCH 12/31] refactor + fmt --- grovedb/src/lib.rs | 12 ++--- grovedb/src/operations/get.rs | 11 +++-- grovedb/src/operations/is_empty_tree.rs | 2 +- grovedb/src/subtrees.rs | 32 +++++++++----- grovedb/src/tests.rs | 58 ++++++++++++++++++------- 5 files changed, 77 insertions(+), 38 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 9bf538dac..68122e7db 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -1,9 +1,9 @@ mod operations; mod subtree; +mod subtrees; #[cfg(test)] mod tests; mod transaction; -mod subtrees; use std::{collections::HashMap, path::Path, rc::Rc}; @@ -366,11 +366,11 @@ impl GroveDb { } fn get_subtrees(&self) -> Subtrees { - Subtrees{ - root_leaf_keys: &self.root_leaf_keys, - temp_subtrees: &self.temp_subtrees, - storage: self.storage(), - } + Subtrees { + root_leaf_keys: &self.root_leaf_keys, + temp_subtrees: &self.temp_subtrees, + storage: self.storage(), + } } /// A helper method to build a prefix to rocksdb keys or identify a subtree diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 18ab7549e..64fdd696a 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -71,7 +71,7 @@ impl GroveDb { key: &[u8], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - Element::get(&self.get_subtrees().get_subtree(path, transaction)?, key) + Element::get(&self.get_subtrees().get(path, transaction)?, key) } pub fn get_path_queries( @@ -90,9 +90,12 @@ impl GroveDb { } else { Err(Error::InvalidQuery("the reference must result in an item")) } - }, - _ => Err(Error::InvalidQuery("path_queries can only refer to references")), - }).collect::>, Error>>()?; + } + _ => Err(Error::InvalidQuery( + "path_queries can only refer to references", + )), + }) + .collect::>, Error>>()?; Ok(results) } diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index e53c8bcf6..1f243c1ed 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -8,7 +8,7 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let merk = self.get_subtrees().get_subtree(path, transaction)?; + let merk = self.get_subtrees().get(path, transaction)?; let mut iter = merk.raw_iter(); iter.seek_to_first(); diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs index 4d24f6725..04e0b03aa 100644 --- a/grovedb/src/subtrees.rs +++ b/grovedb/src/subtrees.rs @@ -1,20 +1,26 @@ //! Module for retrieving subtrees -use std::{collections::HashMap}; -use merk::{Merk}; -use crate::{GroveDb, Error, Element}; -use std::{rc::Rc}; +use std::{collections::HashMap, rc::Rc}; -use storage::rocksdb_storage::{OptimisticTransactionDBTransaction, PrefixedRocksDbStorage}; -use storage::RawIterator; +use merk::Merk; +use storage::{ + rocksdb_storage::{OptimisticTransactionDBTransaction, PrefixedRocksDbStorage}, + RawIterator, +}; -pub struct Subtrees<'a>{ +use crate::{Element, Error, GroveDb}; + +pub struct Subtrees<'a> { pub root_leaf_keys: &'a HashMap, usize>, pub temp_subtrees: &'a HashMap, Merk>, pub storage: Rc, } impl Subtrees<'_> { - pub fn get_subtree(&self, path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>) -> Result, Error> { + pub fn get( + &self, + path: &[&[u8]], + transaction: Option<&OptimisticTransactionDBTransaction>, + ) -> Result, Error> { match transaction { None => self.get_subtree_without_transaction(path), Some(_) => self.get_subtree_with_transaction(path), @@ -80,8 +86,11 @@ impl Subtrees<'_> { key: Option<&[u8]>, ) -> Result<(Merk, bool), Error> { let subtree_prefix = GroveDb::compress_subtree_key(path, key); - let merk = Merk::open(PrefixedRocksDbStorage::new(self.storage.clone(), subtree_prefix)?) - .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + let merk = Merk::open(PrefixedRocksDbStorage::new( + self.storage.clone(), + subtree_prefix, + )?) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; let mut has_keys = false; { let mut iter = merk.raw_iter(); @@ -92,5 +101,4 @@ impl Subtrees<'_> { } Ok((merk, has_keys)) } - -} \ No newline at end of file +} diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 2b59c00b3..6e56e4b5a 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -347,26 +347,40 @@ fn test_proof_construction() { // Insert level 2 nodes let mut inner_tree = TempMerk::new(); let value_one = Element::Item(b"value1".to_vec()); - value_one.insert(&mut inner_tree, b"key1".to_vec(), None).unwrap(); + value_one + .insert(&mut inner_tree, b"key1".to_vec(), None) + .unwrap(); let value_two = Element::Item(b"value2".to_vec()); - value_two.insert(&mut inner_tree, b"key2".to_vec(), None).unwrap(); + value_two + .insert(&mut inner_tree, b"key2".to_vec(), None) + .unwrap(); let mut inner_tree_2 = TempMerk::new(); let value_three = Element::Item(b"value3".to_vec()); - value_three.insert(&mut inner_tree_2, b"key3".to_vec(), None).unwrap(); + value_three + .insert(&mut inner_tree_2, b"key3".to_vec(), None) + .unwrap(); let mut inner_tree_3 = TempMerk::new(); let value_four = Element::Item(b"value4".to_vec()); - value_four.insert(&mut inner_tree_3, b"key4".to_vec(), None).unwrap(); + value_four + .insert(&mut inner_tree_3, b"key4".to_vec(), None) + .unwrap(); // Insert level 1 nodes let mut test_leaf = TempMerk::new(); let inner_tree_root = Element::Tree(inner_tree.root_hash()); - inner_tree_root.insert(&mut test_leaf, b"innertree".to_vec(), None).unwrap(); + inner_tree_root + .insert(&mut test_leaf, b"innertree".to_vec(), None) + .unwrap(); let mut another_test_leaf = TempMerk::new(); let inner_tree_2_root = Element::Tree(inner_tree_2.root_hash()); - inner_tree_2_root.insert(&mut another_test_leaf, b"innertree2".to_vec(), None).unwrap(); + inner_tree_2_root + .insert(&mut another_test_leaf, b"innertree2".to_vec(), None) + .unwrap(); let inner_tree_3_root = Element::Tree(inner_tree_3.root_hash()); - inner_tree_3_root.insert(&mut another_test_leaf, b"innertree3".to_vec(), None).unwrap(); + inner_tree_3_root + .insert(&mut another_test_leaf, b"innertree3".to_vec(), None) + .unwrap(); // Insert root nodes let leaves = [test_leaf.root_hash(), another_test_leaf.root_hash()]; let root_tree = MerkleTree::::from_leaves(&leaves); @@ -738,7 +752,8 @@ fn test_is_empty_tree() { b"innertree".to_vec(), Element::empty_tree(), None, - ).unwrap(); + ) + .unwrap(); assert_eq!( db.is_empty_tree(&[TEST_LEAF, b"innertree"], None) @@ -752,7 +767,8 @@ fn test_is_empty_tree() { b"key1".to_vec(), Element::Item(b"hello".to_vec()), None, - ).unwrap(); + ) + .unwrap(); assert_eq!( db.is_empty_tree(&[TEST_LEAF, b"innertree"], None) .expect("path is valid tree"), @@ -927,7 +943,8 @@ fn transaction_should_be_aborted() { item_key.clone(), element.clone(), Some(&transaction), - ).unwrap(); + ) + .unwrap(); db.abort_transaction(transaction).unwrap(); @@ -1094,11 +1111,13 @@ fn test_get_subtree() { let element = Element::Item(b"ayy".to_vec()); // Returns error is subtree is not valid - let subtree = db.get_subtrees().get_subtree(&[TEST_LEAF, b"invalid_tree"], None); + let subtree = db + .get_subtrees() + .get(&[TEST_LEAF, b"invalid_tree"], None); assert_eq!(subtree.is_err(), true); // Doesn't return an error for subtree that exists but empty - let subtree = db.get_subtrees().get_subtree(&[TEST_LEAF], None); + let subtree = db.get_subtrees().get(&[TEST_LEAF], None); assert_eq!(subtree.is_err(), false); // Insert some nested subtrees @@ -1126,7 +1145,10 @@ fn test_get_subtree() { // Retrieve subtree instance // Check if it returns the same instance that was inserted - let subtree = db.get_subtrees().get_subtree(&[TEST_LEAF, b"key1", b"key2"], None).unwrap(); + let subtree = db + .get_subtrees() + .get(&[TEST_LEAF, b"key1", b"key2"], None) + .unwrap(); let result_element = Element::get(&subtree, b"key3").unwrap(); assert_eq!(result_element, Element::Item(b"ayy".to_vec())); @@ -1152,7 +1174,10 @@ fn test_get_subtree() { .expect("successful value insert"); // Retrieve subtree instance without transaction - let subtree = db.get_subtrees().get_subtree(&[TEST_LEAF, b"key1", b"innertree"], Some(&transaction)).unwrap(); + let subtree = db + .get_subtrees() + .get(&[TEST_LEAF, b"key1", b"innertree"], Some(&transaction)) + .unwrap(); let result_element = Element::get(&subtree, b"key4").unwrap(); assert_eq!(result_element, Element::Item(b"ayy".to_vec())); } @@ -2182,7 +2207,10 @@ fn test_root_hash() { ) .expect("unable to insert an item"); let root_hash_outside = db.root_hash(None); - assert_ne!(db.root_hash(Some(&transaction)).unwrap(), root_hash_outside.unwrap()); + assert_ne!( + db.root_hash(Some(&transaction)).unwrap(), + root_hash_outside.unwrap() + ); assert_eq!(db.root_hash(None).unwrap(), root_hash_outside.unwrap()); db.commit_transaction(transaction).unwrap(); From 2a9d017b2cd3c277102d205014b3530a051e200f Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Sat, 22 Jan 2022 15:55:19 +0700 Subject: [PATCH 13/31] building root tree with new subtree interface --- grovedb/src/lib.rs | 133 ++++++++++++++++++++----------- grovedb/src/operations/insert.rs | 42 ++++++---- grovedb/src/subtrees.rs | 13 ++- grovedb/src/tests.rs | 14 +++- 4 files changed, 135 insertions(+), 67 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 68122e7db..82ffaf840 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -210,8 +210,15 @@ impl GroveDb { HashMap::new() }; + let temp_subtrees: HashMap, Merk> = HashMap::new(); + let subtrees_view = Subtrees { + root_leaf_keys: &root_leaf_keys, + temp_subtrees: &temp_subtrees, + storage: db.clone(), + }; + Ok(GroveDb::new( - Self::build_root_tree(&subtrees, &root_leaf_keys), + Self::build_root_tree(subtrees_view, &root_leaf_keys, None), root_leaf_keys, subtrees, meta_storage, @@ -291,13 +298,15 @@ impl GroveDb { } fn build_root_tree( - subtrees: &HashMap, Merk>, + // subtrees: &HashMap, Merk>, + subtrees: Subtrees, root_leaf_keys: &HashMap, usize>, + transaction: Option<&OptimisticTransactionDBTransaction>, ) -> MerkleTree { let mut leaf_hashes: Vec<[u8; 32]> = vec![[0; 32]; root_leaf_keys.len()]; for (subtree_path, root_leaf_idx) in root_leaf_keys { let subtree_merk = subtrees - .get(subtree_path) + .get(&[subtree_path.as_slice()], transaction) .expect("`root_leaf_keys` must be in sync with `subtrees`"); leaf_hashes[*root_leaf_idx] = subtree_merk.root_hash(); } @@ -309,14 +318,18 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let subtrees = match transaction { - None => &self.subtrees, - Some(_) => &self.temp_subtrees, - }; - - let merk = subtrees - .get(&Self::compress_subtree_key(path, None)) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; + // let subtrees = match transaction { + // None => &self.subtrees, + // Some(_) => &self.temp_subtrees, + // }; + + // let merk = subtrees + // .get(&Self::compress_subtree_key(path, None)) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + let merk = self + .get_subtrees() + .get(path, transaction) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; Ok(Element::iterator(merk.raw_iter())) } @@ -326,42 +339,72 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&'b ::DBTransaction<'b>>, ) -> Result<(), Error> { - let subtrees = match transaction { - None => &mut self.subtrees, - Some(_) => &mut self.temp_subtrees, - }; + // let subtrees = match transaction { + // None => &mut self.subtrees, + // Some(_) => &mut self.temp_subtrees, + // }; + let subtrees = self.get_subtrees(); + + // We are given a path to a subtree that was just changed + // We want to get it's root hash and then propagate changes up + // There is the case were the thing that was changed is the root leaf node + // if length == 1 then a root leaf node was changed + // if length > 1 then a tree further down was changed + // for root leaf nodes, we have to rebuild the root tree + // for others we just update them + let mut path = path; + // let mut split_path = path.split_last(); + // Go up until only one element in path, which means a key of a root tree + while path.len() > 1 { + // non root leaf node + let subtree = subtrees.get(path, transaction)?; + let element = Element::Tree(subtree.root_hash()); - let root_leaf_keys = match transaction { - None => &mut self.root_leaf_keys, - Some(_) => &mut self.temp_root_leaf_keys, - }; + let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; + let mut upper_tree = subtrees.get(path, transaction)?; + element.insert(&mut upper_tree, key.to_vec(), transaction); - let mut split_path = path.split_last(); - // Go up until only one element in path, which means a key of a root tree - while let Some((key, path_slice)) = split_path { - if path_slice.is_empty() { - // Hit the root tree - match transaction { - None => self.root_tree = Self::build_root_tree(subtrees, root_leaf_keys), - Some(_) => { - self.temp_root_tree = Self::build_root_tree(subtrees, root_leaf_keys) - } - }; - break; - } else { - let compressed_path_upper_tree = Self::compress_subtree_key(path_slice, None); - let compressed_path_subtree = Self::compress_subtree_key(path_slice, Some(key)); - let subtree = subtrees - .get(&compressed_path_subtree) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; - let element = Element::Tree(subtree.root_hash()); - let upper_tree = subtrees - .get_mut(&compressed_path_upper_tree) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; - element.insert(upper_tree, key.to_vec(), transaction)?; - split_path = path_slice.split_last(); + path = parent_path; + } + + // root leaf nodes + if path.len() == 1 { + let root_leaf_keys = match transaction { + None => &self.root_leaf_keys, + Some(_) => &self.temp_root_leaf_keys, + }; + let root_tree = Self::build_root_tree(subtrees, root_leaf_keys, transaction); + match transaction { + None => self.root_tree = root_tree, + Some(_) => self.temp_root_tree = root_tree, } } + + // while let Some((key, path_slice)) = split_path { + // if path_slice.is_empty() { + // // Hit the root tree + // match transaction { + // None => self.root_tree = Self::build_root_tree(subtrees, + // root_leaf_keys), Some(_) => { + // self.temp_root_tree = Self::build_root_tree(subtrees, + // root_leaf_keys) } + // }; + // break; + // } else { + // let compressed_path_upper_tree = + // Self::compress_subtree_key(path_slice, None); let + // compressed_path_subtree = Self::compress_subtree_key(path_slice, Some(key)); + // let subtree = subtrees + // .get(&compressed_path_subtree) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + // let element = Element::Tree(subtree.root_hash()); + // let upper_tree = subtrees + // .get_mut(&compressed_path_upper_tree) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + // element.insert(upper_tree, key.to_vec(), transaction)?; + // split_path = path_slice.split_last(); + // } + // } Ok(()) } @@ -464,7 +507,7 @@ impl GroveDb { // Cloning all the trees to maintain original state before the transaction self.temp_root_tree = self.root_tree.clone(); self.temp_root_leaf_keys = self.root_leaf_keys.clone(); - self.temp_subtrees = self.subtrees.clone(); + // self.temp_subtrees = self.subtrees.clone(); Ok(()) } @@ -506,7 +549,7 @@ impl GroveDb { // Cloning all the trees to maintain to rollback transactional changes self.temp_root_tree = self.root_tree.clone(); self.temp_root_leaf_keys = self.root_leaf_keys.clone(); - self.temp_subtrees = self.subtrees.clone(); + // self.temp_subtrees = self.subtrees.clone(); Ok(db_transaction .rollback() diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index d58d2c379..dce1be211 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -98,10 +98,12 @@ impl GroveDb { subtrees.insert(subtree_prefix.clone(), subtree_merk); // Update root leafs index to persist rs-merkle structure later - if root_leaf_keys.get(&subtree_prefix).is_none() { - root_leaf_keys.insert(subtree_prefix, root_tree.leaves_len()); + if root_leaf_keys.get(&key.to_vec()).is_none() { + root_leaf_keys.insert(key.to_vec(), root_tree.leaves_len()); } + dbg!("propagating after adding root"); self.propagate_changes(&[key], transaction)?; + dbg!("done propagating after adding root"); Ok(()) } @@ -118,27 +120,35 @@ impl GroveDb { } } - let subtrees = match transaction { - None => &mut self.subtrees, - Some(_) => &mut self.temp_subtrees, - }; - - let compressed_path = Self::compress_subtree_key(path, None); // First, check if a subtree exists to create a new subtree under it - subtrees - .get(&compressed_path) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; + self.get_subtrees().get(path, transaction)?; + let (subtree_prefix, subtree_merk) = create_merk_with_prefix(self.db.clone(), path, &key)?; + // Set tree value as a a subtree root hash let element = Element::Tree(subtree_merk.root_hash()); - subtrees.insert(subtree_prefix, subtree_merk); + + // Save subtrees, to be removed + // TODO: Remove this + { + let subtrees = match transaction { + None => &mut self.subtrees, + Some(_) => &mut self.temp_subtrees, + }; + subtrees.insert(subtree_prefix, subtree_merk); + } + // Had to take merk from `subtrees` once again to solve multiple &mut s - let mut merk = subtrees - .get_mut(&compressed_path) - .expect("merk object must exist in `subtrees`"); + let mut merk = self + .get_subtrees() + .get(path, transaction) + .expect("confirmed subtree exists above"); + // need to mark key as taken in the upper tree - element.insert(merk, key, transaction)?; + element.insert(&mut merk, key, transaction)?; + self.propagate_changes(path, transaction)?; + Ok(()) } diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs index 04e0b03aa..c6a65787e 100644 --- a/grovedb/src/subtrees.rs +++ b/grovedb/src/subtrees.rs @@ -9,6 +9,7 @@ use storage::{ use crate::{Element, Error, GroveDb}; +// TODO: should take temp_root_leaf_keys also pub struct Subtrees<'a> { pub root_leaf_keys: &'a HashMap, usize>, pub temp_subtrees: &'a HashMap, Merk>, @@ -42,7 +43,9 @@ impl Subtrees<'_> { // if parent path is empty, we are dealing with root leaf node // we can confirm validity of a root leaf node by checking root_leaf_keys if parent_path.is_empty() { - return if self.root_leaf_keys.contains_key(&subtree_prefix) { + dbg!("parent path is empty, checking the root tree"); + let root_key = path[0].to_vec(); + return if self.root_leaf_keys.contains_key(&root_key) { Ok(subtree) } else { Err(Error::InvalidPath("no subtree found under that path")) @@ -76,7 +79,13 @@ impl Subtrees<'_> { if let Some(merk) = self.temp_subtrees.get(&subtree_prefix) { Ok(merk.clone()) } else { - Err(Error::InvalidPath("no subtree found under that path")) + dbg!("Getting subtree without transaction"); + // if the subtree doesn't exist in temp_subtrees, + // check if it was created before the transaction was started + let merk = self + .get_subtree_without_transaction(path) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + Ok(merk) } } diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 6e56e4b5a..9f229e040 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1111,9 +1111,7 @@ fn test_get_subtree() { let element = Element::Item(b"ayy".to_vec()); // Returns error is subtree is not valid - let subtree = db - .get_subtrees() - .get(&[TEST_LEAF, b"invalid_tree"], None); + let subtree = db.get_subtrees().get(&[TEST_LEAF, b"invalid_tree"], None); assert_eq!(subtree.is_err(), true); // Doesn't return an error for subtree that exists but empty @@ -1173,13 +1171,21 @@ fn test_get_subtree() { ) .expect("successful value insert"); - // Retrieve subtree instance without transaction + // Retrieve subtree instance with transaction let subtree = db .get_subtrees() .get(&[TEST_LEAF, b"key1", b"innertree"], Some(&transaction)) .unwrap(); let result_element = Element::get(&subtree, b"key4").unwrap(); assert_eq!(result_element, Element::Item(b"ayy".to_vec())); + + // Should be able to retrieve instances created before transaction + let subtree = db + .get_subtrees() + .get(&[TEST_LEAF, b"key1", b"key2"], None) + .unwrap(); + let result_element = Element::get(&subtree, b"key3").unwrap(); + assert_eq!(result_element, Element::Item(b"ayy".to_vec())); } #[test] From 046a913361b286cd571328cb21872fb5a5d39770 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Sat, 22 Jan 2022 16:48:04 +0700 Subject: [PATCH 14/31] overflow (wip) --- grovedb/src/lib.rs | 22 +++++++++++----------- grovedb/src/subtrees.rs | 2 +- grovedb/src/tests.rs | 2 ++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 82ffaf840..9e3816423 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -318,18 +318,18 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - // let subtrees = match transaction { - // None => &self.subtrees, - // Some(_) => &self.temp_subtrees, - // }; + let subtrees = match transaction { + None => &self.subtrees, + Some(_) => &self.temp_subtrees, + }; - // let merk = subtrees - // .get(&Self::compress_subtree_key(path, None)) - // .ok_or(Error::InvalidPath("no subtree found under that path"))?; - let merk = self - .get_subtrees() - .get(path, transaction) - .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + let merk = subtrees + .get(&Self::compress_subtree_key(path, None)) + .ok_or(Error::InvalidPath("no subtree found under that path"))?; + // let merk = self + // .get_subtrees() + // .get(path, transaction) + // .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; Ok(Element::iterator(merk.raw_iter())) } diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs index c6a65787e..ac15701ff 100644 --- a/grovedb/src/subtrees.rs +++ b/grovedb/src/subtrees.rs @@ -43,7 +43,7 @@ impl Subtrees<'_> { // if parent path is empty, we are dealing with root leaf node // we can confirm validity of a root leaf node by checking root_leaf_keys if parent_path.is_empty() { - dbg!("parent path is empty, checking the root tree"); + // dbg!("parent path is empty, checking the root tree"); let root_key = path[0].to_vec(); return if self.root_leaf_keys.contains_key(&root_key) { Ok(subtree) diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 9f229e040..cb37b4720 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1646,6 +1646,8 @@ fn test_get_range_inclusive_query_with_non_unique_subquery() { fn test_get_range_inclusive_query_with_non_unique_subquery_on_references() { let mut db = make_grovedb(); populate_tree_by_reference_for_non_unique_range_subquery(&mut db); + panic!("We got here"); + dbg!("failed when populating"); let path = vec![TEST_LEAF, b"1"]; let mut query = Query::new(); From c1ffa8dd1803ac9c5a79e2f22e936d10ee8846a2 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Sun, 23 Jan 2022 09:17:37 +0700 Subject: [PATCH 15/31] fixed overflow issue --- grovedb/src/operations/insert.rs | 7 ++++--- grovedb/src/tests.rs | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index dce1be211..d4282b040 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -56,9 +56,10 @@ impl GroveDb { )); } // Get a Merk by a path - let mut merk = subtrees - .get_mut(&Self::compress_subtree_key(path, None)) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; + // let mut merk = subtrees + // .get_mut(&Self::compress_subtree_key(path, None)) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + let mut merk = self.get_subtrees().get(path, transaction).map_err(|_| Error::InvalidPath("no subtree found under that path"))?; element.insert(&mut merk, key, transaction)?; self.propagate_changes(path, transaction)?; } diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index cb37b4720..3389c8aa9 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1646,7 +1646,6 @@ fn test_get_range_inclusive_query_with_non_unique_subquery() { fn test_get_range_inclusive_query_with_non_unique_subquery_on_references() { let mut db = make_grovedb(); populate_tree_by_reference_for_non_unique_range_subquery(&mut db); - panic!("We got here"); dbg!("failed when populating"); let path = vec![TEST_LEAF, b"1"]; From 22def5b79ca1506b246363c16f09e3ca9f3a9fb9 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Sun, 23 Jan 2022 10:51:21 +0700 Subject: [PATCH 16/31] refactor range queries to use new subtree --- grovedb/src/lib.rs | 2 +- grovedb/src/operations/get.rs | 43 +++++++++++++++++++++++------------ grovedb/src/subtree.rs | 42 ++++++++++++++++++++++------------ 3 files changed, 57 insertions(+), 30 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 9e3816423..5edbaf027 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -361,7 +361,7 @@ impl GroveDb { let element = Element::Tree(subtree.root_hash()); let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; - let mut upper_tree = subtrees.get(path, transaction)?; + let mut upper_tree = subtrees.get(parent_path, transaction)?; element.insert(&mut upper_tree, key.to_vec(), transaction); path = parent_path; diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 64fdd696a..f6a9fb8ad 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -6,9 +6,10 @@ use std::{ }; use merk::Merk; +use storage::RawIterator; use storage::rocksdb_storage::{OptimisticTransactionDBTransaction, PrefixedRocksDbStorage}; -use crate::{Element, Error, GroveDb, PathQuery}; +use crate::{Element, Error, GroveDb, PathQuery, Subtrees}; /// Limit of possible indirections pub(crate) const MAX_REFERENCE_HOPS: usize = 10; @@ -122,6 +123,7 @@ impl GroveDb { .into_iter() .map(|element| match element { Element::Reference(reference_path) => { + dbg!("ref"); let maybe_item = self.follow_reference(reference_path, transaction)?; if let Element::Item(item) = maybe_item { Ok(item) @@ -129,10 +131,17 @@ impl GroveDb { Err(Error::InvalidQuery("the reference must result in an item")) } } - Element::Item(item) => Ok(item), - Element::Tree(_) => Err(Error::InvalidQuery( - "path_queries can only refer to items and references", - )), + Element::Item(item) => { + dbg!("item"); + Ok(item) + }, + Element::Tree(item) => { + dbg!("tree"); + // Err(Error::InvalidQuery( + // "path_queries can only refer to items and references", + // )) + Ok(item.to_vec()) + }, }) .collect::>, Error>>()?; Ok((results, skipped)) @@ -143,22 +152,26 @@ impl GroveDb { path_query: &PathQuery, transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result<(Vec, u16), Error> { - let subtrees = match transaction { - None => &self.subtrees, - Some(_) => &self.temp_subtrees, - }; - self.get_path_query_on_trees_raw(path_query, subtrees) + // let subtrees = match transaction { + // None => &self.subtrees, + // Some(_) => &self.temp_subtrees, + // }; + let subtrees = self.get_subtrees(); + self.get_path_query_on_trees_raw(path_query, subtrees, transaction) } fn get_path_query_on_trees_raw( &self, path_query: &PathQuery, - subtrees: &HashMap, Merk>, + subtrees: Subtrees, + transaction: Option<&OptimisticTransactionDBTransaction>, + // subtrees: &HashMap, Merk>, ) -> Result<(Vec, u16), Error> { let path = path_query.path; - let merk = subtrees - .get(&Self::compress_subtree_key(path, None)) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; - Element::get_path_query(merk, path_query, Some(subtrees)) + // let merk = subtrees + // .get(&Self::compress_subtree_key(path, None)) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + let merk = subtrees.get(path, transaction).map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + Element::get_path_query(&merk, path_query, Some(&subtrees)) } } diff --git a/grovedb/src/subtree.rs b/grovedb/src/subtree.rs index f798a7222..c23905677 100644 --- a/grovedb/src/subtree.rs +++ b/grovedb/src/subtree.rs @@ -2,6 +2,7 @@ //! Subtrees handling is isolated so basically this module is about adapting //! Merk API to GroveDB needs. use std::collections::HashMap; +use std::ops::Sub; use merk::{ proofs::{query::QueryItem, Query}, @@ -17,7 +18,7 @@ use storage::{ RawIterator, Storage, Store, }; -use crate::{Error, GroveDb, Merk, PathQuery, SizedQuery}; +use crate::{Error, GroveDb, Merk, PathQuery, SizedQuery, Subtrees}; /// Variants of GroveDB stored entities #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -73,7 +74,8 @@ impl Element { } fn basic_push( - _subtrees: Option<&HashMap, Merk>>, + // _subtrees: Option<&HashMap, Merk>>, + _subtree: Option<&Subtrees>, _key: Option<&[u8]>, element: Element, _path: Option<&[&[u8]]>, @@ -84,6 +86,7 @@ impl Element { limit: &mut Option, offset: &mut Option, ) -> Result<(), Error> { + dbg!("basic push"); if offset.is_none() || offset.is_some() && offset.unwrap() == 0 { results.push(element); if limit.is_some() { @@ -96,7 +99,8 @@ impl Element { } fn path_query_push( - subtrees_option: Option<&HashMap, Merk>>, + // subtrees_option: Option<&HashMap, Merk>>, + subtrees_option: Option<&Subtrees>, key: Option<&[u8]>, element: Element, path: Option<&[&[u8]]>, @@ -107,10 +111,13 @@ impl Element { limit: &mut Option, offset: &mut Option, ) -> Result<(), Error> { + dbg!("path query push"); match element { Element::Tree(_) => { + dbg!("tree element"); // if the query had a subquery then we should get elements from it if let Some(subquery_key) = subquery_key_option { + dbg!("getting the actual value"); let subtrees = subtrees_option.ok_or(Error::MissingParameter( "subtrees must be provided when using a subquery key", ))?; @@ -128,12 +135,14 @@ impl Element { if let Some(subquery) = subquery { path_vec.push(subquery_key.as_slice()); - let inner_merk = subtrees - .get(&GroveDb::compress_subtree_key(path_vec.as_slice(), None)) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; + // let inner_merk = subtrees + // .get(&GroveDb::compress_subtree_key(path_vec.as_slice(), None)) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + let inner_merk = subtrees.get(path_vec.as_slice(), None).unwrap(); let inner_query = SizedQuery::new(subquery, *limit, *offset, left_to_right); let (mut sub_elements, skipped) = - Element::get_sized_query(inner_merk, &inner_query)?; + Element::get_sized_query(&inner_merk, &inner_query)?; + dbg!(sub_elements.len()); if let Some(limit) = limit { *limit -= sub_elements.len() as u16; } @@ -142,11 +151,12 @@ impl Element { } results.append(&mut sub_elements); } else { - let inner_merk = subtrees - .get(&GroveDb::compress_subtree_key(path_vec.as_slice(), None)) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; + let inner_merk = subtrees.get(path_vec.as_slice(), None).unwrap(); + // let inner_merk = subtrees + // .get(&GroveDb::compress_subtree_key(path_vec.as_slice(), None)) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; if offset.is_none() || offset.is_some() && offset.unwrap() == 0 { - results.push(Element::get(inner_merk, subquery_key.as_slice())?); + results.push(Element::get(&inner_merk, subquery_key.as_slice())?); if limit.is_some() { *limit = Some(limit.unwrap() - 1); } @@ -192,9 +202,11 @@ impl Element { path: Option<&[&[u8]]>, subquery_key: Option>, subquery: Option, - subtrees: Option<&HashMap, Merk>>, + // subtrees: Option<&HashMap, Merk>>, + subtrees: Option<&Subtrees>, add_element_function: fn( - subtrees: Option<&HashMap, Merk>>, + // subtrees: Option<&HashMap, Merk>>, + subtrees: Option<&Subtrees>, key: Option<&[u8]>, element: Element, path: Option<&[&[u8]]>, @@ -273,6 +285,7 @@ impl Element { } else { 0 }; + dbg!("results {}", results.len()); Ok((results, skipped)) } @@ -280,7 +293,8 @@ impl Element { pub fn get_path_query( merk: &Merk, path_query: &PathQuery, - subtrees: Option<&HashMap, Merk>>, + // subtrees: Option<&HashMap, Merk>>, + subtrees: Option<&Subtrees>, ) -> Result<(Vec, u16), Error> { Element::get_query_apply_function( merk, From 88208c4c5eba75f34a65243540522a53e2976923 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 11:28:52 +0700 Subject: [PATCH 17/31] compiles --- grovedb/src/lib.rs | 130 +++++++++++++++---------------- grovedb/src/operations/delete.rs | 58 ++++++++++---- grovedb/src/operations/get.rs | 60 +++++++++++++- grovedb/src/operations/insert.rs | 40 +++++----- grovedb/src/subtrees.rs | 77 ++++++++++++------ 5 files changed, 239 insertions(+), 126 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 5edbaf027..28e7137a6 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -5,7 +5,7 @@ mod subtrees; mod tests; mod transaction; -use std::{collections::HashMap, path::Path, rc::Rc}; +use std::{collections::HashMap, path::Path, rc::Rc, cell::RefCell}; pub use merk::proofs::{query::QueryItem, Query}; use merk::{self, Merk}; @@ -149,7 +149,7 @@ pub struct GroveDb { // Temp trees used for writes during transaction temp_root_tree: MerkleTree, temp_root_leaf_keys: HashMap, usize>, - temp_subtrees: HashMap, Merk>, + temp_subtrees: RefCell, Merk>>, } impl GroveDb { @@ -168,7 +168,7 @@ impl GroveDb { db, temp_root_tree: MerkleTree::new(), temp_root_leaf_keys: HashMap::new(), - temp_subtrees: HashMap::new(), + temp_subtrees: RefCell::new(HashMap::new()), is_readonly: false, } } @@ -210,7 +210,7 @@ impl GroveDb { HashMap::new() }; - let temp_subtrees: HashMap, Merk> = HashMap::new(); + let temp_subtrees: RefCell, Merk>> = RefCell::new(HashMap::new()); let subtrees_view = Subtrees { root_leaf_keys: &root_leaf_keys, temp_subtrees: &temp_subtrees, @@ -249,53 +249,53 @@ impl GroveDb { } } - fn store_subtrees_keys_data( - &self, - db_transaction: Option<&OptimisticTransactionDBTransaction>, - ) -> Result<(), Error> { - let subtrees = match db_transaction { - None => &self.subtrees, - Some(_) => &self.temp_subtrees, - }; - - let prefixes: Vec> = subtrees.keys().cloned().collect(); - - // TODO: make StorageOrTransaction which will has the access to either storage - // or transaction - match db_transaction { - None => { - self.meta_storage.put_meta( - SUBTREES_SERIALIZED_KEY, - &bincode::serialize(&prefixes).map_err(|_| { - Error::CorruptedData(String::from("unable to serialize prefixes")) - })?, - )?; - self.meta_storage.put_meta( - ROOT_LEAFS_SERIALIZED_KEY, - &bincode::serialize(&self.temp_root_leaf_keys).map_err(|_| { - Error::CorruptedData(String::from("unable to serialize root leafs")) - })?, - )?; - } - Some(tx) => { - let transaction = self.meta_storage.transaction(tx); - transaction.put_meta( - SUBTREES_SERIALIZED_KEY, - &bincode::serialize(&prefixes).map_err(|_| { - Error::CorruptedData(String::from("unable to serialize prefixes")) - })?, - )?; - transaction.put_meta( - ROOT_LEAFS_SERIALIZED_KEY, - &bincode::serialize(&self.root_leaf_keys).map_err(|_| { - Error::CorruptedData(String::from("unable to serialize root leafs")) - })?, - )?; - } - } - - Ok(()) - } + // fn store_subtrees_keys_data( + // &self, + // db_transaction: Option<&OptimisticTransactionDBTransaction>, + // ) -> Result<(), Error> { + // let subtrees = match db_transaction { + // None => &self.subtrees, + // Some(_) => &self.temp_subtrees, + // }; + // + // let prefixes: Vec> = subtrees.keys().cloned().collect(); + // + // // TODO: make StorageOrTransaction which will has the access to either storage + // // or transaction + // match db_transaction { + // None => { + // self.meta_storage.put_meta( + // SUBTREES_SERIALIZED_KEY, + // &bincode::serialize(&prefixes).map_err(|_| { + // Error::CorruptedData(String::from("unable to serialize prefixes")) + // })?, + // )?; + // self.meta_storage.put_meta( + // ROOT_LEAFS_SERIALIZED_KEY, + // &bincode::serialize(&self.temp_root_leaf_keys).map_err(|_| { + // Error::CorruptedData(String::from("unable to serialize root leafs")) + // })?, + // )?; + // } + // Some(tx) => { + // let transaction = self.meta_storage.transaction(tx); + // transaction.put_meta( + // SUBTREES_SERIALIZED_KEY, + // &bincode::serialize(&prefixes).map_err(|_| { + // Error::CorruptedData(String::from("unable to serialize prefixes")) + // })?, + // )?; + // transaction.put_meta( + // ROOT_LEAFS_SERIALIZED_KEY, + // &bincode::serialize(&self.root_leaf_keys).map_err(|_| { + // Error::CorruptedData(String::from("unable to serialize root leafs")) + // })?, + // )?; + // } + // } + // + // Ok(()) + // } fn build_root_tree( // subtrees: &HashMap, Merk>, @@ -318,19 +318,19 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let subtrees = match transaction { - None => &self.subtrees, - Some(_) => &self.temp_subtrees, - }; - - let merk = subtrees - .get(&Self::compress_subtree_key(path, None)) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; - // let merk = self - // .get_subtrees() + // let subtrees = match transaction { + // None => &self.subtrees, + // Some(_) => &self.temp_subtrees, + // }; + todo!() + // let merk = self.get_subtrees() // .get(path, transaction) - // .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; - Ok(Element::iterator(merk.raw_iter())) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + // // let merk = self + // // .get_subtrees() + // // .get(path, transaction) + // // .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + // Ok(Element::iterator(merk.raw_iter())) } /// Method to propagate updated subtree root hashes up to GroveDB root @@ -530,7 +530,7 @@ impl GroveDb { // code can be reworked to account for that self.root_tree = self.temp_root_tree.clone(); self.root_leaf_keys = self.temp_root_leaf_keys.drain().collect(); - self.subtrees = self.temp_subtrees.drain().collect(); + // self.subtrees = self.temp_subtrees.drain().collect(); self.cleanup_transactional_data(); @@ -577,6 +577,6 @@ impl GroveDb { // Free transactional data self.temp_root_tree = MerkleTree::new(); self.temp_root_leaf_keys = HashMap::new(); - self.temp_subtrees = HashMap::new(); + self.temp_subtrees = RefCell::new(HashMap::new()); } } diff --git a/grovedb/src/operations/delete.rs b/grovedb/src/operations/delete.rs index b4c1f97d8..c8e4e0950 100644 --- a/grovedb/src/operations/delete.rs +++ b/grovedb/src/operations/delete.rs @@ -22,15 +22,42 @@ impl GroveDb { } else { let element = self.get_raw(path, &key, transaction)?; { - let subtrees = match transaction { - None => &mut self.subtrees, - Some(_) => &mut self.temp_subtrees, - }; + // To get the merk, we have two functions, + // 1. get the subtree without transaction + // 2. get with transaction + // - if temp doesn't have get without transaction + // how do we return a reference to the memory location tho + // if we might have to remove the merk so it is always the correct instance + let mut merk = self.get_subtrees().get(path, transaction)?; + // let mut merk; + // match transaction { + // None => { + // merk = self.get_subtrees().get_subtree_without_transaction(path)?; + // }, + // Some(_) => { + // let prefix = &Self::compress_subtree_key(path, None); + // if self.temp_subtrees.borrow().contains_key(prefix) { + // // get the merk out + // merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); + // } else { + // // merk is not in the hash map get it without transaction + // merk = self.get_subtrees().get_subtree_without_transaction(path)?; + // } + // } + // } - let mut merk = subtrees - .get_mut(&Self::compress_subtree_key(path, None)) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; - Element::delete(merk, key.clone(), transaction)?; + // let subtrees = match transaction { + // None => &mut self.subtrees, + // Some(_) => &mut self.temp_subtrees, + // }; + + // let mut merk = subtrees + // .get_mut(&Self::compress_subtree_key(path, None)) + // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + Element::delete(&mut merk, key.clone(), transaction)?; + + // after deletion, if there is a transaction, add the merk back into the hashmap + self.get_subtrees().insert_temp_tree(path, merk, transaction); } if let Element::Tree(_) = element { @@ -38,18 +65,21 @@ impl GroveDb { let mut concat_path: Vec> = path.iter().map(|x| x.to_vec()).collect(); concat_path.push(key); let subtrees_paths = self.find_subtrees(concat_path, transaction)?; - let subtrees = match transaction { - None => &mut self.subtrees, - Some(_) => &mut self.temp_subtrees, - }; + // let subtrees = match transaction { + // None => &mut self.subtrees, + // Some(_) => &mut self.temp_subtrees, + // }; for subtree_path in subtrees_paths { // TODO: eventually we need to do something about this nested slices let subtree_path_ref: Vec<&[u8]> = subtree_path.iter().map(|x| x.as_slice()).collect(); let prefix = Self::compress_subtree_key(&subtree_path_ref, None); - if let Some(mut subtree) = subtrees.remove(&prefix) { - subtree.clear(transaction).map_err(|e| { + // Here we seem to get the subtree, then call clear on it + // Seems to only care about subtrees that are non transactional + let subtree = self.get_subtrees().get_subtree_without_transaction(subtree_path_ref.as_slice()); + if subtree.is_ok(){ + subtree.expect("confirmed it's valid").clear(transaction).map_err(|e| { Error::CorruptedData(format!( "unable to cleanup tree from storage: {}", e diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index f6a9fb8ad..a5a7adaa7 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -72,7 +72,34 @@ impl GroveDb { key: &[u8], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - Element::get(&self.get_subtrees().get(path, transaction)?, key) + let merk; + match transaction { + None => { + merk = self.get_subtrees().get_subtree_without_transaction(path)?; + }, + Some(_) => { + let prefix = &Self::compress_subtree_key(path, None); + if self.temp_subtrees.borrow().contains_key(prefix) { + // get the merk out + merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); + } else { + // merk is not in the hash map get it without transaction + merk = self.get_subtrees().get_subtree_without_transaction(path)?; + } + } + } + + let elem = Element::get(&merk, key); + + match transaction { + None => {}, + Some(_) => { + let prefix = Self::compress_subtree_key(path, None); + self.temp_subtrees.borrow_mut().insert(prefix, merk); + } + }; + + elem } pub fn get_path_queries( @@ -171,7 +198,34 @@ impl GroveDb { // let merk = subtrees // .get(&Self::compress_subtree_key(path, None)) // .ok_or(Error::InvalidPath("no subtree found under that path"))?; - let merk = subtrees.get(path, transaction).map_err(|_| Error::InvalidPath("no subtree found under that path"))?; - Element::get_path_query(&merk, path_query, Some(&subtrees)) + let merk; + match transaction { + None => { + merk = self.get_subtrees().get_subtree_without_transaction(path)?; + }, + Some(_) => { + let prefix = &Self::compress_subtree_key(path, None); + if self.temp_subtrees.borrow().contains_key(prefix) { + // get the merk out + merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); + } else { + // merk is not in the hash map get it without transaction + merk = self.get_subtrees().get_subtree_without_transaction(path)?; + } + } + } + + let elem = Element::get_path_query(&merk, path_query, Some(&subtrees)); + + + match transaction { + None => {}, + Some(_) => { + let prefix = Self::compress_subtree_key(path, None); + self.temp_subtrees.borrow_mut().insert(prefix, merk); + } + }; + + elem } } diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index d4282b040..40021b411 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -33,11 +33,6 @@ impl GroveDb { } } - let subtrees = match transaction { - None => &mut self.subtrees, - Some(_) => &mut self.temp_subtrees, - }; - match element { Element::Tree(_) => { if path.is_empty() { @@ -45,7 +40,6 @@ impl GroveDb { } else { self.add_non_root_subtree(path, key, transaction)?; } - self.store_subtrees_keys_data(transaction)?; } _ => { // If path is empty that means there is an attempt to insert something into a @@ -61,6 +55,7 @@ impl GroveDb { // .ok_or(Error::InvalidPath("no subtree found under that path"))?; let mut merk = self.get_subtrees().get(path, transaction).map_err(|_| Error::InvalidPath("no subtree found under that path"))?; element.insert(&mut merk, key, transaction)?; + self.get_subtrees().insert_temp_tree(path, merk, transaction); self.propagate_changes(path, transaction)?; } } @@ -79,10 +74,16 @@ impl GroveDb { } } - let subtrees = match transaction { - None => &mut self.subtrees, - Some(_) => &mut self.temp_subtrees, - }; + // let subtrees = match transaction { + // None => &mut self.subtrees, + // Some(_) => &mut self.temp_subtrees, + // }; + + // Open Merk and put handle into `subtrees` dictionary accessible by its + // compressed path + let (subtree_prefix, subtree_merk) = create_merk_with_prefix(self.db.clone(), &[], key)?; + // subtrees.insert(subtree_prefix.clone(), subtree_merk); + self.get_subtrees().insert_temp_tree_with_prefix(subtree_prefix, subtree_merk, transaction); let root_leaf_keys = match transaction { None => &mut self.root_leaf_keys, @@ -93,11 +94,6 @@ impl GroveDb { None => &mut self.root_tree, Some(_) => &mut self.temp_root_tree, }; - // Open Merk and put handle into `subtrees` dictionary accessible by its - // compressed path - let (subtree_prefix, subtree_merk) = create_merk_with_prefix(self.db.clone(), &[], key)?; - subtrees.insert(subtree_prefix.clone(), subtree_merk); - // Update root leafs index to persist rs-merkle structure later if root_leaf_keys.get(&key.to_vec()).is_none() { root_leaf_keys.insert(key.to_vec(), root_tree.leaves_len()); @@ -131,13 +127,13 @@ impl GroveDb { // Save subtrees, to be removed // TODO: Remove this - { - let subtrees = match transaction { - None => &mut self.subtrees, - Some(_) => &mut self.temp_subtrees, - }; - subtrees.insert(subtree_prefix, subtree_merk); - } + // { + // let subtrees = match transaction { + // None => &mut self.subtrees, + // Some(_) => &mut self.temp_subtrees, + // }; + // subtrees.insert(subtree_prefix, subtree_merk); + // } // Had to take merk from `subtrees` once again to solve multiple &mut s let mut merk = self diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs index ac15701ff..d224134a8 100644 --- a/grovedb/src/subtrees.rs +++ b/grovedb/src/subtrees.rs @@ -1,5 +1,5 @@ //! Module for retrieving subtrees -use std::{collections::HashMap, rc::Rc}; +use std::{collections::HashMap, rc::Rc, cell::RefCell}; use merk::Merk; use storage::{ @@ -12,23 +12,55 @@ use crate::{Element, Error, GroveDb}; // TODO: should take temp_root_leaf_keys also pub struct Subtrees<'a> { pub root_leaf_keys: &'a HashMap, usize>, - pub temp_subtrees: &'a HashMap, Merk>, + pub temp_subtrees: &'a RefCell, Merk>>, pub storage: Rc, } impl Subtrees<'_> { + pub fn insert_temp_tree(self, path: &[&[u8]], merk: Merk, transaction: Option<&OptimisticTransactionDBTransaction>) -> Option> { + match transaction{ + None => None, + Some(_) => { + let prefix = GroveDb::compress_subtree_key(path, None); + self.temp_subtrees.borrow_mut().insert(prefix, merk) + } + } + } + + pub fn insert_temp_tree_with_prefix(self, prefix: Vec, merk: Merk, transaction: Option<&OptimisticTransactionDBTransaction>) -> Option> { + match transaction{ + None => None, + Some(_) => { + self.temp_subtrees.borrow_mut().insert(prefix, merk) + } + } + } + pub fn get( &self, path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result, Error> { + let merk; match transaction { - None => self.get_subtree_without_transaction(path), - Some(_) => self.get_subtree_with_transaction(path), + None => { + merk = self.get_subtree_without_transaction(path)?; + }, + Some(_) => { + let prefix = &GroveDb::compress_subtree_key(path, None); + if self.temp_subtrees.borrow().contains_key(prefix) { + // get the merk out + merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); + } else { + // merk is not in the hash map get it without transaction + merk = self.get_subtree_without_transaction(path)?; + } + } } + Ok(merk) } - fn get_subtree_without_transaction( + pub fn get_subtree_without_transaction( &self, path: &[&[u8]], ) -> Result, Error> { @@ -71,23 +103,24 @@ impl Subtrees<'_> { } } - fn get_subtree_with_transaction( - &self, - path: &[&[u8]], - ) -> Result, Error> { - let subtree_prefix = GroveDb::compress_subtree_key(path, None); - if let Some(merk) = self.temp_subtrees.get(&subtree_prefix) { - Ok(merk.clone()) - } else { - dbg!("Getting subtree without transaction"); - // if the subtree doesn't exist in temp_subtrees, - // check if it was created before the transaction was started - let merk = self - .get_subtree_without_transaction(path) - .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; - Ok(merk) - } - } + // pub fn get_subtree_with_transaction( + // &self, + // path: &[&[u8]], + // ) -> Result<&Merk, Error> { + // let subtree_prefix = GroveDb::compress_subtree_key(path, None); + // if let Some(merk) = self.temp_subtrees.borrow().get(&subtree_prefix) { + // Ok(merk) + // } else { + // Err(Error::InvalidPath("no subtree found under that path")) + // // dbg!("Getting subtree without transaction"); + // // // if the subtree doesn't exist in temp_subtrees, + // // // check if it was created before the transaction was started + // // let merk = self + // // .get_subtree_without_transaction(path) + // // .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + // // Ok(merk) + // } + // } fn get_subtree_with_key_info( &self, From 596bcd7065e9fcdc2db39d068cc30802efd005ea Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 12:58:19 +0700 Subject: [PATCH 18/31] fixed transaction insert tests --- grovedb/src/lib.rs | 9 ++++-- grovedb/src/operations/get.rs | 48 +++++++++++++++++--------------- grovedb/src/operations/insert.rs | 1 + grovedb/src/subtrees.rs | 4 +-- 4 files changed, 34 insertions(+), 28 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 28e7137a6..e21e73df2 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -218,7 +218,7 @@ impl GroveDb { }; Ok(GroveDb::new( - Self::build_root_tree(subtrees_view, &root_leaf_keys, None), + Self::build_root_tree(&subtrees_view, &root_leaf_keys, None), root_leaf_keys, subtrees, meta_storage, @@ -299,7 +299,7 @@ impl GroveDb { fn build_root_tree( // subtrees: &HashMap, Merk>, - subtrees: Subtrees, + subtrees: &Subtrees, root_leaf_keys: &HashMap, usize>, transaction: Option<&OptimisticTransactionDBTransaction>, ) -> MerkleTree { @@ -309,6 +309,7 @@ impl GroveDb { .get(&[subtree_path.as_slice()], transaction) .expect("`root_leaf_keys` must be in sync with `subtrees`"); leaf_hashes[*root_leaf_idx] = subtree_merk.root_hash(); + subtrees.insert_temp_tree(&[subtree_path.as_slice()], subtree_merk, transaction); } MerkleTree::::from_leaves(&leaf_hashes) } @@ -363,17 +364,19 @@ impl GroveDb { let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; let mut upper_tree = subtrees.get(parent_path, transaction)?; element.insert(&mut upper_tree, key.to_vec(), transaction); + self.get_subtrees().insert_temp_tree(parent_path, upper_tree, transaction); path = parent_path; } // root leaf nodes if path.len() == 1 { + let root_leaf_keys = match transaction { None => &self.root_leaf_keys, Some(_) => &self.temp_root_leaf_keys, }; - let root_tree = Self::build_root_tree(subtrees, root_leaf_keys, transaction); + let root_tree = GroveDb::build_root_tree(&subtrees, root_leaf_keys, transaction); match transaction { None => self.root_tree = root_tree, Some(_) => self.temp_root_tree = root_tree, diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index a5a7adaa7..45d20f852 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -72,32 +72,34 @@ impl GroveDb { key: &[u8], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let merk; - match transaction { - None => { - merk = self.get_subtrees().get_subtree_without_transaction(path)?; - }, - Some(_) => { - let prefix = &Self::compress_subtree_key(path, None); - if self.temp_subtrees.borrow().contains_key(prefix) { - // get the merk out - merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); - } else { - // merk is not in the hash map get it without transaction - merk = self.get_subtrees().get_subtree_without_transaction(path)?; - } - } - } + let merk = self.get_subtrees().get(path, transaction)?; + // let merk; + // match transaction { + // None => { + // merk = self.get_subtrees().get_subtree_without_transaction(path)?; + // }, + // Some(_) => { + // let prefix = &Self::compress_subtree_key(path, None); + // if self.temp_subtrees.borrow().contains_key(prefix) { + // // get the merk out + // merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); + // } else { + // // merk is not in the hash map get it without transaction + // merk = self.get_subtrees().get_subtree_without_transaction(path)?; + // } + // } + // } let elem = Element::get(&merk, key); - match transaction { - None => {}, - Some(_) => { - let prefix = Self::compress_subtree_key(path, None); - self.temp_subtrees.borrow_mut().insert(prefix, merk); - } - }; + self.get_subtrees().insert_temp_tree(path, merk, transaction); + // match transaction { + // None => {}, + // Some(_) => { + // let prefix = Self::compress_subtree_key(path, None); + // self.temp_subtrees.borrow_mut().insert(prefix, merk); + // } + // }; elem } diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index 40021b411..881b44a9b 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -143,6 +143,7 @@ impl GroveDb { // need to mark key as taken in the upper tree element.insert(&mut merk, key, transaction)?; + self.get_subtrees().insert_temp_tree(path, merk, transaction); self.propagate_changes(path, transaction)?; diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs index d224134a8..92fbed4b2 100644 --- a/grovedb/src/subtrees.rs +++ b/grovedb/src/subtrees.rs @@ -17,7 +17,7 @@ pub struct Subtrees<'a> { } impl Subtrees<'_> { - pub fn insert_temp_tree(self, path: &[&[u8]], merk: Merk, transaction: Option<&OptimisticTransactionDBTransaction>) -> Option> { + pub fn insert_temp_tree(&self, path: &[&[u8]], merk: Merk, transaction: Option<&OptimisticTransactionDBTransaction>) -> Option> { match transaction{ None => None, Some(_) => { @@ -27,7 +27,7 @@ impl Subtrees<'_> { } } - pub fn insert_temp_tree_with_prefix(self, prefix: Vec, merk: Merk, transaction: Option<&OptimisticTransactionDBTransaction>) -> Option> { + pub fn insert_temp_tree_with_prefix(&self, prefix: Vec, merk: Merk, transaction: Option<&OptimisticTransactionDBTransaction>) -> Option> { match transaction{ None => None, Some(_) => { From 21e99fd248b60a8b7d65c31940ffd6dcd6a49302 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 14:13:20 +0700 Subject: [PATCH 19/31] fixed rollback and get subtree tests --- grovedb/src/lib.rs | 3 ++- grovedb/src/operations/insert.rs | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index e21e73df2..dc591b8d7 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -360,6 +360,7 @@ impl GroveDb { // non root leaf node let subtree = subtrees.get(path, transaction)?; let element = Element::Tree(subtree.root_hash()); + self.get_subtrees().insert_temp_tree(path, subtree, transaction); let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; let mut upper_tree = subtrees.get(parent_path, transaction)?; @@ -552,7 +553,7 @@ impl GroveDb { // Cloning all the trees to maintain to rollback transactional changes self.temp_root_tree = self.root_tree.clone(); self.temp_root_leaf_keys = self.root_leaf_keys.clone(); - // self.temp_subtrees = self.subtrees.clone(); + self.temp_subtrees = RefCell::new(HashMap::new()); Ok(db_transaction .rollback() diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index 881b44a9b..b76a6e041 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -105,6 +105,11 @@ impl GroveDb { } // Add subtree to another subtree. + // We want to add a new empty merk to another merk at a key + // first make sure other merk exist + // if it exists, then create merk to be inserted, and get root hash + // we only care about root hash of merk to be inserted + // fn add_non_root_subtree<'a: 'b, 'b>( &'a mut self, path: &[&[u8]], @@ -118,12 +123,14 @@ impl GroveDb { } // First, check if a subtree exists to create a new subtree under it - self.get_subtrees().get(path, transaction)?; + let parent = self.get_subtrees().get(path, transaction)?; + self.get_subtrees().insert_temp_tree(path, parent, transaction); let (subtree_prefix, subtree_merk) = create_merk_with_prefix(self.db.clone(), path, &key)?; // Set tree value as a a subtree root hash let element = Element::Tree(subtree_merk.root_hash()); + self.get_subtrees().insert_temp_tree_with_prefix(subtree_prefix, subtree_merk, transaction); // Save subtrees, to be removed // TODO: Remove this From 51787cc8031bf123bb177e6e0ed54fc716a41e7b Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 14:38:54 +0700 Subject: [PATCH 20/31] fixed all tests except proofs --- grovedb/src/lib.rs | 3 +++ grovedb/src/operations/delete.rs | 5 ++++- grovedb/src/tests.rs | 14 ++++++++------ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index dc591b8d7..1a6b7007c 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -319,11 +319,13 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { + // Get the correct subtrees instance // let subtrees = match transaction { // None => &self.subtrees, // Some(_) => &self.temp_subtrees, // }; todo!() + // Get the merk at the current path // let merk = self.get_subtrees() // .get(path, transaction) // .ok_or(Error::InvalidPath("no subtree found under that path"))?; @@ -331,6 +333,7 @@ impl GroveDb { // // .get_subtrees() // // .get(path, transaction) // // .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + // Return the raw iter of the merk // Ok(Element::iterator(merk.raw_iter())) } diff --git a/grovedb/src/operations/delete.rs b/grovedb/src/operations/delete.rs index c8e4e0950..39d6405dd 100644 --- a/grovedb/src/operations/delete.rs +++ b/grovedb/src/operations/delete.rs @@ -108,7 +108,10 @@ impl GroveDb { while let Some(q) = queue.pop() { // TODO: eventually we need to do something about this nested slices let q_ref: Vec<&[u8]> = q.iter().map(|x| x.as_slice()).collect(); - let mut iter = self.elements_iterator(&q_ref, transaction)?; + // Get the correct subtree with q_ref as path + let merk = self.get_subtrees().get(&q_ref, transaction)?; + let mut iter = Element::iterator(merk.raw_iter()); + // let mut iter = self.elements_iterator(&q_ref, transaction)?; while let Some((key, value)) = iter.next()? { match value { Element::Tree(_) => { diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 3389c8aa9..0805d9477 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -262,8 +262,8 @@ fn test_tree_structure_is_presistent() { fn test_root_tree_leafs_are_noted() { let db = make_grovedb(); let mut hm = HashMap::new(); - hm.insert(GroveDb::compress_subtree_key(&[TEST_LEAF], None), 0); - hm.insert(GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF], None), 1); + hm.insert(TEST_LEAF.to_vec(), 0); + hm.insert(ANOTHER_TEST_LEAF.to_vec(), 1); assert_eq!(db.root_leaf_keys, hm); assert_eq!(db.root_tree.leaves_len(), 2); } @@ -1020,9 +1020,11 @@ fn test_subtree_pairs_iterator() { .expect("succesful value insert"); // Iterate over subtree1 to see if keys of other subtrees messed up - let mut iter = db - .elements_iterator(&[TEST_LEAF, b"subtree1"], None) - .expect("cannot create iterator"); + // let mut iter = db + // .elements_iterator(&[TEST_LEAF, b"subtree1"], None) + // .expect("cannot create iterator"); + let merk = db.get_subtrees().get(&[TEST_LEAF, b"subtree1"], None).unwrap(); + let mut iter = Element::iterator(merk.raw_iter()); assert_eq!(iter.next().unwrap(), Some((b"key1".to_vec(), element))); assert_eq!(iter.next().unwrap(), Some((b"key2".to_vec(), element2))); let subtree_element = iter.next().unwrap().unwrap(); @@ -1220,7 +1222,7 @@ fn test_subtree_deletion() { db.get(&[TEST_LEAF, b"key1", b"key2"], b"key3", None), Err(Error::InvalidPath(_)) )); - assert_eq!(db.subtrees.len(), 3); // TEST_LEAF, ANOTHER_TEST_LEAF TEST_LEAF.key4 stay + // assert_eq!(db.subtrees.len(), 3); // TEST_LEAF, ANOTHER_TEST_LEAF TEST_LEAF.key4 stay assert!(db.get(&[TEST_LEAF], b"key4", None).is_ok()); assert_ne!(root_hash, db.root_tree.root().unwrap()); } From f10515e42e475397d2b042e08b8f317292e9fac4 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 15:05:00 +0700 Subject: [PATCH 21/31] fmt --- grovedb/src/operations/get.rs | 48 ++++++++++++++++++----------------- grovedb/src/subtree.rs | 2 ++ 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 45d20f852..5ff9ae8f0 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -200,33 +200,35 @@ impl GroveDb { // let merk = subtrees // .get(&Self::compress_subtree_key(path, None)) // .ok_or(Error::InvalidPath("no subtree found under that path"))?; - let merk; - match transaction { - None => { - merk = self.get_subtrees().get_subtree_without_transaction(path)?; - }, - Some(_) => { - let prefix = &Self::compress_subtree_key(path, None); - if self.temp_subtrees.borrow().contains_key(prefix) { - // get the merk out - merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); - } else { - // merk is not in the hash map get it without transaction - merk = self.get_subtrees().get_subtree_without_transaction(path)?; - } - } - } + // let merk; + let merk = subtrees.get(path, transaction)?; + // match transaction { + // None => { + // merk = self.get_subtrees().get_subtree_without_transaction(path)?; + // }, + // Some(_) => { + // let prefix = &Self::compress_subtree_key(path, None); + // if self.temp_subtrees.borrow().contains_key(prefix) { + // // get the merk out + // merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); + // } else { + // // merk is not in the hash map get it without transaction + // merk = self.get_subtrees().get_subtree_without_transaction(path)?; + // } + // } + // } let elem = Element::get_path_query(&merk, path_query, Some(&subtrees)); - match transaction { - None => {}, - Some(_) => { - let prefix = Self::compress_subtree_key(path, None); - self.temp_subtrees.borrow_mut().insert(prefix, merk); - } - }; + subtrees.insert_temp_tree(path, merk, transaction); + // match transaction { + // None => {}, + // Some(_) => { + // let prefix = Self::compress_subtree_key(path, None); + // self.temp_subtrees.borrow_mut().insert(prefix, merk); + // } + // }; elem } diff --git a/grovedb/src/subtree.rs b/grovedb/src/subtree.rs index c23905677..af58f9282 100644 --- a/grovedb/src/subtree.rs +++ b/grovedb/src/subtree.rs @@ -143,6 +143,7 @@ impl Element { let (mut sub_elements, skipped) = Element::get_sized_query(&inner_merk, &inner_query)?; dbg!(sub_elements.len()); + subtrees.insert_temp_tree(path_vec.as_slice(), inner_merk, None); if let Some(limit) = limit { *limit -= sub_elements.len() as u16; } @@ -163,6 +164,7 @@ impl Element { } else if offset.is_some() { *offset = Some(offset.unwrap() - 1); } + subtrees.insert_temp_tree(path_vec.as_slice(), inner_merk, None); } } } From faa165f6b5c03ab553e7a904f8c0c1ab2bcf0f88 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 15:05:33 +0700 Subject: [PATCH 22/31] fmt --- grovedb/src/lib.rs | 36 +++++++++-------- grovedb/src/operations/delete.rs | 32 +++++++++------ grovedb/src/operations/get.rs | 32 ++++++++------- grovedb/src/operations/insert.rs | 20 ++++++--- grovedb/src/subtree.rs | 3 +- grovedb/src/subtrees.rs | 36 +++++++++++------ grovedb/src/tests.rs | 8 +++- merk/src/proofs/query/mod.rs | 69 +++++++++++++++++--------------- 8 files changed, 138 insertions(+), 98 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 1a6b7007c..b0b92f938 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -5,7 +5,7 @@ mod subtrees; mod tests; mod transaction; -use std::{collections::HashMap, path::Path, rc::Rc, cell::RefCell}; +use std::{cell::RefCell, collections::HashMap, path::Path, rc::Rc}; pub use merk::proofs::{query::QueryItem, Query}; use merk::{self, Merk}; @@ -210,7 +210,8 @@ impl GroveDb { HashMap::new() }; - let temp_subtrees: RefCell, Merk>> = RefCell::new(HashMap::new()); + let temp_subtrees: RefCell, Merk>> = + RefCell::new(HashMap::new()); let subtrees_view = Subtrees { root_leaf_keys: &root_leaf_keys, temp_subtrees: &temp_subtrees, @@ -260,21 +261,21 @@ impl GroveDb { // // let prefixes: Vec> = subtrees.keys().cloned().collect(); // - // // TODO: make StorageOrTransaction which will has the access to either storage - // // or transaction + // // TODO: make StorageOrTransaction which will has the access to either + // storage // or transaction // match db_transaction { // None => { // self.meta_storage.put_meta( // SUBTREES_SERIALIZED_KEY, // &bincode::serialize(&prefixes).map_err(|_| { - // Error::CorruptedData(String::from("unable to serialize prefixes")) - // })?, + // Error::CorruptedData(String::from("unable to serialize + // prefixes")) })?, // )?; // self.meta_storage.put_meta( // ROOT_LEAFS_SERIALIZED_KEY, // &bincode::serialize(&self.temp_root_leaf_keys).map_err(|_| { - // Error::CorruptedData(String::from("unable to serialize root leafs")) - // })?, + // Error::CorruptedData(String::from("unable to serialize + // root leafs")) })?, // )?; // } // Some(tx) => { @@ -282,14 +283,14 @@ impl GroveDb { // transaction.put_meta( // SUBTREES_SERIALIZED_KEY, // &bincode::serialize(&prefixes).map_err(|_| { - // Error::CorruptedData(String::from("unable to serialize prefixes")) - // })?, + // Error::CorruptedData(String::from("unable to serialize + // prefixes")) })?, // )?; // transaction.put_meta( // ROOT_LEAFS_SERIALIZED_KEY, // &bincode::serialize(&self.root_leaf_keys).map_err(|_| { - // Error::CorruptedData(String::from("unable to serialize root leafs")) - // })?, + // Error::CorruptedData(String::from("unable to serialize + // root leafs")) })?, // )?; // } // } @@ -332,8 +333,8 @@ impl GroveDb { // // let merk = self // // .get_subtrees() // // .get(path, transaction) - // // .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; - // Return the raw iter of the merk + // // .map_err(|_| Error::InvalidPath("no subtree found under that + // path"))?; Return the raw iter of the merk // Ok(Element::iterator(merk.raw_iter())) } @@ -363,19 +364,20 @@ impl GroveDb { // non root leaf node let subtree = subtrees.get(path, transaction)?; let element = Element::Tree(subtree.root_hash()); - self.get_subtrees().insert_temp_tree(path, subtree, transaction); + self.get_subtrees() + .insert_temp_tree(path, subtree, transaction); let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; let mut upper_tree = subtrees.get(parent_path, transaction)?; element.insert(&mut upper_tree, key.to_vec(), transaction); - self.get_subtrees().insert_temp_tree(parent_path, upper_tree, transaction); + self.get_subtrees() + .insert_temp_tree(parent_path, upper_tree, transaction); path = parent_path; } // root leaf nodes if path.len() == 1 { - let root_leaf_keys = match transaction { None => &self.root_leaf_keys, Some(_) => &self.temp_root_leaf_keys, diff --git a/grovedb/src/operations/delete.rs b/grovedb/src/operations/delete.rs index 39d6405dd..0025c2c63 100644 --- a/grovedb/src/operations/delete.rs +++ b/grovedb/src/operations/delete.rs @@ -38,10 +38,12 @@ impl GroveDb { // let prefix = &Self::compress_subtree_key(path, None); // if self.temp_subtrees.borrow().contains_key(prefix) { // // get the merk out - // merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); - // } else { + // merk = + // self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the + // hashmap"); } else { // // merk is not in the hash map get it without transaction - // merk = self.get_subtrees().get_subtree_without_transaction(path)?; + // merk = + // self.get_subtrees().get_subtree_without_transaction(path)?; // } // } // } @@ -57,7 +59,8 @@ impl GroveDb { Element::delete(&mut merk, key.clone(), transaction)?; // after deletion, if there is a transaction, add the merk back into the hashmap - self.get_subtrees().insert_temp_tree(path, merk, transaction); + self.get_subtrees() + .insert_temp_tree(path, merk, transaction); } if let Element::Tree(_) = element { @@ -77,14 +80,19 @@ impl GroveDb { let prefix = Self::compress_subtree_key(&subtree_path_ref, None); // Here we seem to get the subtree, then call clear on it // Seems to only care about subtrees that are non transactional - let subtree = self.get_subtrees().get_subtree_without_transaction(subtree_path_ref.as_slice()); - if subtree.is_ok(){ - subtree.expect("confirmed it's valid").clear(transaction).map_err(|e| { - Error::CorruptedData(format!( - "unable to cleanup tree from storage: {}", - e - )) - })?; + let subtree = self + .get_subtrees() + .get_subtree_without_transaction(subtree_path_ref.as_slice()); + if subtree.is_ok() { + subtree + .expect("confirmed it's valid") + .clear(transaction) + .map_err(|e| { + Error::CorruptedData(format!( + "unable to cleanup tree from storage: {}", + e + )) + })?; } } } diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 5ff9ae8f0..05a32261f 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -6,8 +6,10 @@ use std::{ }; use merk::Merk; -use storage::RawIterator; -use storage::rocksdb_storage::{OptimisticTransactionDBTransaction, PrefixedRocksDbStorage}; +use storage::{ + rocksdb_storage::{OptimisticTransactionDBTransaction, PrefixedRocksDbStorage}, + RawIterator, +}; use crate::{Element, Error, GroveDb, PathQuery, Subtrees}; @@ -82,17 +84,19 @@ impl GroveDb { // let prefix = &Self::compress_subtree_key(path, None); // if self.temp_subtrees.borrow().contains_key(prefix) { // // get the merk out - // merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); - // } else { + // merk = + // self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the + // hashmap"); } else { // // merk is not in the hash map get it without transaction - // merk = self.get_subtrees().get_subtree_without_transaction(path)?; - // } + // merk = + // self.get_subtrees().get_subtree_without_transaction(path)?; } // } // } let elem = Element::get(&merk, key); - self.get_subtrees().insert_temp_tree(path, merk, transaction); + self.get_subtrees() + .insert_temp_tree(path, merk, transaction); // match transaction { // None => {}, // Some(_) => { @@ -163,14 +167,14 @@ impl GroveDb { Element::Item(item) => { dbg!("item"); Ok(item) - }, + } Element::Tree(item) => { dbg!("tree"); // Err(Error::InvalidQuery( // "path_queries can only refer to items and references", // )) Ok(item.to_vec()) - }, + } }) .collect::>, Error>>()?; Ok((results, skipped)) @@ -210,17 +214,17 @@ impl GroveDb { // let prefix = &Self::compress_subtree_key(path, None); // if self.temp_subtrees.borrow().contains_key(prefix) { // // get the merk out - // merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); - // } else { + // merk = + // self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the + // hashmap"); } else { // // merk is not in the hash map get it without transaction - // merk = self.get_subtrees().get_subtree_without_transaction(path)?; - // } + // merk = + // self.get_subtrees().get_subtree_without_transaction(path)?; } // } // } let elem = Element::get_path_query(&merk, path_query, Some(&subtrees)); - subtrees.insert_temp_tree(path, merk, transaction); // match transaction { // None => {}, diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index b76a6e041..54aa5e4c4 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -53,9 +53,13 @@ impl GroveDb { // let mut merk = subtrees // .get_mut(&Self::compress_subtree_key(path, None)) // .ok_or(Error::InvalidPath("no subtree found under that path"))?; - let mut merk = self.get_subtrees().get(path, transaction).map_err(|_| Error::InvalidPath("no subtree found under that path"))?; + let mut merk = self + .get_subtrees() + .get(path, transaction) + .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; element.insert(&mut merk, key, transaction)?; - self.get_subtrees().insert_temp_tree(path, merk, transaction); + self.get_subtrees() + .insert_temp_tree(path, merk, transaction); self.propagate_changes(path, transaction)?; } } @@ -83,7 +87,8 @@ impl GroveDb { // compressed path let (subtree_prefix, subtree_merk) = create_merk_with_prefix(self.db.clone(), &[], key)?; // subtrees.insert(subtree_prefix.clone(), subtree_merk); - self.get_subtrees().insert_temp_tree_with_prefix(subtree_prefix, subtree_merk, transaction); + self.get_subtrees() + .insert_temp_tree_with_prefix(subtree_prefix, subtree_merk, transaction); let root_leaf_keys = match transaction { None => &mut self.root_leaf_keys, @@ -124,13 +129,15 @@ impl GroveDb { // First, check if a subtree exists to create a new subtree under it let parent = self.get_subtrees().get(path, transaction)?; - self.get_subtrees().insert_temp_tree(path, parent, transaction); + self.get_subtrees() + .insert_temp_tree(path, parent, transaction); let (subtree_prefix, subtree_merk) = create_merk_with_prefix(self.db.clone(), path, &key)?; // Set tree value as a a subtree root hash let element = Element::Tree(subtree_merk.root_hash()); - self.get_subtrees().insert_temp_tree_with_prefix(subtree_prefix, subtree_merk, transaction); + self.get_subtrees() + .insert_temp_tree_with_prefix(subtree_prefix, subtree_merk, transaction); // Save subtrees, to be removed // TODO: Remove this @@ -150,7 +157,8 @@ impl GroveDb { // need to mark key as taken in the upper tree element.insert(&mut merk, key, transaction)?; - self.get_subtrees().insert_temp_tree(path, merk, transaction); + self.get_subtrees() + .insert_temp_tree(path, merk, transaction); self.propagate_changes(path, transaction)?; diff --git a/grovedb/src/subtree.rs b/grovedb/src/subtree.rs index af58f9282..53abb7806 100644 --- a/grovedb/src/subtree.rs +++ b/grovedb/src/subtree.rs @@ -1,8 +1,7 @@ //! Module for subtrees handling. //! Subtrees handling is isolated so basically this module is about adapting //! Merk API to GroveDB needs. -use std::collections::HashMap; -use std::ops::Sub; +use std::{collections::HashMap, ops::Sub}; use merk::{ proofs::{query::QueryItem, Query}, diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs index 92fbed4b2..982a8d1e4 100644 --- a/grovedb/src/subtrees.rs +++ b/grovedb/src/subtrees.rs @@ -1,5 +1,5 @@ //! Module for retrieving subtrees -use std::{collections::HashMap, rc::Rc, cell::RefCell}; +use std::{cell::RefCell, collections::HashMap, rc::Rc}; use merk::Merk; use storage::{ @@ -17,8 +17,13 @@ pub struct Subtrees<'a> { } impl Subtrees<'_> { - pub fn insert_temp_tree(&self, path: &[&[u8]], merk: Merk, transaction: Option<&OptimisticTransactionDBTransaction>) -> Option> { - match transaction{ + pub fn insert_temp_tree( + &self, + path: &[&[u8]], + merk: Merk, + transaction: Option<&OptimisticTransactionDBTransaction>, + ) -> Option> { + match transaction { None => None, Some(_) => { let prefix = GroveDb::compress_subtree_key(path, None); @@ -27,12 +32,15 @@ impl Subtrees<'_> { } } - pub fn insert_temp_tree_with_prefix(&self, prefix: Vec, merk: Merk, transaction: Option<&OptimisticTransactionDBTransaction>) -> Option> { - match transaction{ + pub fn insert_temp_tree_with_prefix( + &self, + prefix: Vec, + merk: Merk, + transaction: Option<&OptimisticTransactionDBTransaction>, + ) -> Option> { + match transaction { None => None, - Some(_) => { - self.temp_subtrees.borrow_mut().insert(prefix, merk) - } + Some(_) => self.temp_subtrees.borrow_mut().insert(prefix, merk), } } @@ -45,12 +53,16 @@ impl Subtrees<'_> { match transaction { None => { merk = self.get_subtree_without_transaction(path)?; - }, + } Some(_) => { let prefix = &GroveDb::compress_subtree_key(path, None); if self.temp_subtrees.borrow().contains_key(prefix) { // get the merk out - merk = self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the hashmap"); + merk = self + .temp_subtrees + .borrow_mut() + .remove(prefix) + .expect("confirmed it's in the hashmap"); } else { // merk is not in the hash map get it without transaction merk = self.get_subtree_without_transaction(path)?; @@ -117,8 +129,8 @@ impl Subtrees<'_> { // // // check if it was created before the transaction was started // // let merk = self // // .get_subtree_without_transaction(path) - // // .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; - // // Ok(merk) + // // .map_err(|_| Error::InvalidPath("no subtree found under that + // path"))?; // Ok(merk) // } // } diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 0805d9477..f6a8007b2 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1023,7 +1023,10 @@ fn test_subtree_pairs_iterator() { // let mut iter = db // .elements_iterator(&[TEST_LEAF, b"subtree1"], None) // .expect("cannot create iterator"); - let merk = db.get_subtrees().get(&[TEST_LEAF, b"subtree1"], None).unwrap(); + let merk = db + .get_subtrees() + .get(&[TEST_LEAF, b"subtree1"], None) + .unwrap(); let mut iter = Element::iterator(merk.raw_iter()); assert_eq!(iter.next().unwrap(), Some((b"key1".to_vec(), element))); assert_eq!(iter.next().unwrap(), Some((b"key2".to_vec(), element2))); @@ -1222,7 +1225,8 @@ fn test_subtree_deletion() { db.get(&[TEST_LEAF, b"key1", b"key2"], b"key3", None), Err(Error::InvalidPath(_)) )); - // assert_eq!(db.subtrees.len(), 3); // TEST_LEAF, ANOTHER_TEST_LEAF TEST_LEAF.key4 stay + // assert_eq!(db.subtrees.len(), 3); // TEST_LEAF, ANOTHER_TEST_LEAF + // TEST_LEAF.key4 stay assert!(db.get(&[TEST_LEAF], b"key4", None).is_ok()); assert_ne!(root_hash, db.root_tree.root().unwrap()); } diff --git a/merk/src/proofs/query/mod.rs b/merk/src/proofs/query/mod.rs index d77aed129..223b5b5ad 100644 --- a/merk/src/proofs/query/mod.rs +++ b/merk/src/proofs/query/mod.rs @@ -665,46 +665,49 @@ where }; // if left_to_right { - let (mut proof, left_absence, new_limit, new_offset) = - self.create_child_proof(true, left_items, limit, offset, left_to_right)?; - let (mut right_proof, right_absence, new_limit, new_offset) = - self.create_child_proof(false, right_items, new_limit, new_offset, left_to_right)?; - - let (has_left, has_right) = (!proof.is_empty(), !right_proof.is_empty()); - - proof.push_back(match search { - Ok(_) => Op::Push(self.to_kv_node()), - Err(_) => { - if left_absence.1 || right_absence.0 { - Op::Push(self.to_kv_node()) - } else { - Op::Push(self.to_kvhash_node()) - } + let (mut proof, left_absence, new_limit, new_offset) = + self.create_child_proof(true, left_items, limit, offset, left_to_right)?; + let (mut right_proof, right_absence, new_limit, new_offset) = + self.create_child_proof(false, right_items, new_limit, new_offset, left_to_right)?; + + let (has_left, has_right) = (!proof.is_empty(), !right_proof.is_empty()); + + proof.push_back(match search { + Ok(_) => Op::Push(self.to_kv_node()), + Err(_) => { + if left_absence.1 || right_absence.0 { + Op::Push(self.to_kv_node()) + } else { + Op::Push(self.to_kvhash_node()) } - }); - - if has_left { - proof.push_back(Op::Parent); } + }); - if has_right { - proof.append(&mut right_proof); - proof.push_back(Op::Child); - } + if has_left { + proof.push_back(Op::Parent); + } + + if has_right { + proof.append(&mut right_proof); + proof.push_back(Op::Child); + } - Ok(( - proof, - (left_absence.0, right_absence.1), - new_limit, - new_offset, - )) + Ok(( + proof, + (left_absence.0, right_absence.1), + new_limit, + new_offset, + )) // } else { // let (mut proof, left_absence, new_limit, new_offset) = - // self.create_child_proof(true, left_items, limit, offset, left_to_right)?; - // let (mut right_proof, right_absence, new_limit, new_offset) = - // self.create_child_proof(false, right_items, new_limit, new_offset, left_to_right)?; + // self.create_child_proof(true, left_items, limit, offset, + // left_to_right)?; let (mut right_proof, right_absence, + // new_limit, new_offset) = self. + // create_child_proof(false, right_items, new_limit, new_offset, + // left_to_right)?; // - // let (has_left, has_right) = (!proof.is_empty(), !right_proof.is_empty()); + // let (has_left, has_right) = (!proof.is_empty(), + // !right_proof.is_empty()); // // proof.push_back(match search { // Ok(_) => Op::Push(self.to_kv_node()), From 201be5769134ce680ccce7573227954c115d88a6 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 15:23:02 +0700 Subject: [PATCH 23/31] delete dbg's --- grovedb/src/operations/get.rs | 1 - grovedb/src/operations/insert.rs | 2 -- grovedb/src/subtree.rs | 3 --- grovedb/src/subtrees.rs | 1 - grovedb/src/tests.rs | 3 --- 5 files changed, 10 deletions(-) diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 33cbfa477..b5db5aa46 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -156,7 +156,6 @@ impl GroveDb { .into_iter() .map(|element| match element { Element::Reference(reference_path) => { - dbg!("ref"); let maybe_item = self.follow_reference(reference_path, transaction)?; if let Element::Item(item) = maybe_item { Ok(item) diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index 54aa5e4c4..99fb810d8 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -103,9 +103,7 @@ impl GroveDb { if root_leaf_keys.get(&key.to_vec()).is_none() { root_leaf_keys.insert(key.to_vec(), root_tree.leaves_len()); } - dbg!("propagating after adding root"); self.propagate_changes(&[key], transaction)?; - dbg!("done propagating after adding root"); Ok(()) } diff --git a/grovedb/src/subtree.rs b/grovedb/src/subtree.rs index f3ce7ed0f..746fa28a9 100644 --- a/grovedb/src/subtree.rs +++ b/grovedb/src/subtree.rs @@ -88,7 +88,6 @@ impl Element { limit: &mut Option, offset: &mut Option, ) -> Result<(), Error> { - dbg!("basic push"); if offset.is_none() || offset.is_some() && offset.unwrap() == 0 { results.push(element); if limit.is_some() { @@ -113,7 +112,6 @@ impl Element { limit: &mut Option, offset: &mut Option, ) -> Result<(), Error> { - dbg!("path query push"); match element { Element::Tree(_) => { if subtrees_option.is_none() && subquery.is_none() { @@ -278,7 +276,6 @@ impl Element { } else { 0 }; - dbg!("results {}", results.len()); Ok((results, skipped)) } diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs index 982a8d1e4..27623598a 100644 --- a/grovedb/src/subtrees.rs +++ b/grovedb/src/subtrees.rs @@ -87,7 +87,6 @@ impl Subtrees<'_> { // if parent path is empty, we are dealing with root leaf node // we can confirm validity of a root leaf node by checking root_leaf_keys if parent_path.is_empty() { - // dbg!("parent path is empty, checking the root tree"); let root_key = path[0].to_vec(); return if self.root_leaf_keys.contains_key(&root_key) { Ok(subtree) diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 4d1e2bee6..0ca6d5aa9 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -807,8 +807,6 @@ fn transaction_insert_item_with_transaction_should_use_transaction() { let result_with_transaction = db .get(&[TEST_LEAF], &item_key, Some(&transaction)) .expect("Expected to work"); - dbg!("We got here"); - dbg!(&result_with_transaction); assert_eq!(result_with_transaction, Element::Item(b"ayy".to_vec())); // Test that commit works @@ -1707,7 +1705,6 @@ fn test_get_range_inclusive_query_with_non_unique_subquery() { fn test_get_range_inclusive_query_with_non_unique_subquery_on_references() { let mut db = make_grovedb(); populate_tree_by_reference_for_non_unique_range_subquery(&mut db); - dbg!("failed when populating"); let path = vec![TEST_LEAF, b"1"]; let mut query = Query::new(); From 380d090f5cc64d711603b639b1727bd8be891d7d Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 15:35:04 +0700 Subject: [PATCH 24/31] Removed proof tests --- grovedb/src/lib.rs | 151 +------ grovedb/src/operations/proof.rs | 510 +++++++++++----------- grovedb/src/tests.rs | 740 ++++++++++++++++---------------- 3 files changed, 648 insertions(+), 753 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index f39259e2b..15c3328af 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -108,7 +108,6 @@ pub struct Proof { pub struct GroveDb { root_tree: MerkleTree, root_leaf_keys: HashMap, usize>, - subtrees: HashMap, Merk>, meta_storage: PrefixedRocksDbStorage, db: Rc, // Locks the database for writes during the transaction @@ -123,14 +122,12 @@ impl GroveDb { pub fn new( root_tree: MerkleTree, root_leaf_keys: HashMap, usize>, - subtrees: HashMap, Merk>, meta_storage: PrefixedRocksDbStorage, db: Rc, ) -> Self { Self { root_tree, root_leaf_keys, - subtrees, meta_storage, db, temp_root_tree: MerkleTree::new(), @@ -151,21 +148,6 @@ impl GroveDb { ); let meta_storage = PrefixedRocksDbStorage::new(db.clone(), Vec::new())?; - let mut subtrees = HashMap::new(); - // TODO: owned `get` is not required for deserialization - if let Some(prefixes_serialized) = meta_storage.get_meta(SUBTREES_SERIALIZED_KEY)? { - let subtrees_prefixes: Vec> = bincode::deserialize(&prefixes_serialized) - .map_err(|_| { - Error::CorruptedData(String::from("unable to deserialize prefixes")) - })?; - for prefix in subtrees_prefixes { - let subtree_merk = - Merk::open(PrefixedRocksDbStorage::new(db.clone(), prefix.to_vec())?) - .map_err(|e| Error::CorruptedData(e.to_string()))?; - subtrees.insert(prefix.to_vec(), subtree_merk); - } - } - // TODO: owned `get` is not required for deserialization let root_leaf_keys: HashMap, usize> = if let Some(root_leaf_keys_serialized) = meta_storage.get_meta(ROOT_LEAFS_SERIALIZED_KEY)? @@ -188,7 +170,6 @@ impl GroveDb { Ok(GroveDb::new( Self::build_root_tree(&subtrees_view, &root_leaf_keys, None), root_leaf_keys, - subtrees, meta_storage, db, )) @@ -217,54 +198,6 @@ impl GroveDb { } } - // fn store_subtrees_keys_data( - // &self, - // db_transaction: Option<&OptimisticTransactionDBTransaction>, - // ) -> Result<(), Error> { - // let subtrees = match db_transaction { - // None => &self.subtrees, - // Some(_) => &self.temp_subtrees, - // }; - // - // let prefixes: Vec> = subtrees.keys().cloned().collect(); - // - // // TODO: make StorageOrTransaction which will has the access to either - // storage // or transaction - // match db_transaction { - // None => { - // self.meta_storage.put_meta( - // SUBTREES_SERIALIZED_KEY, - // &bincode::serialize(&prefixes).map_err(|_| { - // Error::CorruptedData(String::from("unable to serialize - // prefixes")) })?, - // )?; - // self.meta_storage.put_meta( - // ROOT_LEAFS_SERIALIZED_KEY, - // &bincode::serialize(&self.temp_root_leaf_keys).map_err(|_| { - // Error::CorruptedData(String::from("unable to serialize - // root leafs")) })?, - // )?; - // } - // Some(tx) => { - // let transaction = self.meta_storage.transaction(tx); - // transaction.put_meta( - // SUBTREES_SERIALIZED_KEY, - // &bincode::serialize(&prefixes).map_err(|_| { - // Error::CorruptedData(String::from("unable to serialize - // prefixes")) })?, - // )?; - // transaction.put_meta( - // ROOT_LEAFS_SERIALIZED_KEY, - // &bincode::serialize(&self.root_leaf_keys).map_err(|_| { - // Error::CorruptedData(String::from("unable to serialize - // root leafs")) })?, - // )?; - // } - // } - // - // Ok(()) - // } - fn build_root_tree( // subtrees: &HashMap, Merk>, subtrees: &Subtrees, @@ -282,28 +215,28 @@ impl GroveDb { MerkleTree::::from_leaves(&leaf_hashes) } - pub fn elements_iterator( - &self, - path: &[&[u8]], - transaction: Option<&OptimisticTransactionDBTransaction>, - ) -> Result { - // Get the correct subtrees instance - // let subtrees = match transaction { - // None => &self.subtrees, - // Some(_) => &self.temp_subtrees, - // }; - todo!() - // Get the merk at the current path - // let merk = self.get_subtrees() - // .get(path, transaction) - // .ok_or(Error::InvalidPath("no subtree found under that path"))?; - // // let merk = self - // // .get_subtrees() - // // .get(path, transaction) - // // .map_err(|_| Error::InvalidPath("no subtree found under that - // path"))?; Return the raw iter of the merk - // Ok(Element::iterator(merk.raw_iter())) - } + // pub fn elements_iterator( + // &self, + // path: &[&[u8]], + // transaction: Option<&OptimisticTransactionDBTransaction>, + // ) -> Result { + // // Get the correct subtrees instance + // // let subtrees = match transaction { + // // None => &self.subtrees, + // // Some(_) => &self.temp_subtrees, + // // }; + // todo!() + // // Get the merk at the current path + // // let merk = self.get_subtrees() + // // .get(path, transaction) + // // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + // // // let merk = self + // // // .get_subtrees() + // // // .get(path, transaction) + // // // .map_err(|_| Error::InvalidPath("no subtree found under that + // // path"))?; Return the raw iter of the merk + // // Ok(Element::iterator(merk.raw_iter())) + // } /// Method to propagate updated subtree root hashes up to GroveDB root fn propagate_changes<'a: 'b, 'b>( @@ -311,21 +244,10 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&'b ::DBTransaction<'b>>, ) -> Result<(), Error> { - // let subtrees = match transaction { - // None => &mut self.subtrees, - // Some(_) => &mut self.temp_subtrees, - // }; let subtrees = self.get_subtrees(); - // We are given a path to a subtree that was just changed - // We want to get it's root hash and then propagate changes up - // There is the case were the thing that was changed is the root leaf node - // if length == 1 then a root leaf node was changed - // if length > 1 then a tree further down was changed - // for root leaf nodes, we have to rebuild the root tree - // for others we just update them let mut path = path; - // let mut split_path = path.split_last(); + // Go up until only one element in path, which means a key of a root tree while path.len() > 1 { // non root leaf node @@ -356,31 +278,6 @@ impl GroveDb { } } - // while let Some((key, path_slice)) = split_path { - // if path_slice.is_empty() { - // // Hit the root tree - // match transaction { - // None => self.root_tree = Self::build_root_tree(subtrees, - // root_leaf_keys), Some(_) => { - // self.temp_root_tree = Self::build_root_tree(subtrees, - // root_leaf_keys) } - // }; - // break; - // } else { - // let compressed_path_upper_tree = - // Self::compress_subtree_key(path_slice, None); let - // compressed_path_subtree = Self::compress_subtree_key(path_slice, Some(key)); - // let subtree = subtrees - // .get(&compressed_path_subtree) - // .ok_or(Error::InvalidPath("no subtree found under that path"))?; - // let element = Element::Tree(subtree.root_hash()); - // let upper_tree = subtrees - // .get_mut(&compressed_path_upper_tree) - // .ok_or(Error::InvalidPath("no subtree found under that path"))?; - // element.insert(upper_tree, key.to_vec(), transaction)?; - // split_path = path_slice.split_last(); - // } - // } Ok(()) } @@ -483,7 +380,6 @@ impl GroveDb { // Cloning all the trees to maintain original state before the transaction self.temp_root_tree = self.root_tree.clone(); self.temp_root_leaf_keys = self.root_leaf_keys.clone(); - // self.temp_subtrees = self.subtrees.clone(); Ok(()) } @@ -506,7 +402,6 @@ impl GroveDb { // code can be reworked to account for that self.root_tree = self.temp_root_tree.clone(); self.root_leaf_keys = self.temp_root_leaf_keys.drain().collect(); - // self.subtrees = self.temp_subtrees.drain().collect(); self.cleanup_transactional_data(); diff --git a/grovedb/src/operations/proof.rs b/grovedb/src/operations/proof.rs index e7d36a085..4a945a8ac 100644 --- a/grovedb/src/operations/proof.rs +++ b/grovedb/src/operations/proof.rs @@ -1,255 +1,255 @@ -use std::collections::HashMap; - -use merk::proofs::query::Map; -use rs_merkle::{algorithms::Sha256, MerkleProof}; - -use crate::{Element, Error, GroveDb, PathQuery, Proof, Query}; - -impl GroveDb { - pub fn proof(&mut self, proof_queries: Vec) -> Result, Error> { - // To prove a path we need to return a proof for each node on the path including - // the root. With multiple paths, nodes can overlap i.e two or more paths can - // share the same nodes. We should only have one proof for each node, - // if a node forks into multiple relevant paths then we should create a - // combined proof for that node with all the relevant keys - let mut query_paths = Vec::new(); - let mut intermediate_proof_spec: HashMap, Query> = HashMap::new(); - let mut root_keys: Vec> = Vec::new(); - let mut proofs: HashMap, Vec> = HashMap::new(); - - // For each unique node including the root - // determine what keys would need to be included in the proof - for proof_query in proof_queries.iter() { - query_paths.push( - proof_query - .path - .iter() - .map(|x| x.to_vec()) - .collect::>(), - ); - - let mut split_path = proof_query.path.split_last(); - while let Some((key, path_slice)) = split_path { - if path_slice.is_empty() { - // We have gotten to the root node - let compressed_path = GroveDb::compress_subtree_key(&[], Some(key)); - root_keys.push(compressed_path); - } else { - let compressed_path = GroveDb::compress_subtree_key(path_slice, None); - if let Some(path_query) = intermediate_proof_spec.get_mut(&compressed_path) { - path_query.insert_key(key.to_vec()); - } else { - let mut path_query = Query::new(); - path_query.insert_key(key.to_vec()); - intermediate_proof_spec.insert(compressed_path, path_query); - } - } - split_path = path_slice.split_last(); - } - } - - // Construct the path proofs - for (path, query) in intermediate_proof_spec { - let proof = self.prove_item(&path, query)?; - proofs.insert(path, proof); - } - - // Construct the leaf proofs - for proof_query in proof_queries { - let path = proof_query.path; - - // If there is a subquery with a limit it's possible that we only need a reduced - // proof for this leaf. - let reduced_proof_query = proof_query; - - // First we must get elements - - if let Some(subquery_key) = reduced_proof_query.query.query.subquery_key.clone() { - self.get_path_queries_raw(&[&reduced_proof_query], None)?; - - let mut path_vec = path.to_vec(); - path_vec.push(subquery_key.as_slice()); - let compressed_path = GroveDb::compress_subtree_key(path_vec.as_slice(), None); - } - - // Now we must insert the final proof for the sub leaves - let compressed_path = GroveDb::compress_subtree_key(path, None); - let proof = self.prove_path_item(&compressed_path, reduced_proof_query)?; - proofs.insert(compressed_path, proof); - } - - // Construct the root proof - let mut root_index: Vec = Vec::new(); - for key in root_keys { - let index = self - .root_leaf_keys - .get(&key) - .ok_or(Error::InvalidPath("root key not found"))?; - root_index.push(*index); - } - let root_proof = self.root_tree.proof(&root_index).to_bytes(); - - let proof = Proof { - query_paths, - proofs, - root_proof, - root_leaf_keys: self.root_leaf_keys.clone(), - }; - - let seralized_proof = bincode::serialize(&proof) - .map_err(|_| Error::CorruptedData(String::from("unable to serialize proof")))?; - - Ok(seralized_proof) - } - - fn prove_path_item( - &self, - compressed_path: &Vec, - path_query: PathQuery, - ) -> Result, Error> { - let merk = self - .subtrees - .get(compressed_path) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; - - let sized_query = path_query.query; - - if sized_query.query.subquery.is_none() { - // then limit should be applied directly to the proof here - let proof_result = merk - .prove( - sized_query.query, - sized_query.limit, - sized_query.offset, - ) - .expect("should prove both inclusion and absence"); - Ok(proof_result) - } else { - let proof_result = merk - .prove(sized_query.query, None, None) - .expect("should prove both inclusion and absence"); - Ok(proof_result) - } - } - - fn prove_item(&self, path: &Vec, query: Query) -> Result, Error> { - let merk = self - .subtrees - .get(path) - .ok_or(Error::InvalidPath("no subtree found under that path"))?; - - // then limit should be applied directly to the proof here - let proof_result = merk - .prove(query, None, None) - .expect("should prove both inclusion and absence"); - Ok(proof_result) - } - - pub fn execute_proof(proof: Vec) -> Result<([u8; 32], HashMap, Map>), Error> { - // Deserialize the proof - let proof: Proof = bincode::deserialize(&proof) - .map_err(|_| Error::CorruptedData(String::from("unable to deserialize proof")))?; - - // Required to execute the root proof - let mut root_keys_index: Vec = Vec::new(); - let mut root_hashes: Vec<[u8; 32]> = Vec::new(); - - // Collects the result map for each query - let mut result_map: HashMap, Map> = HashMap::new(); - - for path in proof.query_paths { - let path = path.iter().map(|x| x.as_slice()).collect::>(); - // For each query path, get the result map after execution - // and store hash + index for later root proof execution - let root_key = &path[0]; - let (hash, proof_result_map) = GroveDb::execute_path(&path, &proof.proofs)?; - let compressed_root_key_path = GroveDb::compress_subtree_key(&[], Some(root_key)); - let compressed_query_path = GroveDb::compress_subtree_key(&path, None); - - let index = proof - .root_leaf_keys - .get(&compressed_root_key_path) - .ok_or(Error::InvalidPath("Bad path"))?; - if !root_keys_index.contains(index) { - root_keys_index.push(*index); - root_hashes.push(hash); - } - - result_map.insert(compressed_query_path, proof_result_map); - } - - let root_proof = match MerkleProof::::try_from(proof.root_proof) { - Ok(proof) => Ok(proof), - Err(_) => Err(Error::InvalidProof("Invalid proof element")), - }?; - - let root_hash = - match root_proof.root(&root_keys_index, &root_hashes, proof.root_leaf_keys.len()) { - Ok(hash) => Ok(hash), - Err(_) => Err(Error::InvalidProof("Invalid proof element")), - }?; - - Ok((root_hash, result_map)) - } - - // Given a query path and a set of proofs - // execute_path validates that the nodes represented by the paths - // are connected to one another i.e root hash of child node is in parent node - // at the correct key. - // If path is valid, it returns the root hash of topmost merk and result map of - // leaf merk. - fn execute_path( - path: &[&[u8]], - proofs: &HashMap, Vec>, - ) -> Result<([u8; 32], Map), Error> { - let compressed_path = GroveDb::compress_subtree_key(path, None); - let proof = proofs - .get(&compressed_path) - .ok_or(Error::InvalidPath("Bad path"))?; - - // Execute the leaf merk proof - let (mut last_root_hash, result_map) = match merk::execute_proof(&proof[..]) { - Ok(result) => Ok(result), - Err(_) => Err(Error::InvalidPath("Invalid proof element")), - }?; - - // Validate the path - let mut split_path = path.split_last(); - while let Some((key, path_slice)) = split_path { - if !path_slice.is_empty() { - let compressed_path = GroveDb::compress_subtree_key(path_slice, None); - let proof = proofs - .get(&compressed_path) - .ok_or(Error::InvalidPath("Bad path"))?; - - let proof_result = match merk::execute_proof(&proof[..]) { - Ok(result) => Ok(result), - Err(_) => Err(Error::InvalidPath("Invalid proof element")), - }?; - - let result_map = proof_result.1; - // TODO: Handle the error better here - let elem: Element = - bincode::deserialize(result_map.get(key).unwrap().unwrap()).unwrap(); - let merk_root_hash = match elem { - Element::Tree(hash) => Ok(hash), - _ => Err(Error::InvalidProof( - "Intermediate proofs should be for trees", - )), - }?; - - if merk_root_hash != last_root_hash { - return Err(Error::InvalidProof("Bad path")); - } - - last_root_hash = proof_result.0; - } else { - break; - } - - split_path = path_slice.split_last(); - } - - Ok((last_root_hash, result_map)) - } -} +// use std::collections::HashMap; +// +// use merk::proofs::query::Map; +// use rs_merkle::{algorithms::Sha256, MerkleProof}; +// +// use crate::{Element, Error, GroveDb, PathQuery, Proof, Query}; +// +// impl GroveDb { +// pub fn proof(&mut self, proof_queries: Vec) -> Result, Error> { +// // To prove a path we need to return a proof for each node on the path including +// // the root. With multiple paths, nodes can overlap i.e two or more paths can +// // share the same nodes. We should only have one proof for each node, +// // if a node forks into multiple relevant paths then we should create a +// // combined proof for that node with all the relevant keys +// let mut query_paths = Vec::new(); +// let mut intermediate_proof_spec: HashMap, Query> = HashMap::new(); +// let mut root_keys: Vec> = Vec::new(); +// let mut proofs: HashMap, Vec> = HashMap::new(); +// +// // For each unique node including the root +// // determine what keys would need to be included in the proof +// for proof_query in proof_queries.iter() { +// query_paths.push( +// proof_query +// .path +// .iter() +// .map(|x| x.to_vec()) +// .collect::>(), +// ); +// +// let mut split_path = proof_query.path.split_last(); +// while let Some((key, path_slice)) = split_path { +// if path_slice.is_empty() { +// // We have gotten to the root node +// let compressed_path = GroveDb::compress_subtree_key(&[], Some(key)); +// root_keys.push(compressed_path); +// } else { +// let compressed_path = GroveDb::compress_subtree_key(path_slice, None); +// if let Some(path_query) = intermediate_proof_spec.get_mut(&compressed_path) { +// path_query.insert_key(key.to_vec()); +// } else { +// let mut path_query = Query::new(); +// path_query.insert_key(key.to_vec()); +// intermediate_proof_spec.insert(compressed_path, path_query); +// } +// } +// split_path = path_slice.split_last(); +// } +// } +// +// // Construct the path proofs +// for (path, query) in intermediate_proof_spec { +// let proof = self.prove_item(&path, query)?; +// proofs.insert(path, proof); +// } +// +// // Construct the leaf proofs +// for proof_query in proof_queries { +// let path = proof_query.path; +// +// // If there is a subquery with a limit it's possible that we only need a reduced +// // proof for this leaf. +// let reduced_proof_query = proof_query; +// +// // First we must get elements +// +// if let Some(subquery_key) = reduced_proof_query.query.query.subquery_key.clone() { +// self.get_path_queries_raw(&[&reduced_proof_query], None)?; +// +// let mut path_vec = path.to_vec(); +// path_vec.push(subquery_key.as_slice()); +// let compressed_path = GroveDb::compress_subtree_key(path_vec.as_slice(), None); +// } +// +// // Now we must insert the final proof for the sub leaves +// let compressed_path = GroveDb::compress_subtree_key(path, None); +// let proof = self.prove_path_item(&compressed_path, reduced_proof_query)?; +// proofs.insert(compressed_path, proof); +// } +// +// // Construct the root proof +// let mut root_index: Vec = Vec::new(); +// for key in root_keys { +// let index = self +// .root_leaf_keys +// .get(&key) +// .ok_or(Error::InvalidPath("root key not found"))?; +// root_index.push(*index); +// } +// let root_proof = self.root_tree.proof(&root_index).to_bytes(); +// +// let proof = Proof { +// query_paths, +// proofs, +// root_proof, +// root_leaf_keys: self.root_leaf_keys.clone(), +// }; +// +// let seralized_proof = bincode::serialize(&proof) +// .map_err(|_| Error::CorruptedData(String::from("unable to serialize proof")))?; +// +// Ok(seralized_proof) +// } +// +// fn prove_path_item( +// &self, +// compressed_path: &Vec, +// path_query: PathQuery, +// ) -> Result, Error> { +// let merk = self +// .subtrees +// .get(compressed_path) +// .ok_or(Error::InvalidPath("no subtree found under that path"))?; +// +// let sized_query = path_query.query; +// +// if sized_query.query.subquery.is_none() { +// // then limit should be applied directly to the proof here +// let proof_result = merk +// .prove( +// sized_query.query, +// sized_query.limit, +// sized_query.offset, +// ) +// .expect("should prove both inclusion and absence"); +// Ok(proof_result) +// } else { +// let proof_result = merk +// .prove(sized_query.query, None, None) +// .expect("should prove both inclusion and absence"); +// Ok(proof_result) +// } +// } +// +// fn prove_item(&self, path: &Vec, query: Query) -> Result, Error> { +// let merk = self +// .subtrees +// .get(path) +// .ok_or(Error::InvalidPath("no subtree found under that path"))?; +// +// // then limit should be applied directly to the proof here +// let proof_result = merk +// .prove(query, None, None) +// .expect("should prove both inclusion and absence"); +// Ok(proof_result) +// } +// +// pub fn execute_proof(proof: Vec) -> Result<([u8; 32], HashMap, Map>), Error> { +// // Deserialize the proof +// let proof: Proof = bincode::deserialize(&proof) +// .map_err(|_| Error::CorruptedData(String::from("unable to deserialize proof")))?; +// +// // Required to execute the root proof +// let mut root_keys_index: Vec = Vec::new(); +// let mut root_hashes: Vec<[u8; 32]> = Vec::new(); +// +// // Collects the result map for each query +// let mut result_map: HashMap, Map> = HashMap::new(); +// +// for path in proof.query_paths { +// let path = path.iter().map(|x| x.as_slice()).collect::>(); +// // For each query path, get the result map after execution +// // and store hash + index for later root proof execution +// let root_key = &path[0]; +// let (hash, proof_result_map) = GroveDb::execute_path(&path, &proof.proofs)?; +// let compressed_root_key_path = GroveDb::compress_subtree_key(&[], Some(root_key)); +// let compressed_query_path = GroveDb::compress_subtree_key(&path, None); +// +// let index = proof +// .root_leaf_keys +// .get(&compressed_root_key_path) +// .ok_or(Error::InvalidPath("Bad path"))?; +// if !root_keys_index.contains(index) { +// root_keys_index.push(*index); +// root_hashes.push(hash); +// } +// +// result_map.insert(compressed_query_path, proof_result_map); +// } +// +// let root_proof = match MerkleProof::::try_from(proof.root_proof) { +// Ok(proof) => Ok(proof), +// Err(_) => Err(Error::InvalidProof("Invalid proof element")), +// }?; +// +// let root_hash = +// match root_proof.root(&root_keys_index, &root_hashes, proof.root_leaf_keys.len()) { +// Ok(hash) => Ok(hash), +// Err(_) => Err(Error::InvalidProof("Invalid proof element")), +// }?; +// +// Ok((root_hash, result_map)) +// } +// +// // Given a query path and a set of proofs +// // execute_path validates that the nodes represented by the paths +// // are connected to one another i.e root hash of child node is in parent node +// // at the correct key. +// // If path is valid, it returns the root hash of topmost merk and result map of +// // leaf merk. +// fn execute_path( +// path: &[&[u8]], +// proofs: &HashMap, Vec>, +// ) -> Result<([u8; 32], Map), Error> { +// let compressed_path = GroveDb::compress_subtree_key(path, None); +// let proof = proofs +// .get(&compressed_path) +// .ok_or(Error::InvalidPath("Bad path"))?; +// +// // Execute the leaf merk proof +// let (mut last_root_hash, result_map) = match merk::execute_proof(&proof[..]) { +// Ok(result) => Ok(result), +// Err(_) => Err(Error::InvalidPath("Invalid proof element")), +// }?; +// +// // Validate the path +// let mut split_path = path.split_last(); +// while let Some((key, path_slice)) = split_path { +// if !path_slice.is_empty() { +// let compressed_path = GroveDb::compress_subtree_key(path_slice, None); +// let proof = proofs +// .get(&compressed_path) +// .ok_or(Error::InvalidPath("Bad path"))?; +// +// let proof_result = match merk::execute_proof(&proof[..]) { +// Ok(result) => Ok(result), +// Err(_) => Err(Error::InvalidPath("Invalid proof element")), +// }?; +// +// let result_map = proof_result.1; +// // TODO: Handle the error better here +// let elem: Element = +// bincode::deserialize(result_map.get(key).unwrap().unwrap()).unwrap(); +// let merk_root_hash = match elem { +// Element::Tree(hash) => Ok(hash), +// _ => Err(Error::InvalidProof( +// "Intermediate proofs should be for trees", +// )), +// }?; +// +// if merk_root_hash != last_root_hash { +// return Err(Error::InvalidProof("Bad path")); +// } +// +// last_root_hash = proof_result.0; +// } else { +// break; +// } +// +// split_path = path_slice.split_last(); +// } +// +// Ok((last_root_hash, result_map)) +// } +// } diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 0ca6d5aa9..9c46d7650 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -268,377 +268,377 @@ fn test_root_tree_leafs_are_noted() { assert_eq!(db.root_tree.leaves_len(), 2); } -#[test] -fn test_proof_construction() { - // Tree Structure - // root - // test_leaf - // innertree - // k1,v1 - // k2,v2 - // another_test_leaf - // innertree2 - // k3,v3 - // innertree3 - // k4,v4 - - // Insert elements into grovedb instance - let mut temp_db = make_grovedb(); - // Insert level 1 nodes - temp_db - .insert( - &[TEST_LEAF], - b"innertree".to_vec(), - Element::empty_tree(), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[ANOTHER_TEST_LEAF], - b"innertree2".to_vec(), - Element::empty_tree(), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[ANOTHER_TEST_LEAF], - b"innertree3".to_vec(), - Element::empty_tree(), - None, - ) - .expect("successful subtree insert"); - // Insert level 2 nodes - temp_db - .insert( - &[TEST_LEAF, b"innertree"], - b"key1".to_vec(), - Element::Item(b"value1".to_vec()), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[TEST_LEAF, b"innertree"], - b"key2".to_vec(), - Element::Item(b"value2".to_vec()), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[ANOTHER_TEST_LEAF, b"innertree2"], - b"key3".to_vec(), - Element::Item(b"value3".to_vec()), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[ANOTHER_TEST_LEAF, b"innertree3"], - b"key4".to_vec(), - Element::Item(b"value4".to_vec()), - None, - ) - .expect("successful subtree insert"); - - // Manually construct HADS bottom up - // Insert level 2 nodes - let mut inner_tree = TempMerk::new(); - let value_one = Element::Item(b"value1".to_vec()); - value_one - .insert(&mut inner_tree, b"key1".to_vec(), None) - .unwrap(); - let value_two = Element::Item(b"value2".to_vec()); - value_two - .insert(&mut inner_tree, b"key2".to_vec(), None) - .unwrap(); - - let mut inner_tree_2 = TempMerk::new(); - let value_three = Element::Item(b"value3".to_vec()); - value_three - .insert(&mut inner_tree_2, b"key3".to_vec(), None) - .unwrap(); - - let mut inner_tree_3 = TempMerk::new(); - let value_four = Element::Item(b"value4".to_vec()); - value_four - .insert(&mut inner_tree_3, b"key4".to_vec(), None) - .unwrap(); - // Insert level 1 nodes - let mut test_leaf = TempMerk::new(); - let inner_tree_root = Element::Tree(inner_tree.root_hash()); - inner_tree_root - .insert(&mut test_leaf, b"innertree".to_vec(), None) - .unwrap(); - let mut another_test_leaf = TempMerk::new(); - let inner_tree_2_root = Element::Tree(inner_tree_2.root_hash()); - inner_tree_2_root - .insert(&mut another_test_leaf, b"innertree2".to_vec(), None) - .unwrap(); - let inner_tree_3_root = Element::Tree(inner_tree_3.root_hash()); - inner_tree_3_root - .insert(&mut another_test_leaf, b"innertree3".to_vec(), None) - .unwrap(); - // Insert root nodes - let leaves = [test_leaf.root_hash(), another_test_leaf.root_hash()]; - let root_tree = MerkleTree::::from_leaves(&leaves); - - // Proof construction - // Generating a proof for two paths - // root -> test_leaf -> innertree (prove both k1 and k2) - // root -> another_test_leaf -> innertree3 (prove k4) - // root -> another_test_leaf -> innertree2 (prove k3) - - // Build reusable query objects - let mut path_one_query = Query::new(); - path_one_query.insert_key(b"key1".to_vec()); - path_one_query.insert_key(b"key2".to_vec()); - - let mut path_two_query = Query::new(); - path_two_query.insert_key(b"key4".to_vec()); - - let mut path_three_query = Query::new(); - path_three_query.insert_key(b"key3".to_vec()); - - // Get grovedb proof - let proof = temp_db - .proof(vec![ - PathQuery::new_unsized(&[TEST_LEAF, b"innertree"], path_one_query), - PathQuery::new_unsized(&[ANOTHER_TEST_LEAF, b"innertree3"], path_two_query), - PathQuery::new_unsized(&[ANOTHER_TEST_LEAF, b"innertree2"], path_three_query), - ]) - .unwrap(); - - // Deserialize the proof - let proof: Proof = bincode::deserialize(&proof).unwrap(); - - // Perform assertions - assert_eq!(proof.query_paths.len(), 3); - assert_eq!(proof.query_paths[0], &[TEST_LEAF, b"innertree"]); - assert_eq!(proof.query_paths[1], &[ANOTHER_TEST_LEAF, b"innertree3"]); - assert_eq!(proof.query_paths[2], &[ANOTHER_TEST_LEAF, b"innertree2"]); - - // For path 1 to path 3, there are 9 nodes - // root is repeated three times and another_test_leaf is repeated twice - // Accounting for duplication, there are 6 unique nodes - // root, test_leaf, another_test_leaf, innertree, innertree2, innertree3 - // proof.proofs contains all nodes except the root so we expect 5 sub proofs - assert_eq!(proof.proofs.len(), 5); - - // Check that all the subproofs were constructed correctly for each path and - // subpath - let path_one_as_vec = GroveDb::compress_subtree_key(&[TEST_LEAF, b"innertree"], None); - let path_two_as_vec = GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF, b"innertree3"], None); - let path_three_as_vec = - GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF, b"innertree2"], None); - let test_leaf_path_as_vec = GroveDb::compress_subtree_key(&[TEST_LEAF], None); - let another_test_leaf_path_as_vec = GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF], None); - - let proof_for_path_one = proof.proofs.get(&path_one_as_vec).unwrap(); - let proof_for_path_two = proof.proofs.get(&path_two_as_vec).unwrap(); - let proof_for_path_three = proof.proofs.get(&path_three_as_vec).unwrap(); - let proof_for_test_leaf = proof.proofs.get(&test_leaf_path_as_vec).unwrap(); - let proof_for_another_test_leaf = proof.proofs.get(&another_test_leaf_path_as_vec).unwrap(); - - // Assert path 1 proof - let mut proof_query = Query::new(); - proof_query.insert_key(b"key1".to_vec()); - proof_query.insert_key(b"key2".to_vec()); - assert_eq!( - *proof_for_path_one, - inner_tree.prove(proof_query, None, None).unwrap() - ); - - // Assert path 2 proof - let mut proof_query = Query::new(); - proof_query.insert_key(b"key4".to_vec()); - assert_eq!( - *proof_for_path_two, - inner_tree_3.prove(proof_query, None, None).unwrap() - ); - - // Assert path 3 proof - let mut proof_query = Query::new(); - proof_query.insert_key(b"key3".to_vec()); - assert_eq!( - *proof_for_path_three, - inner_tree_2.prove(proof_query, None, None).unwrap() - ); - - // Assert test leaf proof - let mut proof_query = Query::new(); - proof_query.insert_key(b"innertree".to_vec()); - assert_eq!( - *proof_for_test_leaf, - test_leaf.prove(proof_query, None, None).unwrap() - ); - - // Assert another test leaf proof - // another test leaf appeared in two path, - // hence it should contain proofs for both keys - let mut proof_query = Query::new(); - proof_query.insert_key(b"innertree2".to_vec()); - proof_query.insert_key(b"innertree3".to_vec()); - assert_eq!( - *proof_for_another_test_leaf, - another_test_leaf - .prove(proof_query, None, None) - .unwrap() - ); - - // Check that the root proof is valid - // Root proof should contain proof for both test_leaf and another_test_leaf - let test_leaf_root_key = GroveDb::compress_subtree_key(&[], Some(TEST_LEAF)); - let another_test_leaf_root_key = GroveDb::compress_subtree_key(&[], Some(ANOTHER_TEST_LEAF)); - assert_eq!( - proof.root_proof, - root_tree - .proof(&[ - temp_db.root_leaf_keys[&test_leaf_root_key], - temp_db.root_leaf_keys[&another_test_leaf_root_key], - ]) - .to_bytes() - ); - - // Assert that we got the correct root leaf keys - assert_eq!(proof.root_leaf_keys.len(), 2); - assert_eq!(proof.root_leaf_keys[&test_leaf_root_key], 0); - assert_eq!(proof.root_leaf_keys[&another_test_leaf_root_key], 1); -} - -#[test] -fn test_successful_proof_verification() { - // Build a grovedb database - // Tree Structure - // root - // test_leaf - // innertree - // k1,v1 - // k2,v2 - // another_test_leaf - // innertree2 - // k3,v3 - // innertree3 - // k4,v4 - - // Insert elements into grovedb instance - let mut temp_db = make_grovedb(); - // Insert level 1 nodes - temp_db - .insert( - &[TEST_LEAF], - b"innertree".to_vec(), - Element::empty_tree(), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[ANOTHER_TEST_LEAF], - b"innertree2".to_vec(), - Element::empty_tree(), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[ANOTHER_TEST_LEAF], - b"innertree3".to_vec(), - Element::empty_tree(), - None, - ) - .expect("successful subtree insert"); - // Insert level 2 nodes - temp_db - .insert( - &[TEST_LEAF, b"innertree"], - b"key1".to_vec(), - Element::Item(b"value1".to_vec()), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[TEST_LEAF, b"innertree"], - b"key2".to_vec(), - Element::Item(b"value2".to_vec()), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[ANOTHER_TEST_LEAF, b"innertree2"], - b"key3".to_vec(), - Element::Item(b"value3".to_vec()), - None, - ) - .expect("successful subtree insert"); - temp_db - .insert( - &[ANOTHER_TEST_LEAF, b"innertree3"], - b"key4".to_vec(), - Element::Item(b"value4".to_vec()), - None, - ) - .expect("successful subtree insert"); - - // Single query proof verification - let mut path_one_query = Query::new(); - path_one_query.insert_key(b"key1".to_vec()); - path_one_query.insert_key(b"key2".to_vec()); - - let proof = temp_db - .proof(vec![PathQuery::new_unsized( - &[TEST_LEAF, b"innertree"], - path_one_query, - )]) - .unwrap(); - - // Assert correct root hash - let (root_hash, result_maps) = GroveDb::execute_proof(proof).unwrap(); - assert_eq!(temp_db.root_tree.root().unwrap(), root_hash); - - // Assert correct result object - // Proof query was for two keys key1 and key2 - let path_as_vec = GroveDb::compress_subtree_key(&[TEST_LEAF, b"innertree"], None); - let result_map = result_maps.get(&path_as_vec).unwrap(); - let elem_1: Element = bincode::deserialize(result_map.get(b"key1").unwrap().unwrap()).unwrap(); - let elem_2: Element = bincode::deserialize(result_map.get(b"key2").unwrap().unwrap()).unwrap(); - assert_eq!(elem_1, Element::Item(b"value1".to_vec())); - assert_eq!(elem_2, Element::Item(b"value2".to_vec())); - - // Multi query proof verification - let mut path_two_query = Query::new(); - path_two_query.insert_key(b"key4".to_vec()); - - let mut path_three_query = Query::new(); - path_three_query.insert_key(b"key3".to_vec()); - - // Get grovedb proof - let proof = temp_db - .proof(vec![ - PathQuery::new_unsized(&[ANOTHER_TEST_LEAF, b"innertree3"], path_two_query), - PathQuery::new_unsized(&[ANOTHER_TEST_LEAF, b"innertree2"], path_three_query), - ]) - .unwrap(); - - // Assert correct root hash - let (root_hash, result_maps) = GroveDb::execute_proof(proof).unwrap(); - assert_eq!(temp_db.root_tree.root().unwrap(), root_hash); - - // Assert correct result object - let path_one_as_vec = GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF, b"innertree3"], None); - let result_map = result_maps.get(&path_one_as_vec).unwrap(); - let elem: Element = bincode::deserialize(result_map.get(b"key4").unwrap().unwrap()).unwrap(); - assert_eq!(elem, Element::Item(b"value4".to_vec())); +// #[test] +// fn test_proof_construction() { +// // Tree Structure +// // root +// // test_leaf +// // innertree +// // k1,v1 +// // k2,v2 +// // another_test_leaf +// // innertree2 +// // k3,v3 +// // innertree3 +// // k4,v4 +// +// // Insert elements into grovedb instance +// let mut temp_db = make_grovedb(); +// // Insert level 1 nodes +// temp_db +// .insert( +// &[TEST_LEAF], +// b"innertree".to_vec(), +// Element::empty_tree(), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[ANOTHER_TEST_LEAF], +// b"innertree2".to_vec(), +// Element::empty_tree(), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[ANOTHER_TEST_LEAF], +// b"innertree3".to_vec(), +// Element::empty_tree(), +// None, +// ) +// .expect("successful subtree insert"); +// // Insert level 2 nodes +// temp_db +// .insert( +// &[TEST_LEAF, b"innertree"], +// b"key1".to_vec(), +// Element::Item(b"value1".to_vec()), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[TEST_LEAF, b"innertree"], +// b"key2".to_vec(), +// Element::Item(b"value2".to_vec()), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[ANOTHER_TEST_LEAF, b"innertree2"], +// b"key3".to_vec(), +// Element::Item(b"value3".to_vec()), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[ANOTHER_TEST_LEAF, b"innertree3"], +// b"key4".to_vec(), +// Element::Item(b"value4".to_vec()), +// None, +// ) +// .expect("successful subtree insert"); +// +// // Manually construct HADS bottom up +// // Insert level 2 nodes +// let mut inner_tree = TempMerk::new(); +// let value_one = Element::Item(b"value1".to_vec()); +// value_one +// .insert(&mut inner_tree, b"key1".to_vec(), None) +// .unwrap(); +// let value_two = Element::Item(b"value2".to_vec()); +// value_two +// .insert(&mut inner_tree, b"key2".to_vec(), None) +// .unwrap(); +// +// let mut inner_tree_2 = TempMerk::new(); +// let value_three = Element::Item(b"value3".to_vec()); +// value_three +// .insert(&mut inner_tree_2, b"key3".to_vec(), None) +// .unwrap(); +// +// let mut inner_tree_3 = TempMerk::new(); +// let value_four = Element::Item(b"value4".to_vec()); +// value_four +// .insert(&mut inner_tree_3, b"key4".to_vec(), None) +// .unwrap(); +// // Insert level 1 nodes +// let mut test_leaf = TempMerk::new(); +// let inner_tree_root = Element::Tree(inner_tree.root_hash()); +// inner_tree_root +// .insert(&mut test_leaf, b"innertree".to_vec(), None) +// .unwrap(); +// let mut another_test_leaf = TempMerk::new(); +// let inner_tree_2_root = Element::Tree(inner_tree_2.root_hash()); +// inner_tree_2_root +// .insert(&mut another_test_leaf, b"innertree2".to_vec(), None) +// .unwrap(); +// let inner_tree_3_root = Element::Tree(inner_tree_3.root_hash()); +// inner_tree_3_root +// .insert(&mut another_test_leaf, b"innertree3".to_vec(), None) +// .unwrap(); +// // Insert root nodes +// let leaves = [test_leaf.root_hash(), another_test_leaf.root_hash()]; +// let root_tree = MerkleTree::::from_leaves(&leaves); +// +// // Proof construction +// // Generating a proof for two paths +// // root -> test_leaf -> innertree (prove both k1 and k2) +// // root -> another_test_leaf -> innertree3 (prove k4) +// // root -> another_test_leaf -> innertree2 (prove k3) +// +// // Build reusable query objects +// let mut path_one_query = Query::new(); +// path_one_query.insert_key(b"key1".to_vec()); +// path_one_query.insert_key(b"key2".to_vec()); +// +// let mut path_two_query = Query::new(); +// path_two_query.insert_key(b"key4".to_vec()); +// +// let mut path_three_query = Query::new(); +// path_three_query.insert_key(b"key3".to_vec()); +// +// // Get grovedb proof +// let proof = temp_db +// .proof(vec![ +// PathQuery::new_unsized(&[TEST_LEAF, b"innertree"], path_one_query), +// PathQuery::new_unsized(&[ANOTHER_TEST_LEAF, b"innertree3"], path_two_query), +// PathQuery::new_unsized(&[ANOTHER_TEST_LEAF, b"innertree2"], path_three_query), +// ]) +// .unwrap(); +// +// // Deserialize the proof +// let proof: Proof = bincode::deserialize(&proof).unwrap(); +// +// // Perform assertions +// assert_eq!(proof.query_paths.len(), 3); +// assert_eq!(proof.query_paths[0], &[TEST_LEAF, b"innertree"]); +// assert_eq!(proof.query_paths[1], &[ANOTHER_TEST_LEAF, b"innertree3"]); +// assert_eq!(proof.query_paths[2], &[ANOTHER_TEST_LEAF, b"innertree2"]); +// +// // For path 1 to path 3, there are 9 nodes +// // root is repeated three times and another_test_leaf is repeated twice +// // Accounting for duplication, there are 6 unique nodes +// // root, test_leaf, another_test_leaf, innertree, innertree2, innertree3 +// // proof.proofs contains all nodes except the root so we expect 5 sub proofs +// assert_eq!(proof.proofs.len(), 5); +// +// // Check that all the subproofs were constructed correctly for each path and +// // subpath +// let path_one_as_vec = GroveDb::compress_subtree_key(&[TEST_LEAF, b"innertree"], None); +// let path_two_as_vec = GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF, b"innertree3"], None); +// let path_three_as_vec = +// GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF, b"innertree2"], None); +// let test_leaf_path_as_vec = GroveDb::compress_subtree_key(&[TEST_LEAF], None); +// let another_test_leaf_path_as_vec = GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF], None); +// +// let proof_for_path_one = proof.proofs.get(&path_one_as_vec).unwrap(); +// let proof_for_path_two = proof.proofs.get(&path_two_as_vec).unwrap(); +// let proof_for_path_three = proof.proofs.get(&path_three_as_vec).unwrap(); +// let proof_for_test_leaf = proof.proofs.get(&test_leaf_path_as_vec).unwrap(); +// let proof_for_another_test_leaf = proof.proofs.get(&another_test_leaf_path_as_vec).unwrap(); +// +// // Assert path 1 proof +// let mut proof_query = Query::new(); +// proof_query.insert_key(b"key1".to_vec()); +// proof_query.insert_key(b"key2".to_vec()); +// assert_eq!( +// *proof_for_path_one, +// inner_tree.prove(proof_query, None, None).unwrap() +// ); +// +// // Assert path 2 proof +// let mut proof_query = Query::new(); +// proof_query.insert_key(b"key4".to_vec()); +// assert_eq!( +// *proof_for_path_two, +// inner_tree_3.prove(proof_query, None, None).unwrap() +// ); +// +// // Assert path 3 proof +// let mut proof_query = Query::new(); +// proof_query.insert_key(b"key3".to_vec()); +// assert_eq!( +// *proof_for_path_three, +// inner_tree_2.prove(proof_query, None, None).unwrap() +// ); +// +// // Assert test leaf proof +// let mut proof_query = Query::new(); +// proof_query.insert_key(b"innertree".to_vec()); +// assert_eq!( +// *proof_for_test_leaf, +// test_leaf.prove(proof_query, None, None).unwrap() +// ); +// +// // Assert another test leaf proof +// // another test leaf appeared in two path, +// // hence it should contain proofs for both keys +// let mut proof_query = Query::new(); +// proof_query.insert_key(b"innertree2".to_vec()); +// proof_query.insert_key(b"innertree3".to_vec()); +// assert_eq!( +// *proof_for_another_test_leaf, +// another_test_leaf +// .prove(proof_query, None, None) +// .unwrap() +// ); +// +// // Check that the root proof is valid +// // Root proof should contain proof for both test_leaf and another_test_leaf +// let test_leaf_root_key = GroveDb::compress_subtree_key(&[], Some(TEST_LEAF)); +// let another_test_leaf_root_key = GroveDb::compress_subtree_key(&[], Some(ANOTHER_TEST_LEAF)); +// assert_eq!( +// proof.root_proof, +// root_tree +// .proof(&[ +// temp_db.root_leaf_keys[&test_leaf_root_key], +// temp_db.root_leaf_keys[&another_test_leaf_root_key], +// ]) +// .to_bytes() +// ); +// +// // Assert that we got the correct root leaf keys +// assert_eq!(proof.root_leaf_keys.len(), 2); +// assert_eq!(proof.root_leaf_keys[&test_leaf_root_key], 0); +// assert_eq!(proof.root_leaf_keys[&another_test_leaf_root_key], 1); +// } - let path_two_as_vec = GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF, b"innertree2"], None); - let result_map = result_maps.get(&path_two_as_vec).unwrap(); - let elem: Element = bincode::deserialize(result_map.get(b"key3").unwrap().unwrap()).unwrap(); - assert_eq!(elem, Element::Item(b"value3".to_vec())); -} +// #[test] +// fn test_successful_proof_verification() { +// // Build a grovedb database +// // Tree Structure +// // root +// // test_leaf +// // innertree +// // k1,v1 +// // k2,v2 +// // another_test_leaf +// // innertree2 +// // k3,v3 +// // innertree3 +// // k4,v4 +// +// // Insert elements into grovedb instance +// let mut temp_db = make_grovedb(); +// // Insert level 1 nodes +// temp_db +// .insert( +// &[TEST_LEAF], +// b"innertree".to_vec(), +// Element::empty_tree(), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[ANOTHER_TEST_LEAF], +// b"innertree2".to_vec(), +// Element::empty_tree(), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[ANOTHER_TEST_LEAF], +// b"innertree3".to_vec(), +// Element::empty_tree(), +// None, +// ) +// .expect("successful subtree insert"); +// // Insert level 2 nodes +// temp_db +// .insert( +// &[TEST_LEAF, b"innertree"], +// b"key1".to_vec(), +// Element::Item(b"value1".to_vec()), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[TEST_LEAF, b"innertree"], +// b"key2".to_vec(), +// Element::Item(b"value2".to_vec()), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[ANOTHER_TEST_LEAF, b"innertree2"], +// b"key3".to_vec(), +// Element::Item(b"value3".to_vec()), +// None, +// ) +// .expect("successful subtree insert"); +// temp_db +// .insert( +// &[ANOTHER_TEST_LEAF, b"innertree3"], +// b"key4".to_vec(), +// Element::Item(b"value4".to_vec()), +// None, +// ) +// .expect("successful subtree insert"); +// +// // Single query proof verification +// let mut path_one_query = Query::new(); +// path_one_query.insert_key(b"key1".to_vec()); +// path_one_query.insert_key(b"key2".to_vec()); +// +// let proof = temp_db +// .proof(vec![PathQuery::new_unsized( +// &[TEST_LEAF, b"innertree"], +// path_one_query, +// )]) +// .unwrap(); +// +// // Assert correct root hash +// let (root_hash, result_maps) = GroveDb::execute_proof(proof).unwrap(); +// assert_eq!(temp_db.root_tree.root().unwrap(), root_hash); +// +// // Assert correct result object +// // Proof query was for two keys key1 and key2 +// let path_as_vec = GroveDb::compress_subtree_key(&[TEST_LEAF, b"innertree"], None); +// let result_map = result_maps.get(&path_as_vec).unwrap(); +// let elem_1: Element = bincode::deserialize(result_map.get(b"key1").unwrap().unwrap()).unwrap(); +// let elem_2: Element = bincode::deserialize(result_map.get(b"key2").unwrap().unwrap()).unwrap(); +// assert_eq!(elem_1, Element::Item(b"value1".to_vec())); +// assert_eq!(elem_2, Element::Item(b"value2".to_vec())); +// +// // Multi query proof verification +// let mut path_two_query = Query::new(); +// path_two_query.insert_key(b"key4".to_vec()); +// +// let mut path_three_query = Query::new(); +// path_three_query.insert_key(b"key3".to_vec()); +// +// // Get grovedb proof +// let proof = temp_db +// .proof(vec![ +// PathQuery::new_unsized(&[ANOTHER_TEST_LEAF, b"innertree3"], path_two_query), +// PathQuery::new_unsized(&[ANOTHER_TEST_LEAF, b"innertree2"], path_three_query), +// ]) +// .unwrap(); +// +// // Assert correct root hash +// let (root_hash, result_maps) = GroveDb::execute_proof(proof).unwrap(); +// assert_eq!(temp_db.root_tree.root().unwrap(), root_hash); +// +// // Assert correct result object +// let path_one_as_vec = GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF, b"innertree3"], None); +// let result_map = result_maps.get(&path_one_as_vec).unwrap(); +// let elem: Element = bincode::deserialize(result_map.get(b"key4").unwrap().unwrap()).unwrap(); +// assert_eq!(elem, Element::Item(b"value4".to_vec())); +// +// let path_two_as_vec = GroveDb::compress_subtree_key(&[ANOTHER_TEST_LEAF, b"innertree2"], None); +// let result_map = result_maps.get(&path_two_as_vec).unwrap(); +// let elem: Element = bincode::deserialize(result_map.get(b"key3").unwrap().unwrap()).unwrap(); +// assert_eq!(elem, Element::Item(b"value3".to_vec())); +// } // #[test] // fn test_checkpoint() { From 41b09924703e66daefae52d8aff81d5887dfd273 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 15:38:23 +0700 Subject: [PATCH 25/31] removed comments --- grovedb/src/operations/delete.rs | 39 ---------------------- grovedb/src/operations/get.rs | 56 -------------------------------- grovedb/src/operations/insert.rs | 20 +----------- grovedb/src/subtrees.rs | 19 ----------- 4 files changed, 1 insertion(+), 133 deletions(-) diff --git a/grovedb/src/operations/delete.rs b/grovedb/src/operations/delete.rs index 0025c2c63..fc411a593 100644 --- a/grovedb/src/operations/delete.rs +++ b/grovedb/src/operations/delete.rs @@ -22,40 +22,7 @@ impl GroveDb { } else { let element = self.get_raw(path, &key, transaction)?; { - // To get the merk, we have two functions, - // 1. get the subtree without transaction - // 2. get with transaction - // - if temp doesn't have get without transaction - // how do we return a reference to the memory location tho - // if we might have to remove the merk so it is always the correct instance let mut merk = self.get_subtrees().get(path, transaction)?; - // let mut merk; - // match transaction { - // None => { - // merk = self.get_subtrees().get_subtree_without_transaction(path)?; - // }, - // Some(_) => { - // let prefix = &Self::compress_subtree_key(path, None); - // if self.temp_subtrees.borrow().contains_key(prefix) { - // // get the merk out - // merk = - // self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the - // hashmap"); } else { - // // merk is not in the hash map get it without transaction - // merk = - // self.get_subtrees().get_subtree_without_transaction(path)?; - // } - // } - // } - - // let subtrees = match transaction { - // None => &mut self.subtrees, - // Some(_) => &mut self.temp_subtrees, - // }; - - // let mut merk = subtrees - // .get_mut(&Self::compress_subtree_key(path, None)) - // .ok_or(Error::InvalidPath("no subtree found under that path"))?; Element::delete(&mut merk, key.clone(), transaction)?; // after deletion, if there is a transaction, add the merk back into the hashmap @@ -68,18 +35,12 @@ impl GroveDb { let mut concat_path: Vec> = path.iter().map(|x| x.to_vec()).collect(); concat_path.push(key); let subtrees_paths = self.find_subtrees(concat_path, transaction)?; - // let subtrees = match transaction { - // None => &mut self.subtrees, - // Some(_) => &mut self.temp_subtrees, - // }; for subtree_path in subtrees_paths { // TODO: eventually we need to do something about this nested slices let subtree_path_ref: Vec<&[u8]> = subtree_path.iter().map(|x| x.as_slice()).collect(); let prefix = Self::compress_subtree_key(&subtree_path_ref, None); - // Here we seem to get the subtree, then call clear on it - // Seems to only care about subtrees that are non transactional let subtree = self .get_subtrees() .get_subtree_without_transaction(subtree_path_ref.as_slice()); diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index b5db5aa46..388455522 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -75,36 +75,11 @@ impl GroveDb { transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { let merk = self.get_subtrees().get(path, transaction)?; - // let merk; - // match transaction { - // None => { - // merk = self.get_subtrees().get_subtree_without_transaction(path)?; - // }, - // Some(_) => { - // let prefix = &Self::compress_subtree_key(path, None); - // if self.temp_subtrees.borrow().contains_key(prefix) { - // // get the merk out - // merk = - // self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the - // hashmap"); } else { - // // merk is not in the hash map get it without transaction - // merk = - // self.get_subtrees().get_subtree_without_transaction(path)?; } - // } - // } let elem = Element::get(&merk, key); self.get_subtrees() .insert_temp_tree(path, merk, transaction); - // match transaction { - // None => {}, - // Some(_) => { - // let prefix = Self::compress_subtree_key(path, None); - // self.temp_subtrees.borrow_mut().insert(prefix, merk); - // } - // }; - elem } @@ -177,10 +152,6 @@ impl GroveDb { path_query: &PathQuery, transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result<(Vec, u16), Error> { - // let subtrees = match transaction { - // None => &self.subtrees, - // Some(_) => &self.temp_subtrees, - // }; let subtrees = self.get_subtrees(); self.get_path_query_on_trees_raw(path_query, subtrees, transaction) } @@ -193,38 +164,11 @@ impl GroveDb { // subtrees: &HashMap, Merk>, ) -> Result<(Vec, u16), Error> { let path = path_query.path; - // let merk = subtrees - // .get(&Self::compress_subtree_key(path, None)) - // .ok_or(Error::InvalidPath("no subtree found under that path"))?; - // let merk; let merk = subtrees.get(path, transaction)?; - // match transaction { - // None => { - // merk = self.get_subtrees().get_subtree_without_transaction(path)?; - // }, - // Some(_) => { - // let prefix = &Self::compress_subtree_key(path, None); - // if self.temp_subtrees.borrow().contains_key(prefix) { - // // get the merk out - // merk = - // self.temp_subtrees.borrow_mut().remove(prefix).expect("confirmed it's in the - // hashmap"); } else { - // // merk is not in the hash map get it without transaction - // merk = - // self.get_subtrees().get_subtree_without_transaction(path)?; } - // } - // } let elem = Element::get_path_query(&merk, path_query, Some(&subtrees)); subtrees.insert_temp_tree(path, merk, transaction); - // match transaction { - // None => {}, - // Some(_) => { - // let prefix = Self::compress_subtree_key(path, None); - // self.temp_subtrees.borrow_mut().insert(prefix, merk); - // } - // }; elem } diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index 99fb810d8..c0919acbc 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -49,10 +49,7 @@ impl GroveDb { "only subtrees are allowed as root tree's leafs", )); } - // Get a Merk by a path - // let mut merk = subtrees - // .get_mut(&Self::compress_subtree_key(path, None)) - // .ok_or(Error::InvalidPath("no subtree found under that path"))?; + let mut merk = self .get_subtrees() .get(path, transaction) @@ -78,15 +75,10 @@ impl GroveDb { } } - // let subtrees = match transaction { - // None => &mut self.subtrees, - // Some(_) => &mut self.temp_subtrees, - // }; // Open Merk and put handle into `subtrees` dictionary accessible by its // compressed path let (subtree_prefix, subtree_merk) = create_merk_with_prefix(self.db.clone(), &[], key)?; - // subtrees.insert(subtree_prefix.clone(), subtree_merk); self.get_subtrees() .insert_temp_tree_with_prefix(subtree_prefix, subtree_merk, transaction); @@ -137,16 +129,6 @@ impl GroveDb { self.get_subtrees() .insert_temp_tree_with_prefix(subtree_prefix, subtree_merk, transaction); - // Save subtrees, to be removed - // TODO: Remove this - // { - // let subtrees = match transaction { - // None => &mut self.subtrees, - // Some(_) => &mut self.temp_subtrees, - // }; - // subtrees.insert(subtree_prefix, subtree_merk); - // } - // Had to take merk from `subtrees` once again to solve multiple &mut s let mut merk = self .get_subtrees() diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs index 27623598a..04582985a 100644 --- a/grovedb/src/subtrees.rs +++ b/grovedb/src/subtrees.rs @@ -114,25 +114,6 @@ impl Subtrees<'_> { } } - // pub fn get_subtree_with_transaction( - // &self, - // path: &[&[u8]], - // ) -> Result<&Merk, Error> { - // let subtree_prefix = GroveDb::compress_subtree_key(path, None); - // if let Some(merk) = self.temp_subtrees.borrow().get(&subtree_prefix) { - // Ok(merk) - // } else { - // Err(Error::InvalidPath("no subtree found under that path")) - // // dbg!("Getting subtree without transaction"); - // // // if the subtree doesn't exist in temp_subtrees, - // // // check if it was created before the transaction was started - // // let merk = self - // // .get_subtree_without_transaction(path) - // // .map_err(|_| Error::InvalidPath("no subtree found under that - // path"))?; // Ok(merk) - // } - // } - fn get_subtree_with_key_info( &self, path: &[&[u8]], From 2555ad4dc4030b5cd797d92d611cf36b9cb152e9 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 16:05:49 +0700 Subject: [PATCH 26/31] removed unused imports --- grovedb/src/operations/get.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 388455522..35f5183e0 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -1,5 +1,4 @@ use std::{ - arch::x86_64::_mm_extract_si64, collections::{HashMap, HashSet}, ops::Range, rc::Rc, From 28571fa16a0baab4c117a898f9d705e001645655 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 17:20:18 +0700 Subject: [PATCH 27/31] Removed duplicate hashing when removing and immmediately inserting subtree --- grovedb/src/lib.rs | 29 ++++++++++++++++------ grovedb/src/operations/delete.rs | 13 +++++++--- grovedb/src/operations/get.rs | 20 +++++++++++---- grovedb/src/operations/insert.rs | 33 ++++++++++++++++++------- grovedb/src/operations/is_empty_tree.rs | 2 +- grovedb/src/subtree.rs | 18 +++++++++++--- grovedb/src/subtrees.rs | 13 +++++----- grovedb/src/tests.rs | 8 +++--- 8 files changed, 95 insertions(+), 41 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 15c3328af..79ac325d7 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -206,11 +206,15 @@ impl GroveDb { ) -> MerkleTree { let mut leaf_hashes: Vec<[u8; 32]> = vec![[0; 32]; root_leaf_keys.len()]; for (subtree_path, root_leaf_idx) in root_leaf_keys { - let subtree_merk = subtrees + let (subtree_merk, prefix) = subtrees .get(&[subtree_path.as_slice()], transaction) .expect("`root_leaf_keys` must be in sync with `subtrees`"); leaf_hashes[*root_leaf_idx] = subtree_merk.root_hash(); - subtrees.insert_temp_tree(&[subtree_path.as_slice()], subtree_merk, transaction); + if prefix.is_some() { + subtrees.insert_temp_tree_with_prefix(prefix.expect("confirmed as some"), subtree_merk, transaction); + } else { + subtrees.insert_temp_tree(&[subtree_path.as_slice()], subtree_merk, transaction); + } } MerkleTree::::from_leaves(&leaf_hashes) } @@ -251,16 +255,25 @@ impl GroveDb { // Go up until only one element in path, which means a key of a root tree while path.len() > 1 { // non root leaf node - let subtree = subtrees.get(path, transaction)?; + let (subtree, prefix) = subtrees.get(path, transaction)?; let element = Element::Tree(subtree.root_hash()); - self.get_subtrees() - .insert_temp_tree(path, subtree, transaction); + if prefix.is_some() { + self.get_subtrees() + .insert_temp_tree_with_prefix(prefix.expect("confirmed as some"), subtree, transaction); + } else { + self.get_subtrees() + .insert_temp_tree(path, subtree, transaction); + } let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; - let mut upper_tree = subtrees.get(parent_path, transaction)?; + let (mut upper_tree, prefix) = subtrees.get(parent_path, transaction)?; element.insert(&mut upper_tree, key.to_vec(), transaction); - self.get_subtrees() - .insert_temp_tree(parent_path, upper_tree, transaction); + if prefix.is_some() { + self.get_subtrees() + .insert_temp_tree(parent_path, upper_tree, transaction); + } else { + self.get_subtrees().insert_temp_tree(path, upper_tree, transaction); + } path = parent_path; } diff --git a/grovedb/src/operations/delete.rs b/grovedb/src/operations/delete.rs index fc411a593..466d7ec2a 100644 --- a/grovedb/src/operations/delete.rs +++ b/grovedb/src/operations/delete.rs @@ -22,12 +22,17 @@ impl GroveDb { } else { let element = self.get_raw(path, &key, transaction)?; { - let mut merk = self.get_subtrees().get(path, transaction)?; + let (mut merk, prefix) = self.get_subtrees().get(path, transaction)?; Element::delete(&mut merk, key.clone(), transaction)?; // after deletion, if there is a transaction, add the merk back into the hashmap - self.get_subtrees() - .insert_temp_tree(path, merk, transaction); + if prefix.is_some(){ + self.get_subtrees() + .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + } else { + self.get_subtrees() + .insert_temp_tree(path, merk, transaction); + } } if let Element::Tree(_) = element { @@ -78,7 +83,7 @@ impl GroveDb { // TODO: eventually we need to do something about this nested slices let q_ref: Vec<&[u8]> = q.iter().map(|x| x.as_slice()).collect(); // Get the correct subtree with q_ref as path - let merk = self.get_subtrees().get(&q_ref, transaction)?; + let (merk, _) = self.get_subtrees().get(&q_ref, transaction)?; let mut iter = Element::iterator(merk.raw_iter()); // let mut iter = self.elements_iterator(&q_ref, transaction)?; while let Some((key, value)) = iter.next()? { diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 35f5183e0..e7669b18b 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -73,12 +73,18 @@ impl GroveDb { key: &[u8], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let merk = self.get_subtrees().get(path, transaction)?; + let (merk, prefix) = self.get_subtrees().get(path, transaction)?; let elem = Element::get(&merk, key); - self.get_subtrees() - .insert_temp_tree(path, merk, transaction); + if prefix.is_some() { + self.get_subtrees() + .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + } else { + self.get_subtrees() + .insert_temp_tree(path, merk, transaction); + } + elem } @@ -163,11 +169,15 @@ impl GroveDb { // subtrees: &HashMap, Merk>, ) -> Result<(Vec, u16), Error> { let path = path_query.path; - let merk = subtrees.get(path, transaction)?; + let (merk, prefix) = subtrees.get(path, transaction)?; let elem = Element::get_path_query(&merk, path_query, Some(&subtrees)); - subtrees.insert_temp_tree(path, merk, transaction); + if prefix.is_some(){ + subtrees.insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + } else { + subtrees.insert_temp_tree(path, merk, transaction); + } elem } diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index c0919acbc..35e252009 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -50,13 +50,18 @@ impl GroveDb { )); } - let mut merk = self + let (mut merk, prefix) = self .get_subtrees() .get(path, transaction) .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; element.insert(&mut merk, key, transaction)?; - self.get_subtrees() - .insert_temp_tree(path, merk, transaction); + if prefix.is_some(){ + self.get_subtrees() + .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + } else { + self.get_subtrees() + .insert_temp_tree(path, merk, transaction); + } self.propagate_changes(path, transaction)?; } } @@ -118,9 +123,14 @@ impl GroveDb { } // First, check if a subtree exists to create a new subtree under it - let parent = self.get_subtrees().get(path, transaction)?; - self.get_subtrees() - .insert_temp_tree(path, parent, transaction); + let (parent, prefix) = self.get_subtrees().get(path, transaction)?; + if prefix.is_some() { + self.get_subtrees() + .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), parent, transaction); + } else { + self.get_subtrees() + .insert_temp_tree(path, parent, transaction); + } let (subtree_prefix, subtree_merk) = create_merk_with_prefix(self.db.clone(), path, &key)?; @@ -130,15 +140,20 @@ impl GroveDb { .insert_temp_tree_with_prefix(subtree_prefix, subtree_merk, transaction); // Had to take merk from `subtrees` once again to solve multiple &mut s - let mut merk = self + let (mut merk, prefix) = self .get_subtrees() .get(path, transaction) .expect("confirmed subtree exists above"); // need to mark key as taken in the upper tree element.insert(&mut merk, key, transaction)?; - self.get_subtrees() - .insert_temp_tree(path, merk, transaction); + if prefix.is_some(){ + self.get_subtrees() + .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + } else { + self.get_subtrees() + .insert_temp_tree(path, merk, transaction); + } self.propagate_changes(path, transaction)?; diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index 1f243c1ed..3e771aac9 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -8,7 +8,7 @@ impl GroveDb { path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let merk = self.get_subtrees().get(path, transaction)?; + let (merk, _) = self.get_subtrees().get(path, transaction)?; let mut iter = merk.raw_iter(); iter.seek_to_first(); diff --git a/grovedb/src/subtree.rs b/grovedb/src/subtree.rs index 746fa28a9..ddfe41eea 100644 --- a/grovedb/src/subtree.rs +++ b/grovedb/src/subtree.rs @@ -137,14 +137,20 @@ impl Element { if let Some(subquery_key) = &subquery_key_option { path_vec.push(subquery_key.as_slice()); } - let inner_merk = subtrees + let (inner_merk, prefix) = subtrees .get(path_vec.as_slice(), None) .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; let inner_query = SizedQuery::new(subquery, *limit, *offset); let inner_path_query = PathQuery::new(path_vec.as_slice(), inner_query); let (mut sub_elements, skipped) = Element::get_path_query(&inner_merk, &inner_path_query, subtrees_option)?; - subtrees.insert_temp_tree(path_vec.as_slice(), inner_merk, None); + + if prefix.is_some() { + subtrees.insert_temp_tree_with_prefix(prefix.expect("confirmed as some"), inner_merk, None); + } else { + subtrees.insert_temp_tree(path_vec.as_slice(), inner_merk, None); + } + if let Some(limit) = limit { *limit = *limit - sub_elements.len() as u16; } @@ -153,7 +159,7 @@ impl Element { } results.append(&mut sub_elements); } else if let Some(subquery_key) = subquery_key_option { - let inner_merk = subtrees + let (inner_merk, prefix) = subtrees .get(path_vec.as_slice(), None) .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; if offset.is_none() || offset.is_some() && offset.unwrap() == 0 { @@ -166,7 +172,11 @@ impl Element { *offset = Some(offset.unwrap() - 1); } } - subtrees.insert_temp_tree(path_vec.as_slice(), inner_merk, None); + if prefix.is_some() { + subtrees.insert_temp_tree_with_prefix(prefix.expect("confirmed as some"), inner_merk, None); + } else { + subtrees.insert_temp_tree(path_vec.as_slice(), inner_merk, None); + } } else { return Err(Error::InvalidPath( "you must provide a subquery or a subquery_key when interacting with a \ diff --git a/grovedb/src/subtrees.rs b/grovedb/src/subtrees.rs index 04582985a..97a67587d 100644 --- a/grovedb/src/subtrees.rs +++ b/grovedb/src/subtrees.rs @@ -48,20 +48,22 @@ impl Subtrees<'_> { &self, path: &[&[u8]], transaction: Option<&OptimisticTransactionDBTransaction>, - ) -> Result, Error> { + ) -> Result<(Merk, Option>), Error> { let merk; + let mut prefix: Option> = None; match transaction { None => { merk = self.get_subtree_without_transaction(path)?; } Some(_) => { - let prefix = &GroveDb::compress_subtree_key(path, None); - if self.temp_subtrees.borrow().contains_key(prefix) { + let tree_prefix = GroveDb::compress_subtree_key(path, None); + prefix = Some(tree_prefix.clone()); + if self.temp_subtrees.borrow().contains_key(&tree_prefix) { // get the merk out merk = self .temp_subtrees .borrow_mut() - .remove(prefix) + .remove(&tree_prefix) .expect("confirmed it's in the hashmap"); } else { // merk is not in the hash map get it without transaction @@ -69,14 +71,13 @@ impl Subtrees<'_> { } } } - Ok(merk) + Ok((merk, prefix)) } pub fn get_subtree_without_transaction( &self, path: &[&[u8]], ) -> Result, Error> { - let subtree_prefix = GroveDb::compress_subtree_key(path, None); let (subtree, has_keys) = self.get_subtree_with_key_info(path, None)?; if !has_keys { // if the subtree has no keys, it's either empty or invalid diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index 9c46d7650..efb05c014 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1020,7 +1020,7 @@ fn test_subtree_pairs_iterator() { // let mut iter = db // .elements_iterator(&[TEST_LEAF, b"subtree1"], None) // .expect("cannot create iterator"); - let merk = db + let (merk, _) = db .get_subtrees() .get(&[TEST_LEAF, b"subtree1"], None) .unwrap(); @@ -1145,7 +1145,7 @@ fn test_get_subtree() { // Retrieve subtree instance // Check if it returns the same instance that was inserted - let subtree = db + let (subtree, _) = db .get_subtrees() .get(&[TEST_LEAF, b"key1", b"key2"], None) .unwrap(); @@ -1174,7 +1174,7 @@ fn test_get_subtree() { .expect("successful value insert"); // Retrieve subtree instance with transaction - let subtree = db + let (subtree, _) = db .get_subtrees() .get(&[TEST_LEAF, b"key1", b"innertree"], Some(&transaction)) .unwrap(); @@ -1182,7 +1182,7 @@ fn test_get_subtree() { assert_eq!(result_element, Element::Item(b"ayy".to_vec())); // Should be able to retrieve instances created before transaction - let subtree = db + let (subtree, _) = db .get_subtrees() .get(&[TEST_LEAF, b"key1", b"key2"], None) .unwrap(); From fbe7b7f3b9961699bbbc43e5b87bcdcb6a370e5a Mon Sep 17 00:00:00 2001 From: pasta Date: Tue, 25 Jan 2022 17:36:09 +0700 Subject: [PATCH 28/31] feat: implement hashing with --- grovedb/Cargo.toml | 1 + grovedb/src/lib.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 78ba365e5..1cf5b1c4b 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -12,6 +12,7 @@ bincode = "1.3.3" serde = { version = "1.0.130", features = ["derive"] } storage = { path = "../storage" } hex = "0.4.3" +blake3 = "1.3.0" [dev-dependencies] rand = "0.8.4" diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 79ac325d7..a5e129917 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -18,6 +18,7 @@ use storage::{ }; pub use subtree::Element; use subtrees::Subtrees; +use blake3; // use crate::transaction::GroveDbTransaction; // pub use transaction::GroveDbTransaction; @@ -323,7 +324,7 @@ impl GroveDb { acc.extend(p.len().to_ne_bytes()); acc }); - res = Sha256::hash(&res).to_vec(); + res = blake3::hash(&res).as_bytes().to_vec(); res } From f7f22f439d6ad5c8381012c26050d7a0d991c728 Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 19:07:19 +0700 Subject: [PATCH 29/31] refactor: use let some to check for prefix --- grovedb/src/lib.rs | 10 +++++----- grovedb/src/operations/delete.rs | 4 ++-- grovedb/src/operations/get.rs | 8 ++++---- grovedb/src/operations/insert.rs | 12 ++++++------ grovedb/src/subtree.rs | 8 ++++---- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a5e129917..a646d3508 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -211,8 +211,8 @@ impl GroveDb { .get(&[subtree_path.as_slice()], transaction) .expect("`root_leaf_keys` must be in sync with `subtrees`"); leaf_hashes[*root_leaf_idx] = subtree_merk.root_hash(); - if prefix.is_some() { - subtrees.insert_temp_tree_with_prefix(prefix.expect("confirmed as some"), subtree_merk, transaction); + if let Some(prefix) = prefix { + subtrees.insert_temp_tree_with_prefix(prefix, subtree_merk, transaction); } else { subtrees.insert_temp_tree(&[subtree_path.as_slice()], subtree_merk, transaction); } @@ -258,9 +258,9 @@ impl GroveDb { // non root leaf node let (subtree, prefix) = subtrees.get(path, transaction)?; let element = Element::Tree(subtree.root_hash()); - if prefix.is_some() { + if let Some(prefix) = prefix { self.get_subtrees() - .insert_temp_tree_with_prefix(prefix.expect("confirmed as some"), subtree, transaction); + .insert_temp_tree_with_prefix(prefix, subtree, transaction); } else { self.get_subtrees() .insert_temp_tree(path, subtree, transaction); @@ -269,7 +269,7 @@ impl GroveDb { let (key, parent_path) = path.split_last().ok_or(Error::InvalidPath("empty path"))?; let (mut upper_tree, prefix) = subtrees.get(parent_path, transaction)?; element.insert(&mut upper_tree, key.to_vec(), transaction); - if prefix.is_some() { + if let Some(prefix) = prefix { self.get_subtrees() .insert_temp_tree(parent_path, upper_tree, transaction); } else { diff --git a/grovedb/src/operations/delete.rs b/grovedb/src/operations/delete.rs index 466d7ec2a..8d6efb471 100644 --- a/grovedb/src/operations/delete.rs +++ b/grovedb/src/operations/delete.rs @@ -26,9 +26,9 @@ impl GroveDb { Element::delete(&mut merk, key.clone(), transaction)?; // after deletion, if there is a transaction, add the merk back into the hashmap - if prefix.is_some(){ + if let Some(prefix) = prefix{ self.get_subtrees() - .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + .insert_temp_tree_with_prefix(prefix, merk, transaction); } else { self.get_subtrees() .insert_temp_tree(path, merk, transaction); diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index e7669b18b..37b22af42 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -77,9 +77,9 @@ impl GroveDb { let elem = Element::get(&merk, key); - if prefix.is_some() { + if let Some(prefix) = prefix { self.get_subtrees() - .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + .insert_temp_tree_with_prefix(prefix, merk, transaction); } else { self.get_subtrees() .insert_temp_tree(path, merk, transaction); @@ -173,8 +173,8 @@ impl GroveDb { let elem = Element::get_path_query(&merk, path_query, Some(&subtrees)); - if prefix.is_some(){ - subtrees.insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + if let Some(prefix) = prefix{ + subtrees.insert_temp_tree_with_prefix(prefix, merk, transaction); } else { subtrees.insert_temp_tree(path, merk, transaction); } diff --git a/grovedb/src/operations/insert.rs b/grovedb/src/operations/insert.rs index 35e252009..67c5bee97 100644 --- a/grovedb/src/operations/insert.rs +++ b/grovedb/src/operations/insert.rs @@ -55,9 +55,9 @@ impl GroveDb { .get(path, transaction) .map_err(|_| Error::InvalidPath("no subtree found under that path"))?; element.insert(&mut merk, key, transaction)?; - if prefix.is_some(){ + if let Some(prefix) = prefix{ self.get_subtrees() - .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + .insert_temp_tree_with_prefix(prefix, merk, transaction); } else { self.get_subtrees() .insert_temp_tree(path, merk, transaction); @@ -124,9 +124,9 @@ impl GroveDb { // First, check if a subtree exists to create a new subtree under it let (parent, prefix) = self.get_subtrees().get(path, transaction)?; - if prefix.is_some() { + if let Some(prefix) = prefix { self.get_subtrees() - .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), parent, transaction); + .insert_temp_tree_with_prefix(prefix, parent, transaction); } else { self.get_subtrees() .insert_temp_tree(path, parent, transaction); @@ -147,9 +147,9 @@ impl GroveDb { // need to mark key as taken in the upper tree element.insert(&mut merk, key, transaction)?; - if prefix.is_some(){ + if let Some(prefix) = prefix{ self.get_subtrees() - .insert_temp_tree_with_prefix(prefix.expect("confirmed it's some"), merk, transaction); + .insert_temp_tree_with_prefix(prefix, merk, transaction); } else { self.get_subtrees() .insert_temp_tree(path, merk, transaction); diff --git a/grovedb/src/subtree.rs b/grovedb/src/subtree.rs index ddfe41eea..9fb03ba4c 100644 --- a/grovedb/src/subtree.rs +++ b/grovedb/src/subtree.rs @@ -145,8 +145,8 @@ impl Element { let (mut sub_elements, skipped) = Element::get_path_query(&inner_merk, &inner_path_query, subtrees_option)?; - if prefix.is_some() { - subtrees.insert_temp_tree_with_prefix(prefix.expect("confirmed as some"), inner_merk, None); + if let Some(prefix) = prefix { + subtrees.insert_temp_tree_with_prefix(prefix, inner_merk, None); } else { subtrees.insert_temp_tree(path_vec.as_slice(), inner_merk, None); } @@ -172,8 +172,8 @@ impl Element { *offset = Some(offset.unwrap() - 1); } } - if prefix.is_some() { - subtrees.insert_temp_tree_with_prefix(prefix.expect("confirmed as some"), inner_merk, None); + if let Some(prefix) = prefix { + subtrees.insert_temp_tree_with_prefix(prefix, inner_merk, None); } else { subtrees.insert_temp_tree(path_vec.as_slice(), inner_merk, None); } From 8b1cdc6e36f5f6512837b1fde32b8d5bfc7465be Mon Sep 17 00:00:00 2001 From: Wisdom Ogwu Date: Tue, 25 Jan 2022 20:23:15 +0700 Subject: [PATCH 30/31] enhanced delete test, propagate changes initiates for leaf insert, get can retreive leaf elements --- grovedb/src/lib.rs | 3 +-- grovedb/src/operations/get.rs | 18 ++++++++++++++++-- grovedb/src/tests.rs | 6 ++---- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a646d3508..69cc55ab3 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -200,7 +200,6 @@ impl GroveDb { } fn build_root_tree( - // subtrees: &HashMap, Merk>, subtrees: &Subtrees, root_leaf_keys: &HashMap, usize>, transaction: Option<&OptimisticTransactionDBTransaction>, @@ -280,7 +279,7 @@ impl GroveDb { } // root leaf nodes - if path.len() == 1 { + if path.len() <= 1 { let root_leaf_keys = match transaction { None => &self.root_leaf_keys, Some(_) => &self.temp_root_leaf_keys, diff --git a/grovedb/src/operations/get.rs b/grovedb/src/operations/get.rs index 37b22af42..84248dac1 100644 --- a/grovedb/src/operations/get.rs +++ b/grovedb/src/operations/get.rs @@ -73,9 +73,23 @@ impl GroveDb { key: &[u8], transaction: Option<&OptimisticTransactionDBTransaction>, ) -> Result { - let (merk, prefix) = self.get_subtrees().get(path, transaction)?; + // If path is empty, then we need to combine the provided key and path + // then use this to get merk. + let merk_result; + if path.is_empty() { + merk_result = self.get_subtrees().get(&[key], transaction)?; + } else { + merk_result = self.get_subtrees().get(path, transaction)?; + } + + let (merk, prefix) = merk_result; - let elem = Element::get(&merk, key); + let elem; + if path.is_empty(){ + elem = Ok(Element::Tree(merk.root_hash())); + } else { + elem = Element::get(&merk, key); + } if let Some(prefix) = prefix { self.get_subtrees() diff --git a/grovedb/src/tests.rs b/grovedb/src/tests.rs index efb05c014..04358b56e 100644 --- a/grovedb/src/tests.rs +++ b/grovedb/src/tests.rs @@ -1057,16 +1057,12 @@ fn test_element_deletion() { db.insert(&[TEST_LEAF], b"key".to_vec(), element.clone(), None) .expect("successful insert"); let root_hash = db.root_tree.root().unwrap(); - dbg!("starting delete"); assert!(db.delete(&[TEST_LEAF], b"key".to_vec(), None).is_ok()); - dbg!("successful first delete"); assert!(matches!( db.get(&[TEST_LEAF], b"key", None), Err(Error::InvalidPathKey(_)) )); - dbg!("successful get"); assert_ne!(root_hash, db.root_tree.root().unwrap()); - dbg!("failed here"); } #[test] @@ -1224,6 +1220,8 @@ fn test_subtree_deletion() { )); // assert_eq!(db.subtrees.len(), 3); // TEST_LEAF, ANOTHER_TEST_LEAF // TEST_LEAF.key4 stay + assert!(db.get(&[], TEST_LEAF, None).is_ok()); + assert!(db.get(&[], ANOTHER_TEST_LEAF, None).is_ok()); assert!(db.get(&[TEST_LEAF], b"key4", None).is_ok()); assert_ne!(root_hash, db.root_tree.root().unwrap()); } From 63bc7a419211c1616ba034988122edee38749c33 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 26 Jan 2022 19:19:01 +0300 Subject: [PATCH 31/31] use early error throw for delete method --- grovedb/src/operations/delete.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/grovedb/src/operations/delete.rs b/grovedb/src/operations/delete.rs index 8d6efb471..6f80f8eee 100644 --- a/grovedb/src/operations/delete.rs +++ b/grovedb/src/operations/delete.rs @@ -26,7 +26,7 @@ impl GroveDb { Element::delete(&mut merk, key.clone(), transaction)?; // after deletion, if there is a transaction, add the merk back into the hashmap - if let Some(prefix) = prefix{ + if let Some(prefix) = prefix { self.get_subtrees() .insert_temp_tree_with_prefix(prefix, merk, transaction); } else { @@ -45,21 +45,12 @@ impl GroveDb { // TODO: eventually we need to do something about this nested slices let subtree_path_ref: Vec<&[u8]> = subtree_path.iter().map(|x| x.as_slice()).collect(); - let prefix = Self::compress_subtree_key(&subtree_path_ref, None); - let subtree = self + let mut subtree = self .get_subtrees() - .get_subtree_without_transaction(subtree_path_ref.as_slice()); - if subtree.is_ok() { - subtree - .expect("confirmed it's valid") - .clear(transaction) - .map_err(|e| { - Error::CorruptedData(format!( - "unable to cleanup tree from storage: {}", - e - )) - })?; - } + .get_subtree_without_transaction(subtree_path_ref.as_slice())?; + subtree.clear(transaction).map_err(|e| { + Error::CorruptedData(format!("unable to cleanup tree from storage: {}", e)) + })?; } } self.propagate_changes(path, transaction)?;