Skip to content

Commit

Permalink
Improve verify grovedb + better debug statements
Browse files Browse the repository at this point in the history
  • Loading branch information
@iammadab
iammadab committed Sep 25, 2023
1 parent 8380363 commit f4eb8d7
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 31 deletions.
50 changes: 28 additions & 22 deletions grovedb/src/batch/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -371,32 +371,34 @@ impl fmt::Debug for GroveDbOp {

let op_dbg = match &self.op {
Op::Insert { element } => match element {
Element::Item(..) => "Insert Item",
Element::Reference(..) => "Insert Ref",
Element::Tree(..) => "Insert Tree",
Element::SumTree(..) => "Insert Sum Tree",
Element::SumItem(..) => "Insert Sum Item",
Element::Item(..) => "Insert Item".to_string(),
Element::Reference(..) => "Insert Ref".to_string(),
Element::Tree(..) => "Insert Tree".to_string(),
Element::SumTree(..) => "Insert Sum Tree".to_string(),
Element::SumItem(..) => "Insert Sum Item".to_string(),
},
Op::Replace { element } => match element {
Element::Item(..) => "Replace Item",
Element::Reference(..) => "Replace Ref",
Element::Tree(..) => "Replace Tree",
Element::SumTree(..) => "Replace Sum Tree",
Element::SumItem(..) => "Replace Sum Item",
Element::Item(..) => "Replace Item".to_string(),
Element::Reference(..) => "Replace Ref".to_string(),
Element::Tree(..) => "Replace Tree".to_string(),
Element::SumTree(..) => "Replace Sum Tree".to_string(),
Element::SumItem(..) => "Replace Sum Item".to_string(),
},
Op::Patch { element, .. } => match element {
Element::Item(..) => "Patch Item",
Element::Reference(..) => "Patch Ref",
Element::Tree(..) => "Patch Tree",
Element::SumTree(..) => "Patch Sum Tree",
Element::SumItem(..) => "Patch Sum Item",
Element::Item(..) => "Patch Item".to_string(),
Element::Reference(..) => "Patch Ref".to_string(),
Element::Tree(..) => "Patch Tree".to_string(),
Element::SumTree(..) => "Patch Sum Tree".to_string(),
Element::SumItem(..) => "Patch Sum Item".to_string(),
},
Op::RefreshReference { .. } => "Refresh Reference",
Op::Delete => "Delete",
Op::DeleteTree => "Delete Tree",
Op::DeleteSumTree => "Delete Sum Tree",
Op::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key",
Op::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key",
Op::RefreshReference { reference_path_type, max_reference_hop, trust_refresh_reference, .. } => {
format!("Refresh Reference: path {:?}, max_hop {:?}, trust_reference {} ", reference_path_type, max_reference_hop, trust_refresh_reference)
},
Op::Delete => "Delete".to_string(),
Op::DeleteTree => "Delete Tree".to_string(),
Op::DeleteSumTree => "Delete Sum Tree".to_string(),
Op::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key".to_string(),
Op::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key".to_string(),
};

f.debug_struct("GroveDbOp")
Expand Down Expand Up @@ -766,7 +768,7 @@ where
if recursions_allowed == 1 {
let referenced_element_value_hash_opt = cost_return_on_error!(
&mut cost,
merk.get_value_hash(key.as_ref(), true)
merk.get_value_hash(key.as_ref(), false)
.map_err(|e| Error::CorruptedData(e.to_string()))
);

Expand Down Expand Up @@ -922,6 +924,7 @@ where
trust_refresh_reference,
..
} => {
dbg!("In RefreshReference");
// We are pointing towards a reference that will be refreshed
let reference_info = if *trust_refresh_reference {
Some(reference_path_type)
Expand Down Expand Up @@ -1168,6 +1171,7 @@ where
)
.wrap_with_cost(OperationCost::default())
);
dbg!(&path_reference);
if path_reference.is_empty() {
return Err(Error::CorruptedReferencePathNotFound(
"attempting to refresh an empty reference".to_string(),
Expand Down Expand Up @@ -1399,6 +1403,8 @@ impl GroveDb {
} = batch_structure;
let mut current_level = last_level;

dbg!(&ops_by_level_paths);

let batch_apply_options = batch_apply_options.unwrap_or_default();
let stop_level = batch_apply_options.batch_pause_height.unwrap_or_default() as u32;

Expand Down
98 changes: 89 additions & 9 deletions grovedb/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -784,7 +784,7 @@ impl GroveDb {

/// Method to visualize hash mismatch after verification
pub fn visualize_verify_grovedb(&self) -> HashMap<String, (String, String, String)> {
self.verify_grovedb()
self.verify_grovedb(None)
.iter()
.map(|(path, (root_hash, expected, actual))| {
(
Expand All @@ -804,19 +804,27 @@ impl GroveDb {

/// Method to check that the value_hash of Element::Tree nodes are computed
/// correctly.
pub fn verify_grovedb(&self) -> HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)> {
let root_merk = self
.open_non_transactional_merk_at_path(SubtreePath::empty(), None)
.unwrap()
.expect("should exist");
self.verify_merk_and_submerks(root_merk, &SubtreePath::empty(), None)
pub fn verify_grovedb(&self, transaction: TransactionArg) -> HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)> {
if let Some(transaction) = transaction {
let root_merk = self
.open_transactional_merk_at_path(SubtreePath::empty(), transaction, None)
.unwrap()
.expect("should exist");
self.verify_merk_and_submerks_in_transaction(root_merk, &SubtreePath::empty(), None, transaction)
} else {
let root_merk = self
.open_non_transactional_merk_at_path(SubtreePath::empty(), None)
.unwrap()
.expect("should exist");
self.verify_merk_and_submerks(root_merk, &SubtreePath::empty(), None)
}
}

/// Verifies that the root hash of the given merk and all submerks match
/// those of the merk and submerks at the given path. Returns any issues.
fn verify_merk_and_submerks<'db, B: AsRef<[u8]>>(
fn verify_merk_and_submerks<'db, B: AsRef<[u8]>, S: StorageContext<'db>>(
&'db self,
merk: Merk<PrefixedRocksDbStorageContext>,
merk: Merk<S>,
path: &SubtreePath<B>,
batch: Option<&'db StorageBatch>,
) -> HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)> {
Expand Down Expand Up @@ -854,6 +862,78 @@ impl GroveDb {
);
}
issues.extend(self.verify_merk_and_submerks(inner_merk, &new_path_ref, batch));
} else if element.is_item() {
let (kv_value, element_value_hash) = merk
.get_value_and_value_hash(&key, true)
.unwrap()
.unwrap()
.unwrap();
let actual_value_hash = value_hash(&kv_value).unwrap();
if actual_value_hash != element_value_hash {
issues.insert(
path.derive_owned_with_child(key).to_vec(),
(actual_value_hash, element_value_hash, actual_value_hash)
);
}
}
}
issues
}

fn verify_merk_and_submerks_in_transaction<'db, B: AsRef<[u8]>, S: StorageContext<'db>>(
&'db self,
merk: Merk<S>,
path: &SubtreePath<B>,
batch: Option<&'db StorageBatch>,
transaction: &Transaction
) -> HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)> {
let mut all_query = Query::new();
all_query.insert_all();

let _in_sum_tree = merk.is_sum_tree;
let mut issues = HashMap::new();
let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap();

while let Some((key, element_value)) = element_iterator.next_kv().unwrap() {
let element = raw_decode(&element_value).unwrap();
if element.is_tree() {
let (kv_value, element_value_hash) = merk
.get_value_and_value_hash(&key, true)
.unwrap()
.unwrap()
.unwrap();
let new_path = path.derive_owned_with_child(key);
let new_path_ref = SubtreePath::from(&new_path);

let inner_merk = self
.open_transactional_merk_at_path(new_path_ref.clone(), transaction, batch)
.unwrap()
.expect("should exist");
let root_hash = inner_merk.root_hash().unwrap();

let actual_value_hash = value_hash(&kv_value).unwrap();
let combined_value_hash = combine_hash(&actual_value_hash, &root_hash).unwrap();

if combined_value_hash != element_value_hash {
issues.insert(
new_path.to_vec(),
(root_hash, combined_value_hash, element_value_hash),
);
}
issues.extend(self.verify_merk_and_submerks_in_transaction(inner_merk, &new_path_ref, batch, transaction));
} else if element.is_item() {
let (kv_value, element_value_hash) = merk
.get_value_and_value_hash(&key, true)
.unwrap()
.unwrap()
.unwrap();
let actual_value_hash = value_hash(&kv_value).unwrap();
if actual_value_hash != element_value_hash {
issues.insert(
path.derive_owned_with_child(key).to_vec(),
(actual_value_hash, element_value_hash, actual_value_hash)
);
}
}
}
issues
Expand Down

0 comments on commit f4eb8d7

Please sign in to comment.