Skip to content

Commit

Permalink
Merge pull request #143 from dashevo/feat/MerkBatches
Browse files Browse the repository at this point in the history 
feat: merk batches
  • Loading branch information
@QuantumExplorer
QuantumExplorer authored Jul 6, 2022
2 parents bdb18b7 + 867b91a commit dbbb807
Show file tree
Hide file tree
Showing 7 changed files with 201 additions and 62 deletions.
118 changes: 82 additions & 36 deletions grovedb/src/batch.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,13 @@ use core::fmt;
use std::{
borrow::Cow,
cmp::Ordering,
collections::{BTreeMap, HashMap, HashSet},
collections::{btree_map::Entry, BTreeMap, HashMap, HashSet},
hash::Hash,
};

use costs::{
cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost,
};
use indexmap::{map::Entry, IndexMap};
use merk::Merk;
use nohash_hasher::IntMap;
use storage::{Storage, StorageBatch, StorageContext};
Expand Down Expand Up @@ -284,7 +283,7 @@ trait TreeCache {
fn execute_ops_on_path(
&mut self,
path: &[Vec<u8>],
ops_at_path_by_key: IndexMap<Vec<u8>, Op>,
ops_at_path_by_key: BTreeMap<Vec<u8>, Op>,
ops_by_qualified_paths: &HashMap<Vec<Vec<u8>>, Op>,
batch_apply_options: &BatchApplyOptions,
) -> CostResult<[u8; 32], Error>;
Expand Down Expand Up @@ -399,7 +398,7 @@ where
fn execute_ops_on_path(
&mut self,
path: &[Vec<u8>],
ops_at_path_by_key: IndexMap<Vec<u8>, Op>,
ops_at_path_by_key: BTreeMap<Vec<u8>, Op>,
ops_by_qualified_paths: &HashMap<Vec<Vec<u8>>, Op>,
batch_apply_options: &BatchApplyOptions,
) -> CostResult<[u8; 32], Error> {
Expand All @@ -412,6 +411,7 @@ where
.unwrap_or_else(|| (self.get_merk_fn)(path));
let mut merk = cost_return_on_error!(&mut cost, merk_wrapped);

let mut batch_operations: Vec<(Vec<u8>, _)> = vec![];
for (key, op) in ops_at_path_by_key.into_iter() {
match op {
Op::Insert { element } => match &element {
Expand All @@ -435,14 +435,22 @@ where

cost_return_on_error!(
&mut cost,
element.insert_reference(&mut merk, key, serialized)
element.insert_reference_into_batch_operations(
key,
serialized,
&mut batch_operations
)
);
}
Element::Item(..) | Element::Tree(..) => {
if batch_apply_options.validate_insertion_does_not_override {
let inserted = cost_return_on_error!(
&mut cost,
element.insert_if_not_exists(&mut merk, key.as_slice())
element.insert_if_not_exists_into_batch_operations(
&mut merk,
key,
&mut batch_operations
)
);
if !inserted {
return Err(Error::InvalidBatchOperation(
Expand All @@ -451,21 +459,36 @@ where
.wrap_with_cost(cost);
}
} else {
cost_return_on_error!(&mut cost, element.insert(&mut merk, key));
cost_return_on_error!(
&mut cost,
element.insert_into_batch_operations(key, &mut batch_operations)
);
}
}
},
Op::Delete => {
cost_return_on_error!(&mut cost, Element::delete(&mut merk, key));
cost_return_on_error!(
&mut cost,
Element::delete_into_batch_operations(key, &mut batch_operations)
);
}
Op::ReplaceTreeHash { hash } => {
cost_return_on_error!(
&mut cost,
GroveDb::update_tree_item_preserve_flag(&mut merk, key.as_slice(), hash,)
GroveDb::update_tree_item_preserve_flag_into_batch_operations(
&merk,
key,
hash,
&mut batch_operations
)
);
}
}
}
cost_return_on_error!(&mut cost, unsafe {
merk.apply_unchecked::<_, Vec<u8>>(&batch_operations, &[])
.map_err(|e| Error::CorruptedData(e.to_string()))
});
merk.root_hash().add_cost(cost).map(Ok)
}
}
Expand All @@ -483,7 +506,7 @@ impl TreeCache for TreeCacheKnownPaths {
fn execute_ops_on_path(
&mut self,
path: &[Vec<u8>],
ops_at_path_by_key: IndexMap<Vec<u8>, Op>,
ops_at_path_by_key: BTreeMap<Vec<u8>, Op>,
_ops_by_qualified_paths: &HashMap<Vec<Vec<u8>>, Op>,
_batch_apply_options: &BatchApplyOptions,
) -> CostResult<[u8; 32], Error> {
Expand All @@ -502,7 +525,7 @@ impl TreeCache for TreeCacheKnownPaths {
}

/// LEVEL PATH KEY OP
type OpsByLevelPath = IntMap<usize, IndexMap<Vec<Vec<u8>>, IndexMap<Vec<u8>, Op>>>;
type OpsByLevelPath = IntMap<usize, BTreeMap<Vec<Vec<u8>>, BTreeMap<Vec<u8>, Op>>>;

struct BatchStructure<C> {
/// Operations by level path
Expand Down Expand Up @@ -548,7 +571,7 @@ where
ops: Vec<GroveDbOp>,
mut merk_tree_cache: C,
) -> CostResult<BatchStructure<C>, Error> {
let cost = OperationCost::default();
let mut cost = OperationCost::default();

let mut ops_by_level_paths: OpsByLevelPath = IntMap::default();
let mut current_last_level: usize = 0;
Expand All @@ -564,7 +587,7 @@ where
let op_result = match &op.op {
Op::Insert { element } => {
if let Element::Tree(..) = element {
merk_tree_cache.insert(&op);
cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op));
}
Ok(())
}
Expand All @@ -582,15 +605,15 @@ where
if let Some(ops_on_path) = ops_on_level.get_mut(op.path.as_slice()) {
ops_on_path.insert(op.key, op.op);
} else {
let mut ops_on_path: IndexMap<Vec<u8>, Op> = IndexMap::new();
let mut ops_on_path: BTreeMap<Vec<u8>, Op> = BTreeMap::new();
ops_on_path.insert(op.key, op.op);
ops_on_level.insert(op.path.clone(), ops_on_path);
}
} else {
let mut ops_on_path: IndexMap<Vec<u8>, Op> = IndexMap::new();
let mut ops_on_path: BTreeMap<Vec<u8>, Op> = BTreeMap::new();
ops_on_path.insert(op.key, op.op);
let mut ops_on_level: IndexMap<Vec<Vec<u8>>, IndexMap<Vec<u8>, Op>> =
IndexMap::new();
let mut ops_on_level: BTreeMap<Vec<Vec<u8>>, BTreeMap<Vec<u8>, Op>> =
BTreeMap::new();
ops_on_level.insert(op.path, ops_on_path);
ops_by_level_paths.insert(level, ops_on_level);
if current_last_level < level {
Expand Down Expand Up @@ -636,7 +659,7 @@ impl GroveDb {
while let Some(ops_at_level) = ops_by_level_paths.remove(&current_level) {
for (path, ops_at_path) in ops_at_level.into_iter() {
if current_level == 0 {
let mut root_tree_ops: IndexMap<Vec<u8>, Op> = IndexMap::new();
let mut root_tree_ops: BTreeMap<Vec<u8>, Op> = BTreeMap::new();
for (key, op) in ops_at_path.into_iter() {
match op {
Op::Insert { .. } => {
Expand All @@ -654,11 +677,14 @@ impl GroveDb {
}
}
// execute the ops at this path
merk_tree_cache.execute_ops_on_path(
&path,
root_tree_ops,
&ops_by_qualified_paths,
&batch_apply_options,
cost_return_on_error!(
&mut cost,
merk_tree_cache.execute_ops_on_path(
&path,
root_tree_ops,
&ops_by_qualified_paths,
&batch_apply_options,
)
);
} else {
let root_hash = cost_return_on_error!(
Expand Down Expand Up @@ -710,21 +736,21 @@ impl GroveDb {
}
}
} else {
let mut ops_on_path: IndexMap<Vec<u8>, Op> = IndexMap::new();
let mut ops_on_path: BTreeMap<Vec<u8>, Op> = BTreeMap::new();
ops_on_path.insert(
key.clone(),
Op::ReplaceTreeHash { hash: root_hash },
);
ops_at_level_above.insert(parent_path.to_vec(), ops_on_path);
}
} else {
let mut ops_on_path: IndexMap<Vec<u8>, Op> = IndexMap::new();
let mut ops_on_path: BTreeMap<Vec<u8>, Op> = BTreeMap::new();
ops_on_path
.insert(key.clone(), Op::ReplaceTreeHash { hash: root_hash });
let mut ops_on_level: IndexMap<
let mut ops_on_level: BTreeMap<
Vec<Vec<u8>>,
IndexMap<Vec<u8>, Op>,
> = IndexMap::new();
BTreeMap<Vec<u8>, Op>,
> = BTreeMap::new();
ops_on_level.insert(parent_path.to_vec(), ops_on_path);
ops_by_level_paths.insert(current_level - 1, ops_on_level);
}
Expand Down Expand Up @@ -1225,21 +1251,28 @@ mod tests {
grove_db_ops
}

// This test no longer works as of version 5, there might be support for it in
// the future
#[ignore]
#[test]
fn test_batch_produces_same_result() {
let db = make_grovedb();
let tx = db.start_transaction();

let ops = grove_db_ops_for_contract_insert();
db.apply_batch(ops, None, Some(&tx));
db.apply_batch(ops, None, Some(&tx))
.unwrap()
.expect("expected to apply batch");

db.root_hash(None).unwrap().expect("cannot get root hash");

let db = make_grovedb();
let tx = db.start_transaction();

let ops = grove_db_ops_for_contract_insert();
db.apply_batch(ops.clone(), None, Some(&tx));
db.apply_batch(ops.clone(), None, Some(&tx))
.unwrap()
.expect("expected to apply batch");

let batch_hash = db
.root_hash(Some(&tx))
Expand All @@ -1248,7 +1281,9 @@ mod tests {

db.rollback_transaction(&tx).expect("expected to rollback");

db.apply_operations_without_batching(ops, Some(&tx));
db.apply_operations_without_batching(ops, Some(&tx))
.unwrap()
.expect("expected to apply batch");

let no_batch_hash = db
.root_hash(Some(&tx))
Expand All @@ -1258,13 +1293,16 @@ mod tests {
assert_eq!(batch_hash, no_batch_hash);
}

#[ignore]
#[test]
fn test_batch_contract_with_document_produces_same_result() {
let db = make_grovedb();
let tx = db.start_transaction();

let ops = grove_db_ops_for_contract_insert();
db.apply_batch(ops, None, Some(&tx));
db.apply_batch(ops, None, Some(&tx))
.unwrap()
.expect("expected to apply batch");

db.root_hash(None).unwrap().expect("cannot get root hash");

Expand All @@ -1273,8 +1311,12 @@ mod tests {

let ops = grove_db_ops_for_contract_insert();
let document_ops = grove_db_ops_for_contract_document_insert();
db.apply_batch(ops.clone(), None, Some(&tx));
db.apply_batch(document_ops.clone(), None, Some(&tx));
db.apply_batch(ops.clone(), None, Some(&tx))
.unwrap()
.expect("expected to apply batch");
db.apply_batch(document_ops.clone(), None, Some(&tx))
.unwrap()
.expect("expected to apply batch");

let batch_hash = db
.root_hash(Some(&tx))
Expand All @@ -1283,8 +1325,12 @@ mod tests {

db.rollback_transaction(&tx).expect("expected to rollback");

db.apply_operations_without_batching(ops, Some(&tx));
db.apply_operations_without_batching(document_ops, Some(&tx));
db.apply_operations_without_batching(ops, Some(&tx))
.unwrap()
.expect("expected to apply batch");
db.apply_operations_without_batching(document_ops, Some(&tx))
.unwrap()
.expect("expected to apply batch");

let no_batch_hash = db
.root_hash(Some(&tx))
Expand Down
23 changes: 22 additions & 1 deletion grovedb/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use std::path::Path;

use costs::{cost_return_on_error, CostResult, CostsExt, OperationCost};
pub use merk::proofs::{query::QueryItem, Query};
use merk::{self, Merk};
use merk::{self, BatchEntry, Merk};
pub use query::{PathQuery, SizedQuery};
pub use storage::{
rocksdb_storage::{self, RocksDbStorage},
Expand Down Expand Up @@ -210,6 +210,27 @@ impl GroveDb {
})
}

pub(crate) fn update_tree_item_preserve_flag_into_batch_operations<
'db,
K: AsRef<[u8]>,
S: StorageContext<'db>,
>(
parent_tree: &Merk<S>,
key: K,
root_hash: [u8; 32],
batch_operations: &mut Vec<BatchEntry<K>>,
) -> CostResult<(), Error> {
Self::get_element_from_subtree(parent_tree, key.as_ref()).flat_map_ok(|element| {
if let Element::Tree(_, flag) = element {
let tree = Element::new_tree_with_flags(root_hash, flag);
tree.insert_into_batch_operations(key, batch_operations)
} else {
Err(Error::InvalidPath("can only propagate on tree items"))
.wrap_with_cost(Default::default())
}
})
}

fn get_element_from_subtree<'db, K: AsRef<[u8]>, S: StorageContext<'db>>(
subtree: &Merk<S>,
key: K,
Expand Down
Loading

0 comments on commit dbbb807

Please sign in to comment.