diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index 094670ce..4fddc922 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -1142,4 +1142,355 @@ mod tests { verify_references(&db, &tx); } + + #[test] + fn test_one_update_bigger_sum_item_same_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let owner_id = [1; 32]; + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_sum_item_with_flags( + 1, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_sum_item_with_flags( + 100000000000, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::new_single_epoch(0, Some(owner_id)), + grove_version, + ); + + verify_references(&db, &tx); + } + + #[test] + fn test_one_update_bigger_sum_item_different_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_sum_item_with_flags( + 1, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let tx = db.start_transaction(); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_sum_item_with_flags( + 10000000000, + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), // no change + grove_version, + ); + + verify_references(&db, &tx); + } + + #[test] + fn test_one_update_bigger_item_add_flags() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), None), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let base_item = b"value1".to_vec(); + + for n in 1..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value100 if n was 2 + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + let expected_added_bytes = if n < 15 { + n as u32 + 3 + } else if n < 124 { + n as u32 + 4 // the varint requires an extra byte + } else { + n as u32 + 5 // the varint requires an extra byte + }; + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(1, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } + #[test] + fn test_one_update_smaller_item_add_flags() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + let base_item = b"value1".to_vec(); + let mut original_item = base_item.clone(); + original_item.extend(std::iter::repeat(0).take(150)); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(original_item, None), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let to = 150usize; + + for n in (0..to).rev() { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value1 if n was 1 + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + let storage_removed_bytes = apply_batch(&db, ops, &tx, grove_version) + .storage_cost + .removed_bytes; + + println!("{} {:?}", n, storage_removed_bytes); + + if n > 113 { + assert_eq!(storage_removed_bytes, StorageRemovedBytes::NoStorageRemoval); + } else if n > 17 { + let removed_bytes = 114 - n as u32; + assert_eq!( + storage_removed_bytes, + StorageRemovedBytes::BasicStorageRemoval(removed_bytes) + ); + } else { + let removed_bytes = 114 - n as u32 + 1; // because of varint + assert_eq!( + storage_removed_bytes, + StorageRemovedBytes::BasicStorageRemoval(removed_bytes) + ); + }; + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } } diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index f50b233e..7be6d220 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use grovedb_costs::{ - cost_return_on_error_no_add, + cost_return_on_error_default, cost_return_on_error_no_add, storage_cost::{ removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, StorageCost, @@ -14,9 +14,11 @@ use grovedb_merk::{ }; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; +use integer_encoding::VarInt; use crate::{ batch::{MerkError, TreeCacheMerkByPath}, + element::SUM_ITEM_COST_SIZE, Element, ElementFlags, Error, }; @@ -44,11 +46,39 @@ where u32, ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, { - let mut maybe_old_flags = old_element.get_flags_owned(); - let mut cost = OperationCost::default(); + if old_element.is_sum_item() { + return if new_element.is_sum_item() { + let mut maybe_old_flags = old_element.get_flags_owned(); + if maybe_old_flags.is_some() { + let mut updated_new_element_with_old_flags = new_element.clone(); + updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); + // There are no storage flags, we can just hash new element + let new_serialized_bytes = cost_return_on_error_no_add!( + &cost, + updated_new_element_with_old_flags.serialize(grove_version) + ); + let val_hash = value_hash(&new_serialized_bytes).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } else { + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } + } else { + Err(Error::NotSupported( + "going from a sum item to a not sum item is not supported".to_string(), + )) + .wrap_with_cost(cost) + }; + } else if new_element.is_sum_item() { + return Err(Error::NotSupported( + "going from an item to a sum item is not supported".to_string(), + )) + .wrap_with_cost(cost); + } + let mut maybe_old_flags = old_element.get_flags_owned(); - let old_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + let old_storage_cost = KV::node_value_byte_cost_size( key.len() as u32, old_serialized_element.len() as u32, is_in_sum_tree, @@ -58,21 +88,24 @@ where let mut serialization_to_use = Cow::Borrowed(serialized); - // we need to get the new storage_cost as if it had the same storage flags as - // before - let mut updated_new_element_with_old_flags = original_new_element.clone(); - updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); - - let serialized_with_old_flags = cost_return_on_error_no_add!( - &cost, - updated_new_element_with_old_flags.serialize(grove_version) - ); + let mut new_storage_cost = if maybe_old_flags.is_some() { + // we need to get the new storage_cost as if it had the same storage flags as + // before + let mut updated_new_element_with_old_flags = original_new_element.clone(); + updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); - let mut new_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( - key.len() as u32, - serialized_with_old_flags.len() as u32, - is_in_sum_tree, - ); + let serialized_with_old_flags = cost_return_on_error_no_add!( + &cost, + updated_new_element_with_old_flags.serialize(grove_version) + ); + KV::node_value_byte_cost_size( + key.len() as u32, + serialized_with_old_flags.len() as u32, + is_in_sum_tree, + ) + } else { + KV::node_value_byte_cost_size(key.len() as u32, serialized.len() as u32, is_in_sum_tree) + }; let mut i = 0; @@ -119,7 +152,7 @@ where new_element_cloned.serialize(grove_version) ); - new_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + new_storage_cost = KV::node_value_byte_cost_size( key.len() as u32, new_serialized_bytes.len() as u32, is_in_sum_tree, diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 2f55753f..822de9dc 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1601,13 +1601,18 @@ where let old_element = Element::deserialize(old_value.as_slice(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; let maybe_old_flags = old_element.get_flags_owned(); - let mut new_element = Element::deserialize(new_value.as_slice(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; - new_element.set_flags(maybe_old_flags); - new_element - .serialize(grove_version) - .map(Some) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + if maybe_old_flags.is_some() { + let mut new_element = + Element::deserialize(new_value.as_slice(), grove_version) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; + new_element.set_flags(maybe_old_flags); + new_element + .serialize(grove_version) + .map(Some) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + } else { + Ok(None) + } }, &mut |storage_costs, old_value, new_value| { // todo: change the flags without full deserialization diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 1fda91dd..6ec625e7 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -173,11 +173,7 @@ impl Element { }); let value_len = cost_size + flags_len; cost.storage_loaded_bytes = - KV::specialized_value_byte_cost_size_for_key_and_value_lengths( - key_ref.len() as u32, - value_len, - false, - ) + KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, false) } Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags)) => { let tree_cost_size = if element.as_ref().unwrap().is_sum_tree() { diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 86c523f2..fd0f17b1 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -297,17 +297,9 @@ impl Element { }); let value_len = SUM_ITEM_COST_SIZE + flags_len; let key_len = key.len() as u32; - KV::specialized_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_len, - is_sum_node, - ) + KV::node_value_byte_cost_size(key_len, value_len, is_sum_node) } - _ => KV::specialized_value_byte_cost_size_for_key_and_value_lengths( - key.len() as u32, - value.len() as u32, - is_sum_node, - ), + _ => KV::node_value_byte_cost_size(key.len() as u32, value.len() as u32, is_sum_node), }; Ok(cost) } diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 293c232d..8620cc97 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -45,8 +45,13 @@ impl TreeNode { // todo: clean up clones let original_new_value = self.value_ref().clone(); - let new_value_with_old_flags = - get_temp_new_value_with_old_flags(&old_value, &original_new_value)?; + let new_value_with_old_flags = if self.inner.kv.value_defined_cost.is_none() { + // for items + get_temp_new_value_with_old_flags(&old_value, &original_new_value)? + } else { + // don't do this for sum items or trees + None + }; let (mut current_tree_plus_hook_size, mut storage_costs) = self .kv_with_parent_hook_size_and_storage_cost_change_for_value( diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index 4fd3f7f7..f4a0e224 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -356,25 +356,6 @@ impl KV { node_value_size + parent_to_child_cost } - /// Get the costs for the node, this has the parent to child hooks - #[inline] - pub fn specialized_value_byte_cost_size_for_key_and_value_lengths( - not_prefixed_key_len: u32, - inner_value_len: u32, - is_sum_node: bool, - ) -> u32 { - // Sum trees are either 1 or 9 bytes. While they might be more or less on disk, - // costs can not take advantage of the varint aspect of the feature. - let feature_len = if is_sum_node { 9 } else { 1 }; - // Each node stores the key and value, and the node hash and the value hash - let node_value_size = inner_value_len + feature_len + HASH_LENGTH_U32_X2; - let node_value_size = node_value_size + node_value_size.required_space() as u32; - // The node will be a child of another node which stores it's key and hash - // That will be added during propagation - let parent_to_child_cost = Link::encoded_link_size(not_prefixed_key_len, is_sum_node); - node_value_size + parent_to_child_cost - } - /// Get the costs for the value with known value_len and non prefixed key /// len sizes, this has the parent to child hooks #[inline] @@ -452,11 +433,7 @@ impl KV { let key_len = self.key.len() as u32; let is_sum_node = self.feature_type.is_sum_feature(); - Self::specialized_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_cost, - is_sum_node, - ) + Self::node_value_byte_cost_size(key_len, value_cost, is_sum_node) } /// Costs based on predefined types (Trees, SumTrees, SumItems) that behave