From fb148f348b4b729de231673b9e65088952725de6 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Mon, 5 Aug 2024 18:08:22 +0200 Subject: [PATCH 1/6] grovedbg fetch by query --- grovedb/src/debugger.rs | 69 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index e3556acd..aed1ab1d 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -23,12 +23,12 @@ use tower_http::services::ServeDir; use crate::{ operations::proof::{GroveDBProof, LayerProof, ProveOptions}, + query_result_type::{QueryResultElement, QueryResultElements, QueryResultType}, reference_path::ReferencePathType, GroveDb, }; -const GROVEDBG_ZIP: [u8; include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")).len()] = - *include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")); +const GROVEDBG_ZIP: &'static [u8] = include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")); pub(super) fn start_visualizer(grovedb: Weak, addr: A) where @@ -48,7 +48,8 @@ where let app = Router::new() .route("/fetch_node", post(fetch_node)) .route("/fetch_root_node", post(fetch_root_node)) - .route("/execute_path_query", post(execute_path_query)) + .route("/prove_path_query", post(prove_path_query)) + .route("/fetch_with_path_query", post(fetch_with_path_query)) .fallback_service(ServeDir::new(grovedbg_www)) .with_state((shutdown_send, grovedb)); @@ -136,7 +137,7 @@ async fn fetch_root_node( } } -async fn execute_path_query( +async fn prove_path_query( State((shutdown, grovedb)): State<(Sender<()>, Weak)>, Json(json_path_query): Json, ) -> Result, AppError> { @@ -153,6 +154,66 @@ async fn execute_path_query( Ok(Json(proof_to_grovedbg(grovedb_proof)?)) } +async fn fetch_with_path_query( + State((shutdown, grovedb)): State<(Sender<()>, Weak)>, + Json(json_path_query): Json, +) -> Result>, AppError> { + let Some(db) = grovedb.upgrade() else { + shutdown.send(()).await.ok(); + return Err(AppError::Closed); + }; + + let path_query = path_query_to_grovedb(json_path_query); + + let grovedb_query_result = db + .query_raw( + &path_query, + false, + true, + false, + QueryResultType::QueryPathKeyElementTrioResultType, + None, + GroveVersion::latest(), + ) + .unwrap()? + .0; + Ok(Json(query_result_to_grovedbg(&db, grovedb_query_result)?)) +} + +fn query_result_to_grovedbg( + db: &GroveDb, + query_result: QueryResultElements, +) -> Result, crate::Error> { + let mut result = Vec::new(); + + let mut last_merk: Option<(Vec>, grovedb_merk::Merk<_>)> = None; + + for qr in query_result.elements.into_iter() { + if let QueryResultElement::PathKeyElementTrioResultItem((path, key, _)) = qr { + let merk: &grovedb_merk::Merk<_> = match &mut last_merk { + Some((last_merk_path, last_merk)) if last_merk_path == &path => last_merk, + _ => { + last_merk = Some(( + path.clone(), + db.open_non_transactional_merk_at_path( + path.as_slice().into(), + None, + GroveVersion::latest(), + ) + .unwrap()?, + )); + &last_merk.as_ref().unwrap().1 + } + }; + + if let Some(node) = merk.get_node_dbg(&key)? { + result.push(node_to_update(path, node)?); + } + } + } + Ok(result) +} + fn proof_to_grovedbg(proof: GroveDBProof) -> Result { match proof { GroveDBProof::V0(p) => Ok(grovedbg_types::Proof { From a57d23de02c8d9ea7c6dd15d8ca70b33d73a50ee Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 14 Aug 2024 09:49:15 +0200 Subject: [PATCH 2/6] wip --- grovedb/src/debugger.rs | 20 +++++++++++++------- grovedbg-types/src/lib.rs | 4 ++++ merk/src/debugger.rs | 6 ++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index aed1ab1d..b17bda42 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -35,14 +35,16 @@ where A: ToSocketAddrs + Send + 'static, { std::thread::spawn(move || { - let grovedbg_tmp = - tempfile::tempdir().expect("cannot create tempdir for grovedbg contents"); - let grovedbg_zip = grovedbg_tmp.path().join("grovedbg.zip"); - let grovedbg_www = grovedbg_tmp.path().join("grovedbg_www"); + // let grovedbg_tmp = + // tempfile::tempdir().expect("cannot create tempdir for grovedbg contents"); + // let grovedbg_zip = grovedbg_tmp.path().join("grovedbg.zip"); + // let grovedbg_www = grovedbg_tmp.path().join("grovedbg_www"); - fs::write(&grovedbg_zip, &GROVEDBG_ZIP).expect("cannot crate grovedbg.zip"); - zip_extensions::read::zip_extract(&grovedbg_zip, &grovedbg_www) - .expect("cannot extract grovedbg contents"); + // fs::write(&grovedbg_zip, &GROVEDBG_ZIP).expect("cannot crate grovedbg.zip"); + // zip_extensions::read::zip_extract(&grovedbg_zip, &grovedbg_www) + // .expect("cannot extract grovedbg contents"); + + let grovedbg_www = "/home/yolo/dash/grovedbg/dist"; let (shutdown_send, mut shutdown_receive) = mpsc::channel::<()>(1); let app = Router::new() @@ -471,7 +473,9 @@ fn node_to_update( key, value, left_child, + left_merk_hash, right_child, + right_merk_hash, value_hash, kv_digest_hash, feature_type, @@ -487,7 +491,9 @@ fn node_to_update( key, element, left_child, + left_merk_hash, right_child, + right_merk_hash, feature_type: match feature_type { TreeFeatureType::BasicMerkNode => grovedbg_types::TreeFeatureType::BasicMerkNode, TreeFeatureType::SummedMerkNode(x) => { diff --git a/grovedbg-types/src/lib.rs b/grovedbg-types/src/lib.rs index 0019db09..ccb7367c 100644 --- a/grovedbg-types/src/lib.rs +++ b/grovedbg-types/src/lib.rs @@ -25,7 +25,11 @@ pub struct NodeUpdate { #[serde_as(as = "Option")] pub left_child: Option, #[serde_as(as = "Option")] + pub left_merk_hash: Option, + #[serde_as(as = "Option")] pub right_child: Option, + #[serde_as(as = "Option")] + pub right_merk_hash: Option, #[serde_as(as = "Vec")] pub path: Path, #[serde_as(as = "Base64")] diff --git a/merk/src/debugger.rs b/merk/src/debugger.rs index c806351d..7682e355 100644 --- a/merk/src/debugger.rs +++ b/merk/src/debugger.rs @@ -15,7 +15,9 @@ impl<'a, S: StorageContext<'a>> Merk { key: tree.inner.key_as_slice().to_owned(), value: tree.inner.value_as_slice().to_owned(), left_child: tree.link(true).map(|link| link.key().to_owned()), + left_merk_hash: tree.link(true).map(|link| link.hash().clone()), right_child: tree.link(false).map(|link| link.key().to_owned()), + right_merk_hash: tree.link(false).map(|link| link.hash().clone()), value_hash: *tree.inner.kv.value_hash(), kv_digest_hash: *tree.inner.kv.hash(), feature_type: tree.inner.kv.feature_type(), @@ -34,7 +36,9 @@ impl<'a, S: StorageContext<'a>> Merk { key: tree.inner.key_as_slice().to_owned(), value: tree.inner.value_as_slice().to_owned(), left_child: tree.link(true).map(|link| link.key().to_owned()), + left_merk_hash: tree.link(true).map(|link| link.hash().clone()), right_child: tree.link(false).map(|link| link.key().to_owned()), + right_merk_hash: tree.link(false).map(|link| link.hash().clone()), value_hash: *tree.inner.kv.value_hash(), kv_digest_hash: *tree.inner.kv.hash(), feature_type: tree.inner.kv.feature_type(), @@ -48,7 +52,9 @@ pub struct NodeDbg { pub key: Vec, pub value: Vec, pub left_child: Option>, + pub left_merk_hash: Option<[u8; 32]>, pub right_child: Option>, + pub right_merk_hash: Option<[u8; 32]>, pub value_hash: CryptoHash, pub kv_digest_hash: CryptoHash, pub feature_type: TreeFeatureType, From 043c452c719bc1678fd01f5b8a973161ba788542 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 21 Aug 2024 15:57:00 +0200 Subject: [PATCH 3/6] wip --- grovedbg-types/src/lib.rs | 56 ++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/grovedbg-types/src/lib.rs b/grovedbg-types/src/lib.rs index ccb7367c..f1cc53bd 100644 --- a/grovedbg-types/src/lib.rs +++ b/grovedbg-types/src/lib.rs @@ -44,31 +44,7 @@ pub struct NodeUpdate { #[serde_as] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum Element { - Subtree { - #[serde_as(as = "Option")] - root_key: Option, - #[serde_as(as = "Option")] - element_flags: Option>, - }, - Sumtree { - #[serde_as(as = "Option")] - root_key: Option, - sum: i64, - #[serde_as(as = "Option")] - element_flags: Option>, - }, - Item { - #[serde_as(as = "Base64")] - value: Vec, - #[serde_as(as = "Option")] - element_flags: Option>, - }, - SumItem { - value: i64, - #[serde_as(as = "Option")] - element_flags: Option>, - }, +pub enum Reference { AbsolutePathReference { #[serde_as(as = "Vec")] path: Path, @@ -116,6 +92,36 @@ pub enum Element { }, } +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum Element { + Subtree { + #[serde_as(as = "Option")] + root_key: Option, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + Sumtree { + #[serde_as(as = "Option")] + root_key: Option, + sum: i64, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + Item { + #[serde_as(as = "Base64")] + value: Vec, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + SumItem { + value: i64, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + Reference(Reference), +} + #[serde_as] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct PathQuery { From 7881b9a397497fc6140ebc456544dce1d1492d52 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 21 Aug 2024 15:59:34 +0200 Subject: [PATCH 4/6] wip --- grovedb/src/debugger.rs | 65 +++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index b17bda42..09c9c619 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -36,9 +36,10 @@ where { std::thread::spawn(move || { // let grovedbg_tmp = - // tempfile::tempdir().expect("cannot create tempdir for grovedbg contents"); - // let grovedbg_zip = grovedbg_tmp.path().join("grovedbg.zip"); - // let grovedbg_www = grovedbg_tmp.path().join("grovedbg_www"); + // tempfile::tempdir().expect("cannot create tempdir for grovedbg + // contents"); let grovedbg_zip = + // grovedbg_tmp.path().join("grovedbg.zip"); let grovedbg_www = + // grovedbg_tmp.path().join("grovedbg_www"); // fs::write(&grovedbg_zip, &GROVEDBG_ZIP).expect("cannot crate grovedbg.zip"); // zip_extensions::read::zip_extract(&grovedbg_zip, &grovedbg_www) @@ -397,19 +398,21 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::AbsolutePathReference(path), _, element_flags, - ) => grovedbg_types::Element::AbsolutePathReference { + ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::AbsolutePathReference { path, element_flags, - }, + }), crate::Element::Reference( ReferencePathType::UpstreamRootHeightReference(n_keep, path_append), _, element_flags, - ) => grovedbg_types::Element::UpstreamRootHeightReference { - n_keep: n_keep.into(), - path_append, - element_flags, - }, + ) => grovedbg_types::Element::Reference( + grovedbg_types::Reference::UpstreamRootHeightReference { + n_keep: n_keep.into(), + path_append, + element_flags, + }, + ), crate::Element::Reference( ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( n_keep, @@ -417,44 +420,50 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ), _, element_flags, - ) => grovedbg_types::Element::UpstreamRootHeightWithParentPathAdditionReference { - n_keep: n_keep.into(), - path_append, - element_flags, - }, + ) => grovedbg_types::Element::Reference( + grovedbg_types::Reference::UpstreamRootHeightWithParentPathAdditionReference { + n_keep: n_keep.into(), + path_append, + element_flags, + }, + ), crate::Element::Reference( ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), _, element_flags, - ) => grovedbg_types::Element::UpstreamFromElementHeightReference { - n_remove: n_remove.into(), - path_append, - element_flags, - }, + ) => grovedbg_types::Element::Reference( + grovedbg_types::Reference::UpstreamFromElementHeightReference { + n_remove: n_remove.into(), + path_append, + element_flags, + }, + ), crate::Element::Reference( ReferencePathType::CousinReference(swap_parent), _, element_flags, - ) => grovedbg_types::Element::CousinReference { + ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::CousinReference { swap_parent, element_flags, - }, + }), crate::Element::Reference( ReferencePathType::RemovedCousinReference(swap_parent), _, element_flags, - ) => grovedbg_types::Element::RemovedCousinReference { - swap_parent, - element_flags, - }, + ) => { + grovedbg_types::Element::Reference(grovedbg_types::Reference::RemovedCousinReference { + swap_parent, + element_flags, + }) + } crate::Element::Reference( ReferencePathType::SiblingReference(sibling_key), _, element_flags, - ) => grovedbg_types::Element::SiblingReference { + ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::SiblingReference { sibling_key, element_flags, - }, + }), crate::Element::SumItem(value, element_flags) => grovedbg_types::Element::SumItem { value, element_flags, From a2c9670109d5a62699e1a3c604f84d85d4bc99ba Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Thu, 22 Aug 2024 12:00:02 +0200 Subject: [PATCH 5/6] update grovedbg app --- grovedb/build.rs | 4 ++-- grovedb/src/debugger.rs | 19 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/grovedb/build.rs b/grovedb/build.rs index 83a38900..d8da9aa7 100644 --- a/grovedb/build.rs +++ b/grovedb/build.rs @@ -6,8 +6,8 @@ fn main() { use sha2::{digest::FixedOutput, Digest, Sha256}; const GROVEDBG_SHA256: [u8; 32] = - hex!("c6636f10b43c703128b621a7c4b94139a1a7d0a603e26fca1771734a7994bb7c"); - const GROVEDBG_VERSION: &str = "v1.0.0-rc.5"; + hex!("ea7d9258973aa765eaf5064451fc83efa22e0ce6eaf2938507e2703571364e35"); + const GROVEDBG_VERSION: &str = "v1.0.0-rc.6"; let out_dir = PathBuf::from(&env::var_os("OUT_DIR").unwrap()); let grovedbg_zip_path = out_dir.join("grovedbg.zip"); diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index 09c9c619..1e03bb43 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -35,17 +35,14 @@ where A: ToSocketAddrs + Send + 'static, { std::thread::spawn(move || { - // let grovedbg_tmp = - // tempfile::tempdir().expect("cannot create tempdir for grovedbg - // contents"); let grovedbg_zip = - // grovedbg_tmp.path().join("grovedbg.zip"); let grovedbg_www = - // grovedbg_tmp.path().join("grovedbg_www"); - - // fs::write(&grovedbg_zip, &GROVEDBG_ZIP).expect("cannot crate grovedbg.zip"); - // zip_extensions::read::zip_extract(&grovedbg_zip, &grovedbg_www) - // .expect("cannot extract grovedbg contents"); - - let grovedbg_www = "/home/yolo/dash/grovedbg/dist"; + let grovedbg_tmp = + tempfile::tempdir().expect("cannot create tempdir for grovedbg contents"); + let grovedbg_zip = grovedbg_tmp.path().join("grovedbg.zip"); + let grovedbg_www = grovedbg_tmp.path().join("grovedbg_www"); + + fs::write(&grovedbg_zip, &GROVEDBG_ZIP).expect("cannot crate grovedbg.zip"); + zip_extensions::read::zip_extract(&grovedbg_zip, &grovedbg_www) + .expect("cannot extract grovedbg contents"); let (shutdown_send, mut shutdown_receive) = mpsc::channel::<()>(1); let app = Router::new() From a42c37ba8c8b76e29b51eab405d93496bd64be62 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Fri, 23 Aug 2024 11:21:27 +0200 Subject: [PATCH 6/6] fix clippy warnings --- grovedb/src/debugger.rs | 4 ++-- merk/src/debugger.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index 1e03bb43..da4657ec 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -28,7 +28,7 @@ use crate::{ GroveDb, }; -const GROVEDBG_ZIP: &'static [u8] = include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")); +const GROVEDBG_ZIP: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")); pub(super) fn start_visualizer(grovedb: Weak, addr: A) where @@ -40,7 +40,7 @@ where let grovedbg_zip = grovedbg_tmp.path().join("grovedbg.zip"); let grovedbg_www = grovedbg_tmp.path().join("grovedbg_www"); - fs::write(&grovedbg_zip, &GROVEDBG_ZIP).expect("cannot crate grovedbg.zip"); + fs::write(&grovedbg_zip, GROVEDBG_ZIP).expect("cannot crate grovedbg.zip"); zip_extensions::read::zip_extract(&grovedbg_zip, &grovedbg_www) .expect("cannot extract grovedbg contents"); diff --git a/merk/src/debugger.rs b/merk/src/debugger.rs index 7682e355..5dc4710a 100644 --- a/merk/src/debugger.rs +++ b/merk/src/debugger.rs @@ -15,9 +15,9 @@ impl<'a, S: StorageContext<'a>> Merk { key: tree.inner.key_as_slice().to_owned(), value: tree.inner.value_as_slice().to_owned(), left_child: tree.link(true).map(|link| link.key().to_owned()), - left_merk_hash: tree.link(true).map(|link| link.hash().clone()), + left_merk_hash: tree.link(true).map(|link| *link.hash()), right_child: tree.link(false).map(|link| link.key().to_owned()), - right_merk_hash: tree.link(false).map(|link| link.hash().clone()), + right_merk_hash: tree.link(false).map(|link| *link.hash()), value_hash: *tree.inner.kv.value_hash(), kv_digest_hash: *tree.inner.kv.hash(), feature_type: tree.inner.kv.feature_type(), @@ -36,9 +36,9 @@ impl<'a, S: StorageContext<'a>> Merk { key: tree.inner.key_as_slice().to_owned(), value: tree.inner.value_as_slice().to_owned(), left_child: tree.link(true).map(|link| link.key().to_owned()), - left_merk_hash: tree.link(true).map(|link| link.hash().clone()), + left_merk_hash: tree.link(true).map(|link| *link.hash()), right_child: tree.link(false).map(|link| link.key().to_owned()), - right_merk_hash: tree.link(false).map(|link| link.hash().clone()), + right_merk_hash: tree.link(false).map(|link| *link.hash()), value_hash: *tree.inner.kv.value_hash(), kv_digest_hash: *tree.inner.kv.hash(), feature_type: tree.inner.kv.feature_type(),