From 75d59799b6383bc3555a1e31258b42d59215b565 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Sat, 17 Feb 2024 08:17:49 +0100 Subject: [PATCH 1/8] commit: add specialized commit methods to CommitEngine --- commit_verify/derive/src/derive.rs | 8 +-- commit_verify/src/id.rs | 105 ++++++++++++++++++++++++----- commit_verify/src/lib.rs | 4 +- 3 files changed, 96 insertions(+), 21 deletions(-) diff --git a/commit_verify/derive/src/derive.rs b/commit_verify/derive/src/derive.rs index 56896a44..31cc8242 100644 --- a/commit_verify/derive/src/derive.rs +++ b/commit_verify/derive/src/derive.rs @@ -33,19 +33,19 @@ impl CommitDerive { let inner = match self.conf.strategy { StrategyAttr::Strict => quote! { - engine.commit_to(self); + engine.commit_to_serialized(self); }, StrategyAttr::ConcealStrict => quote! { use #trait_crate::Conceal; - engine.commit_to(&self.conceal()); + engine.commit_to_concealed(&self.conceal()); }, StrategyAttr::Transparent => quote! { use amplify::Wrapper; - engine.commit_to(self.as_inner()); + engine.commit_to_serialized(self.as_inner()); }, StrategyAttr::Merklize => quote! { use amplify::Wrapper; - engine.commit_to(self.as_inner().merklize()); + engine.commit_to_merkle(self.as_inner().merklize()); }, }; diff --git a/commit_verify/src/id.rs b/commit_verify/src/id.rs index 56b9c374..c5e52d4d 100644 --- a/commit_verify/src/id.rs +++ b/commit_verify/src/id.rs @@ -19,21 +19,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -use amplify::confinement::U64 as U64MAX; +use std::hash::Hash; + +use amplify::confinement::{Collection, Confined, U64 as U64MAX}; use amplify::Bytes32; use sha2::Sha256; -use strict_encoding::{StreamWriter, StrictEncode, StrictType}; +use strict_encoding::{StreamWriter, StrictDumb, StrictEncode, StrictType}; use strict_types::typesys::TypeFqn; -use crate::{DigestExt, LIB_NAME_COMMIT_VERIFY}; +use crate::{Conceal, DigestExt, MerkleHash, MerkleLeaves, LIB_NAME_COMMIT_VERIFY}; const COMMIT_MAX_LEN: usize = U64MAX; -#[derive(Debug)] +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub enum CommitStep { + Serialized(TypeFqn), + Collection, + Hashed(TypeFqn), + Merklized(TypeFqn), + Concealed(TypeFqn), +} + +#[derive(Clone, Debug)] pub struct CommitEngine { finished: bool, hasher: Sha256, - layout: Vec, + layout: Vec, +} + +fn commitment_fqn() -> TypeFqn { + TypeFqn::with( + libname!(T::STRICT_LIB_NAME), + T::strict_name().expect("commit encoder can commit only to named types"), + ) } impl CommitEngine { @@ -45,30 +63,85 @@ impl CommitEngine { } } - pub fn commit_to(&mut self, value: &T) { + fn inner_commit_to(&mut self, value: &T) { debug_assert!(!self.finished); - let writer = StreamWriter::new::(&mut self.hasher); + let writer = StreamWriter::new::(&mut self.hasher); let ok = value.strict_write(writer).is_ok(); - let fqn = TypeFqn::with( - libname!(T::STRICT_LIB_NAME), - T::strict_name().expect("commit encoder can commit only to named types"), - ); - self.layout.push(fqn); debug_assert!(ok); } - pub fn as_layout(&mut self) -> &[TypeFqn] { + pub fn commit_to_serialized(&mut self, value: &T) { + let fqn = commitment_fqn::(); + debug_assert!( + Some(&fqn.name) != MerkleHash::strict_name().as_ref() || + fqn.lib.as_str() != MerkleHash::STRICT_LIB_NAME, + "do not use commit_to_serialized for merklized collections, use commit_to_merkle \ + instead" + ); + debug_assert!( + Some(&fqn.name) != StrictHash::strict_name().as_ref() || + fqn.lib.as_str() != StrictHash::STRICT_LIB_NAME, + "do not use commit_to_serialized for StrictHash types, use commit_to_hash instead" + ); + self.layout.push(CommitStep::Serialized(fqn)); + + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&value); + } + + pub fn commit_to_hash + StrictType>( + &mut self, + value: T, + ) { + let fqn = commitment_fqn::(); + self.layout.push(CommitStep::Hashed(fqn)); + + self.inner_commit_to::<_, 32>(&value.commit_id()); + } + + pub fn commit_to_merkle(&mut self, value: &T) + where T::Leaf: StrictType { + let fqn = commitment_fqn::(); + self.layout.push(CommitStep::Merklized(fqn)); + + let root = MerkleHash::merklize(value); + self.inner_commit_to::<_, 32>(&root); + } + + pub fn commit_to_concealed(&mut self, value: &T) + where + T: StrictType, + T::Concealed: StrictEncode, + { + let fqn = commitment_fqn::(); + self.layout.push(CommitStep::Concealed(fqn)); + + let concealed = value.conceal(); + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&concealed); + } + + pub fn commit_to_collection( + &mut self, + collection: &Confined, + ) where + C: Collection, + Confined: StrictEncode, + { + self.layout.push(CommitStep::Collection); + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&collection); + } + + pub fn as_layout(&mut self) -> &[CommitStep] { self.finished = true; self.layout.as_ref() } - pub fn into_layout(self) -> Vec { self.layout } + pub fn into_layout(self) -> Vec { self.layout } pub fn set_finished(&mut self) { self.finished = true; } pub fn finish(self) -> Sha256 { self.hasher } - pub fn finish_layout(self) -> (Sha256, Vec) { (self.hasher, self.layout) } + pub fn finish_layout(self) -> (Sha256, Vec) { (self.hasher, self.layout) } } /// Prepares the data to the *consensus commit* procedure by first running @@ -87,7 +160,7 @@ pub trait CommitEncode { pub struct CommitmentLayout { ty: TypeFqn, tag: &'static str, - fields: Vec, + fields: Vec, } pub trait CommitmentId: Copy + Ord + From + StrictType { diff --git a/commit_verify/src/lib.rs b/commit_verify/src/lib.rs index 0e0d9945..121071ea 100644 --- a/commit_verify/src/lib.rs +++ b/commit_verify/src/lib.rs @@ -62,7 +62,9 @@ pub use conceal::Conceal; pub use convolve::{ConvolveCommit, ConvolveCommitProof, ConvolveVerifyError}; pub use digest::{Digest, DigestExt, Ripemd160, Sha256}; pub use embed::{EmbedCommitProof, EmbedCommitVerify, EmbedVerifyError, VerifyEq}; -pub use id::{CommitEncode, CommitEngine, CommitId, CommitmentId, CommitmentLayout, StrictHash}; +pub use id::{ + CommitEncode, CommitEngine, CommitId, CommitStep, CommitmentId, CommitmentLayout, StrictHash, +}; pub use merkle::{MerkleBuoy, MerkleHash, MerkleLeaves, MerkleNode, NodeBranching}; pub const LIB_NAME_COMMIT_VERIFY: &str = "CommitVerify"; From ec95f742c94612836e1e20698215bf74bee50201 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Sun, 18 Feb 2024 18:54:47 +0100 Subject: [PATCH 2/8] commit: add vesper translator --- Cargo.lock | 1 + commit_verify/Cargo.toml | 1 + commit_verify/src/id.rs | 96 +++++++++++++++---- commit_verify/src/lib.rs | 4 +- commit_verify/src/vesper.rs | 185 ++++++++++++++++++++++++++++++++++++ 5 files changed, 266 insertions(+), 21 deletions(-) create mode 100644 commit_verify/src/vesper.rs diff --git a/Cargo.lock b/Cargo.lock index 13ca53e8..338d9126 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,7 @@ dependencies = [ "sha2", "strict_encoding", "strict_types", + "vesper-lang", ] [[package]] diff --git a/commit_verify/Cargo.toml b/commit_verify/Cargo.toml index b3bf4203..d142ca37 100644 --- a/commit_verify/Cargo.toml +++ b/commit_verify/Cargo.toml @@ -25,6 +25,7 @@ required-features = ["stl"] amplify = { workspace = true, features = ["hex", "apfloat"] } strict_encoding = { workspace = true } strict_types = { workspace = true } +vesper-lang = "0.1.0" commit_encoding_derive = { version = "0.11.0-beta.3", path = "derive" } sha2 = "0.10.8" ripemd = "0.1.3" diff --git a/commit_verify/src/id.rs b/commit_verify/src/id.rs index c5e52d4d..b66b1b4a 100644 --- a/commit_verify/src/id.rs +++ b/commit_verify/src/id.rs @@ -19,22 +19,30 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::{BTreeMap, BTreeSet}; use std::hash::Hash; -use amplify::confinement::{Collection, Confined, U64 as U64MAX}; +use amplify::confinement::{Confined, TinyVec, U64 as U64MAX}; use amplify::Bytes32; use sha2::Sha256; -use strict_encoding::{StreamWriter, StrictDumb, StrictEncode, StrictType}; +use strict_encoding::{Sizing, StreamWriter, StrictDumb, StrictEncode, StrictType}; use strict_types::typesys::TypeFqn; use crate::{Conceal, DigestExt, MerkleHash, MerkleLeaves, LIB_NAME_COMMIT_VERIFY}; const COMMIT_MAX_LEN: usize = U64MAX; +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub enum CommitColType { + List, + Set, + Map { key: TypeFqn }, +} + #[derive(Clone, Eq, PartialEq, Hash, Debug)] pub enum CommitStep { Serialized(TypeFqn), - Collection, + Collection(CommitColType, Sizing, TypeFqn), Hashed(TypeFqn), Merklized(TypeFqn), Concealed(TypeFqn), @@ -44,7 +52,7 @@ pub enum CommitStep { pub struct CommitEngine { finished: bool, hasher: Sha256, - layout: Vec, + layout: TinyVec, } fn commitment_fqn() -> TypeFqn { @@ -59,7 +67,7 @@ impl CommitEngine { Self { finished: false, hasher: Sha256::from_tag(tag), - layout: vec![], + layout: empty!(), } } @@ -83,7 +91,9 @@ impl CommitEngine { fqn.lib.as_str() != StrictHash::STRICT_LIB_NAME, "do not use commit_to_serialized for StrictHash types, use commit_to_hash instead" ); - self.layout.push(CommitStep::Serialized(fqn)); + self.layout + .push(CommitStep::Serialized(fqn)) + .expect("too many fields for commitment"); self.inner_commit_to::<_, COMMIT_MAX_LEN>(&value); } @@ -93,7 +103,9 @@ impl CommitEngine { value: T, ) { let fqn = commitment_fqn::(); - self.layout.push(CommitStep::Hashed(fqn)); + self.layout + .push(CommitStep::Hashed(fqn)) + .expect("too many fields for commitment"); self.inner_commit_to::<_, 32>(&value.commit_id()); } @@ -101,7 +113,9 @@ impl CommitEngine { pub fn commit_to_merkle(&mut self, value: &T) where T::Leaf: StrictType { let fqn = commitment_fqn::(); - self.layout.push(CommitStep::Merklized(fqn)); + self.layout + .push(CommitStep::Merklized(fqn)) + .expect("too many fields for commitment"); let root = MerkleHash::merklize(value); self.inner_commit_to::<_, 32>(&root); @@ -113,20 +127,61 @@ impl CommitEngine { T::Concealed: StrictEncode, { let fqn = commitment_fqn::(); - self.layout.push(CommitStep::Concealed(fqn)); + self.layout + .push(CommitStep::Concealed(fqn)) + .expect("too many fields for commitment"); let concealed = value.conceal(); self.inner_commit_to::<_, COMMIT_MAX_LEN>(&concealed); } - pub fn commit_to_collection( + pub fn commit_to_list( + &mut self, + collection: &Confined, MIN, MAX>, + ) where + T: StrictEncode + StrictDumb, + { + let fqn = commitment_fqn::(); + let step = + CommitStep::Collection(CommitColType::List, Sizing::new(MIN as u64, MAX as u64), fqn); + self.layout + .push(step) + .expect("too many fields for commitment"); + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&collection); + } + + pub fn commit_to_set( &mut self, - collection: &Confined, + collection: &Confined, MIN, MAX>, ) where - C: Collection, - Confined: StrictEncode, + T: Ord + StrictEncode + StrictDumb, { - self.layout.push(CommitStep::Collection); + let fqn = commitment_fqn::(); + let step = + CommitStep::Collection(CommitColType::Set, Sizing::new(MIN as u64, MAX as u64), fqn); + self.layout + .push(step) + .expect("too many fields for commitment"); + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&collection); + } + + pub fn commit_to_map( + &mut self, + collection: &Confined, MIN, MAX>, + ) where + K: Ord + Hash + StrictEncode + StrictDumb, + V: StrictEncode + StrictDumb, + { + let key_fqn = commitment_fqn::(); + let val_fqn = commitment_fqn::(); + let step = CommitStep::Collection( + CommitColType::Map { key: key_fqn }, + Sizing::new(MIN as u64, MAX as u64), + val_fqn, + ); + self.layout + .push(step) + .expect("too many fields for commitment"); self.inner_commit_to::<_, COMMIT_MAX_LEN>(&collection); } @@ -135,13 +190,13 @@ impl CommitEngine { self.layout.as_ref() } - pub fn into_layout(self) -> Vec { self.layout } + pub fn into_layout(self) -> TinyVec { self.layout } pub fn set_finished(&mut self) { self.finished = true; } pub fn finish(self) -> Sha256 { self.hasher } - pub fn finish_layout(self) -> (Sha256, Vec) { (self.hasher, self.layout) } + pub fn finish_layout(self) -> (Sha256, TinyVec) { (self.hasher, self.layout) } } /// Prepares the data to the *consensus commit* procedure by first running @@ -152,15 +207,16 @@ pub trait CommitEncode { type CommitmentId: CommitmentId; /// Encodes the data for the commitment by writing them directly into a - /// [`io::Write`] writer instance + /// [`std::io::Write`] writer instance fn commit_encode(&self, e: &mut CommitEngine); } #[derive(Getters, Clone, Eq, PartialEq, Hash, Debug)] pub struct CommitmentLayout { - ty: TypeFqn, + idty: TypeFqn, + #[getter(as_copy)] tag: &'static str, - fields: Vec, + fields: TinyVec, } pub trait CommitmentId: Copy + Ord + From + StrictType { @@ -197,7 +253,7 @@ impl CommitId for T { fn commitment_layout(&self) -> CommitmentLayout { let fields = self.commit().into_layout(); CommitmentLayout { - ty: TypeFqn::with( + idty: TypeFqn::with( libname!(Self::CommitmentId::STRICT_LIB_NAME), Self::CommitmentId::strict_name() .expect("commitment types must have explicit type name"), diff --git a/commit_verify/src/lib.rs b/commit_verify/src/lib.rs index 121071ea..d3c617b4 100644 --- a/commit_verify/src/lib.rs +++ b/commit_verify/src/lib.rs @@ -56,6 +56,7 @@ pub mod stl; pub mod merkle; pub mod mpc; mod digest; +pub mod vesper; pub use commit::{CommitVerify, TryCommitVerify, VerifyError}; pub use conceal::Conceal; @@ -63,7 +64,8 @@ pub use convolve::{ConvolveCommit, ConvolveCommitProof, ConvolveVerifyError}; pub use digest::{Digest, DigestExt, Ripemd160, Sha256}; pub use embed::{EmbedCommitProof, EmbedCommitVerify, EmbedVerifyError, VerifyEq}; pub use id::{ - CommitEncode, CommitEngine, CommitId, CommitStep, CommitmentId, CommitmentLayout, StrictHash, + CommitColType, CommitEncode, CommitEngine, CommitId, CommitStep, CommitmentId, + CommitmentLayout, StrictHash, }; pub use merkle::{MerkleBuoy, MerkleHash, MerkleLeaves, MerkleNode, NodeBranching}; diff --git a/commit_verify/src/vesper.rs b/commit_verify/src/vesper.rs new file mode 100644 index 00000000..c9f6ce4b --- /dev/null +++ b/commit_verify/src/vesper.rs @@ -0,0 +1,185 @@ +// Client-side-validation foundation libraries. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Written in 2024 by +// Dr. Maxim Orlovsky +// +// Copyright (C) 2024 LNP/BP Standards Association. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use amplify::confinement::{Confined, SmallVec, TinyVec}; +use strict_encoding::Ident; +use strict_types::layout::vesper::LenRange; +use strict_types::typesys::TypeFqn; +use vesper::{AttrVal, Attribute, Expression, Predicate, TExpr}; + +use crate::{CommitColType, CommitStep, CommitmentLayout}; + +pub type VesperCommit = TExpr; + +#[derive(Clone, Eq, PartialEq, Hash, Debug, Display)] +#[display(lowercase)] +pub enum Pred { + Commitment, + Serialize, + Hash, + Merklize, + Conceal, + List, + Set, + Element, + Map, + #[display("mapKey")] + MapKey, + #[display("mapValue")] + MapValue, +} + +impl Predicate for Pred { + type Attr = Attr; +} + +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub enum Attr { + Tagged(&'static str), + Concealed(TypeFqn), + LenRange(LenRange), +} +#[derive(Clone, Eq, PartialEq, Hash, Debug, Display)] +#[display(inner)] +pub enum AttrExpr { + Tag(&'static str), + LenRange(LenRange), +} + +impl Expression for AttrExpr {} + +impl Attribute for Attr { + type Expression = AttrExpr; + + fn name(&self) -> Option { + match self { + Attr::Tagged(_) => Some(tn!("tagged")), + Attr::Concealed(_) => Some(tn!("concealed")), + Attr::LenRange(_) => Some(tn!("len")), + } + } + + fn value(&self) -> AttrVal { + match self { + Attr::Tagged(tag) => AttrVal::Expr(AttrExpr::Tag(tag)), + Attr::Concealed(fqn) => AttrVal::Ident(fqn.name.to_ident()), + Attr::LenRange(range) => AttrVal::Expr(AttrExpr::LenRange(range.clone())), + } + } +} + +impl CommitStep { + fn subject(&self) -> Ident { + match self { + CommitStep::Serialized(fqn) => fqn, + CommitStep::Collection(_, _, fqn) => fqn, + CommitStep::Hashed(fqn) => fqn, + CommitStep::Merklized(fqn) => fqn, + CommitStep::Concealed(fqn) => fqn, + } + .name + .to_ident() + } + + fn predicate(&self) -> Pred { + match self { + CommitStep::Serialized(_) => Pred::Serialize, + CommitStep::Collection(CommitColType::List, _, _) => Pred::List, + CommitStep::Collection(CommitColType::Set, _, _) => Pred::Set, + CommitStep::Collection(CommitColType::Map { .. }, _, _) => Pred::Map, + CommitStep::Hashed(_) => Pred::Hash, + CommitStep::Merklized(_) => Pred::Merklize, + CommitStep::Concealed(_) => Pred::Conceal, + } + } + + fn attributes(&self) -> SmallVec { + match self { + CommitStep::Collection(_, sizing, _) => small_vec![Attr::LenRange((*sizing).into())], + CommitStep::Concealed(from) => small_vec![Attr::Concealed(from.clone())], + CommitStep::Serialized(_) | CommitStep::Hashed(_) | CommitStep::Merklized(_) => none!(), + } + } + + fn content(&self) -> TinyVec> { + match self { + CommitStep::Collection(CommitColType::List, _, val) | + CommitStep::Collection(CommitColType::Set, _, val) => { + tiny_vec![Box::new(VesperCommit { + subject: val.name.to_ident(), + predicate: Pred::Element, + attributes: none!(), + content: none!(), + comment: None + })] + } + CommitStep::Collection(CommitColType::Map { key }, _, val) => { + tiny_vec![ + Box::new(VesperCommit { + subject: key.name.to_ident(), + predicate: Pred::MapKey, + attributes: none!(), + content: none!(), + comment: None + }), + Box::new(VesperCommit { + subject: val.name.to_ident(), + predicate: Pred::MapValue, + attributes: none!(), + content: none!(), + comment: None + }) + ] + } + CommitStep::Serialized(_) | + CommitStep::Hashed(_) | + CommitStep::Merklized(_) | + CommitStep::Concealed(_) => empty!(), + } + } +} + +impl CommitmentLayout { + pub fn to_vesper(&self) -> VesperCommit { + let subject = self.idty().name.to_ident(); + + // SecretSeal commitment tagged="" + // BlindSeal rec serialized + + let content = self.fields().iter().map(|field| { + Box::new(VesperCommit { + subject: field.subject(), + predicate: field.predicate(), + attributes: field.attributes(), + content: field.content(), + comment: None, + }) + }); + + VesperCommit { + subject, + predicate: Pred::Commitment, + attributes: confined_vec![Attr::Tagged(self.tag())], + content: Confined::from_iter_unsafe(content), + comment: None, + } + } +} From 2b822b93e4088a36ca3fc23e5034c07452f322ba Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Sun, 18 Feb 2024 20:08:03 +0100 Subject: [PATCH 3/8] commit: refactor CommitLayout --- commit_verify/src/id.rs | 46 ++++++++++++++++++++++++------------- commit_verify/src/lib.rs | 2 +- commit_verify/src/vesper.rs | 4 ++-- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/commit_verify/src/id.rs b/commit_verify/src/id.rs index b66b1b4a..58cc2855 100644 --- a/commit_verify/src/id.rs +++ b/commit_verify/src/id.rs @@ -20,6 +20,7 @@ // limitations under the License. use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::{self, Display, Formatter}; use std::hash::Hash; use amplify::confinement::{Confined, TinyVec, U64 as U64MAX}; @@ -212,17 +213,45 @@ pub trait CommitEncode { } #[derive(Getters, Clone, Eq, PartialEq, Hash, Debug)] -pub struct CommitmentLayout { +pub struct CommitLayout { idty: TypeFqn, #[getter(as_copy)] tag: &'static str, fields: TinyVec, } +impl Display for CommitLayout { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(&self.to_vesper().display(), f) + } +} + pub trait CommitmentId: Copy + Ord + From + StrictType { const TAG: &'static str; } +pub trait CommitmentLayout: CommitEncode { + fn commitment_layout() -> CommitLayout; +} + +impl CommitmentLayout for T +where T: CommitEncode + StrictDumb +{ + fn commitment_layout() -> CommitLayout { + let dumb = Self::strict_dumb(); + let fields = dumb.commit().into_layout(); + CommitLayout { + idty: TypeFqn::with( + libname!(Self::CommitmentId::STRICT_LIB_NAME), + Self::CommitmentId::strict_name() + .expect("commitment types must have explicit type name"), + ), + tag: T::CommitmentId::TAG, + fields, + } + } +} + /// High-level API used in client-side validation for producing a single /// commitment to the data, which includes running all necessary procedures like /// concealment with [`crate::Conceal`], merklization, strict encoding, @@ -236,8 +265,6 @@ pub trait CommitId: CommitEncode { #[doc = hidden] fn commit(&self) -> CommitEngine; - fn commitment_layout(&self) -> CommitmentLayout; - /// Performs commitment to client-side-validated data fn commit_id(&self) -> Self::CommitmentId; } @@ -250,19 +277,6 @@ impl CommitId for T { engine } - fn commitment_layout(&self) -> CommitmentLayout { - let fields = self.commit().into_layout(); - CommitmentLayout { - idty: TypeFqn::with( - libname!(Self::CommitmentId::STRICT_LIB_NAME), - Self::CommitmentId::strict_name() - .expect("commitment types must have explicit type name"), - ), - tag: T::CommitmentId::TAG, - fields, - } - } - fn commit_id(&self) -> Self::CommitmentId { self.commit().finish().into() } } diff --git a/commit_verify/src/lib.rs b/commit_verify/src/lib.rs index d3c617b4..6e9d0e2c 100644 --- a/commit_verify/src/lib.rs +++ b/commit_verify/src/lib.rs @@ -64,7 +64,7 @@ pub use convolve::{ConvolveCommit, ConvolveCommitProof, ConvolveVerifyError}; pub use digest::{Digest, DigestExt, Ripemd160, Sha256}; pub use embed::{EmbedCommitProof, EmbedCommitVerify, EmbedVerifyError, VerifyEq}; pub use id::{ - CommitColType, CommitEncode, CommitEngine, CommitId, CommitStep, CommitmentId, + CommitColType, CommitEncode, CommitEngine, CommitId, CommitLayout, CommitStep, CommitmentId, CommitmentLayout, StrictHash, }; pub use merkle::{MerkleBuoy, MerkleHash, MerkleLeaves, MerkleNode, NodeBranching}; diff --git a/commit_verify/src/vesper.rs b/commit_verify/src/vesper.rs index c9f6ce4b..20b421f8 100644 --- a/commit_verify/src/vesper.rs +++ b/commit_verify/src/vesper.rs @@ -25,7 +25,7 @@ use strict_types::layout::vesper::LenRange; use strict_types::typesys::TypeFqn; use vesper::{AttrVal, Attribute, Expression, Predicate, TExpr}; -use crate::{CommitColType, CommitStep, CommitmentLayout}; +use crate::{CommitColType, CommitLayout, CommitStep}; pub type VesperCommit = TExpr; @@ -157,7 +157,7 @@ impl CommitStep { } } -impl CommitmentLayout { +impl CommitLayout { pub fn to_vesper(&self) -> VesperCommit { let subject = self.idty().name.to_ident(); From 49345a16d6125af5aab58b2b4e83f3bfd222b097 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Sun, 18 Feb 2024 20:31:28 +0100 Subject: [PATCH 4/8] commit: improve vesper syntax --- commit_verify/src/vesper.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/commit_verify/src/vesper.rs b/commit_verify/src/vesper.rs index 20b421f8..fbe3a847 100644 --- a/commit_verify/src/vesper.rs +++ b/commit_verify/src/vesper.rs @@ -33,10 +33,10 @@ pub type VesperCommit = TExpr; #[display(lowercase)] pub enum Pred { Commitment, - Serialize, - Hash, - Merklize, - Conceal, + Serialized, + Hashed, + Merklized, + Concealed, List, Set, Element, @@ -56,6 +56,7 @@ pub enum Attr { Tagged(&'static str), Concealed(TypeFqn), LenRange(LenRange), + Hasher, } #[derive(Clone, Eq, PartialEq, Hash, Debug, Display)] #[display(inner)] @@ -74,6 +75,7 @@ impl Attribute for Attr { Attr::Tagged(_) => Some(tn!("tagged")), Attr::Concealed(_) => Some(tn!("concealed")), Attr::LenRange(_) => Some(tn!("len")), + Attr::Hasher => Some(tn!("hasher")), } } @@ -82,6 +84,7 @@ impl Attribute for Attr { Attr::Tagged(tag) => AttrVal::Expr(AttrExpr::Tag(tag)), Attr::Concealed(fqn) => AttrVal::Ident(fqn.name.to_ident()), Attr::LenRange(range) => AttrVal::Expr(AttrExpr::LenRange(range.clone())), + Attr::Hasher => AttrVal::Ident(tn!("SHA256")), } } } @@ -101,13 +104,13 @@ impl CommitStep { fn predicate(&self) -> Pred { match self { - CommitStep::Serialized(_) => Pred::Serialize, + CommitStep::Serialized(_) => Pred::Serialized, CommitStep::Collection(CommitColType::List, _, _) => Pred::List, CommitStep::Collection(CommitColType::Set, _, _) => Pred::Set, CommitStep::Collection(CommitColType::Map { .. }, _, _) => Pred::Map, - CommitStep::Hashed(_) => Pred::Hash, - CommitStep::Merklized(_) => Pred::Merklize, - CommitStep::Concealed(_) => Pred::Conceal, + CommitStep::Hashed(_) => Pred::Hashed, + CommitStep::Merklized(_) => Pred::Merklized, + CommitStep::Concealed(_) => Pred::Concealed, } } @@ -177,7 +180,7 @@ impl CommitLayout { VesperCommit { subject, predicate: Pred::Commitment, - attributes: confined_vec![Attr::Tagged(self.tag())], + attributes: confined_vec![Attr::Hasher, Attr::Tagged(self.tag())], content: Confined::from_iter_unsafe(content), comment: None, } From d81288a7df7d52e5dfc4b2a17ad43780d6107cc4 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Sun, 18 Feb 2024 21:03:02 +0100 Subject: [PATCH 5/8] commit: support committing to optionals --- commit_verify/src/id.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/commit_verify/src/id.rs b/commit_verify/src/id.rs index 58cc2855..600e5470 100644 --- a/commit_verify/src/id.rs +++ b/commit_verify/src/id.rs @@ -99,6 +99,15 @@ impl CommitEngine { self.inner_commit_to::<_, COMMIT_MAX_LEN>(&value); } + pub fn commit_to_option(&mut self, value: &Option) { + let fqn = commitment_fqn::(); + self.layout + .push(CommitStep::Serialized(fqn)) + .expect("too many fields for commitment"); + + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&value); + } + pub fn commit_to_hash + StrictType>( &mut self, value: T, From b2c51280157ac687e0d0812b928c3db9a85928e0 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Mon, 19 Feb 2024 13:49:09 +0100 Subject: [PATCH 6/8] merkle: ensure proper large int conversion. Make var names better --- commit_verify/src/merkle.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/commit_verify/src/merkle.rs b/commit_verify/src/merkle.rs index 581412c4..0d6ceb17 100644 --- a/commit_verify/src/merkle.rs +++ b/commit_verify/src/merkle.rs @@ -154,35 +154,35 @@ impl MerkleHash { /// [LNPBP-81]: https://github.com/LNP-BP/LNPBPs/blob/master/lnpbp-0081.md pub fn merklize(leaves: &impl MerkleLeaves) -> Self { let mut nodes = leaves.merkle_leaves().map(|leaf| leaf.commit_id()); - let len = nodes.len() as u32; - if len == 1 { + let base_width = + u32::try_from(nodes.len()).expect("too many merkle leaves (more than 2^32)"); + if base_width == 1 { // If we have just one leaf, it's MerkleNode value is the root nodes.next().expect("length is 1") } else { - Self::_merklize(nodes, u5::ZERO, len) + Self::_merklize(nodes, u5::ZERO, base_width, base_width) } } - pub fn _merklize( + fn _merklize( mut iter: impl ExactSizeIterator, depth: u5, - width: u32, + branch_width: u32, + base_width: u32, ) -> Self { - let len = iter.len() as u16; - - if len <= 2 { + if branch_width <= 2 { match (iter.next(), iter.next()) { - (None, None) => MerkleHash::void(depth, width), - // Here, a single node means Merkle tree width nonequal to the power of 2, thus we + (None, None) => MerkleHash::void(depth, base_width), + // Here, a single node means Merkle tree width non-equal to the power of 2, thus we // need to process it with a special encoding. - (Some(branch), None) => MerkleHash::single(depth, width, branch), + (Some(branch), None) => MerkleHash::single(depth, base_width, branch), (Some(branch1), Some(branch2)) => { - MerkleHash::branches(depth, width, branch1, branch2) + MerkleHash::branches(depth, base_width, branch1, branch2) } (None, Some(_)) => unreachable!(), } } else { - let div = len / 2 + len % 2; + let div = branch_width / 2 + branch_width % 2; let slice = iter .by_ref() @@ -192,10 +192,10 @@ impl MerkleHash { // TODO: Do this without allocation .collect::>() .into_iter(); - let branch1 = Self::_merklize(slice, depth + 1, width); - let branch2 = Self::_merklize(iter, depth + 1, width); + let branch1 = Self::_merklize(slice, depth + 1, base_width, div); + let branch2 = Self::_merklize(iter, depth + 1, base_width, branch_width - div); - MerkleHash::branches(depth, width, branch1, branch2) + MerkleHash::branches(depth, base_width, branch1, branch2) } } } From 0353648826a2c78fc27bf0222feb6f7d9bd0fe94 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Tue, 20 Feb 2024 09:14:57 +0100 Subject: [PATCH 7/8] merkle: ensure collection size match --- commit_verify/src/merkle.rs | 18 ++++++++++++++++++ commit_verify/src/mpc/tree.rs | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/commit_verify/src/merkle.rs b/commit_verify/src/merkle.rs index 0d6ceb17..90e87d54 100644 --- a/commit_verify/src/merkle.rs +++ b/commit_verify/src/merkle.rs @@ -244,6 +244,24 @@ where T: CommitId + Copy fn merkle_leaves(&self) -> Self::LeafIter<'_> { self.iter().copied() } } +impl MerkleLeaves for Confined, MIN, { u32::MAX as usize }> +where T: CommitId + Copy +{ + type Leaf = T; + type LeafIter<'tmp> = iter::Copied> where Self: 'tmp; + + fn merkle_leaves(&self) -> Self::LeafIter<'_> { self.iter().copied() } +} + +impl MerkleLeaves for Confined, MIN, { u32::MAX as usize }> +where T: CommitId + Copy +{ + type Leaf = T; + type LeafIter<'tmp> = iter::Copied> where Self: 'tmp; + + fn merkle_leaves(&self) -> Self::LeafIter<'_> { self.iter().copied() } +} + /// Helper struct to track depth when working with Merkle blocks. #[derive(Clone, PartialEq, Eq, Debug, Default)] pub struct MerkleBuoy + Default> { diff --git a/commit_verify/src/mpc/tree.rs b/commit_verify/src/mpc/tree.rs index bcb5e699..ddd7c43e 100644 --- a/commit_verify/src/mpc/tree.rs +++ b/commit_verify/src/mpc/tree.rs @@ -19,7 +19,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use amplify::confinement::{MediumOrdMap, SmallVec}; +use amplify::confinement::{LargeVec, MediumOrdMap}; use amplify::num::{u256, u5}; use amplify::Wrapper; @@ -68,7 +68,7 @@ impl MerkleTree { .map(|(protocol, msg)| Leaf::inhabited(*protocol, *msg)) .unwrap_or_else(|| Leaf::entropy(self.entropy, pos)) }); - let leaves = SmallVec::try_from_iter(iter).expect("u16-bound size"); + let leaves = LargeVec::try_from_iter(iter).expect("tree width has u32-bound size"); MerkleHash::merklize(&leaves) } } From 588e97a1cc152c568839fd899d88e17b1e7bf517 Mon Sep 17 00:00:00 2001 From: Dr Maxim Orlovsky Date: Sun, 25 Feb 2024 10:50:13 +0100 Subject: [PATCH 8/8] merkle: improve test for large trees --- commit_verify/src/mpc/tree.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/commit_verify/src/mpc/tree.rs b/commit_verify/src/mpc/tree.rs index ddd7c43e..a0984c1f 100644 --- a/commit_verify/src/mpc/tree.rs +++ b/commit_verify/src/mpc/tree.rs @@ -281,18 +281,35 @@ mod test { #[test] fn tree_huge() { + // Tree with 8192 protocol-messages: depth 23, cofactor 103. Serialized length + // 1081361 bytes. Takes 71589 msecs to generate + // Root is 58755c63bbcb1a648982956c90a471a3fc79b12ae97867828e2f0ce8c9f7e7db. + // Takes 560735 msecs to compute + + use std::time::Instant; + let count = 1_048_576 / 128; let msgs = make_random_messages(count); + + let start = Instant::now(); let tree = make_random_tree(&msgs); + let elapsed_gen = start.elapsed(); + let mut counter = StreamWriter::counter::<{ usize::MAX }>(); tree.strict_write(&mut counter).unwrap(); eprintln!( - "Tree with {} protocol-messages: depth {}, cofactor {}. Serialized length {} bytes", - count, + "Tree with {count} protocol-messages: depth {}, cofactor {}. Serialized length {} \ + bytes. Takes {} msecs to generate", tree.depth, tree.cofactor, - counter.unconfine().count + counter.unconfine().count, + elapsed_gen.as_millis(), ); + + let start = Instant::now(); + let root = tree.root(); + let elapsed_root = start.elapsed(); + eprintln!("Root is {root}. Takes {} msecs to compute", elapsed_root.as_millis(),); } #[test]