diff --git a/Cargo.lock b/Cargo.lock index 13ca53e8..338d9126 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,7 @@ dependencies = [ "sha2", "strict_encoding", "strict_types", + "vesper-lang", ] [[package]] diff --git a/commit_verify/Cargo.toml b/commit_verify/Cargo.toml index b3bf4203..d142ca37 100644 --- a/commit_verify/Cargo.toml +++ b/commit_verify/Cargo.toml @@ -25,6 +25,7 @@ required-features = ["stl"] amplify = { workspace = true, features = ["hex", "apfloat"] } strict_encoding = { workspace = true } strict_types = { workspace = true } +vesper-lang = "0.1.0" commit_encoding_derive = { version = "0.11.0-beta.3", path = "derive" } sha2 = "0.10.8" ripemd = "0.1.3" diff --git a/commit_verify/derive/src/derive.rs b/commit_verify/derive/src/derive.rs index 56896a44..31cc8242 100644 --- a/commit_verify/derive/src/derive.rs +++ b/commit_verify/derive/src/derive.rs @@ -33,19 +33,19 @@ impl CommitDerive { let inner = match self.conf.strategy { StrategyAttr::Strict => quote! { - engine.commit_to(self); + engine.commit_to_serialized(self); }, StrategyAttr::ConcealStrict => quote! { use #trait_crate::Conceal; - engine.commit_to(&self.conceal()); + engine.commit_to_concealed(&self.conceal()); }, StrategyAttr::Transparent => quote! { use amplify::Wrapper; - engine.commit_to(self.as_inner()); + engine.commit_to_serialized(self.as_inner()); }, StrategyAttr::Merklize => quote! { use amplify::Wrapper; - engine.commit_to(self.as_inner().merklize()); + engine.commit_to_merkle(self.as_inner().merklize()); }, }; diff --git a/commit_verify/src/id.rs b/commit_verify/src/id.rs index 56b9c374..600e5470 100644 --- a/commit_verify/src/id.rs +++ b/commit_verify/src/id.rs @@ -19,21 +19,48 @@ // See the License for the specific language governing permissions and // limitations under the License. -use amplify::confinement::U64 as U64MAX; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::{self, Display, Formatter}; +use std::hash::Hash; + +use amplify::confinement::{Confined, TinyVec, U64 as U64MAX}; use amplify::Bytes32; use sha2::Sha256; -use strict_encoding::{StreamWriter, StrictEncode, StrictType}; +use strict_encoding::{Sizing, StreamWriter, StrictDumb, StrictEncode, StrictType}; use strict_types::typesys::TypeFqn; -use crate::{DigestExt, LIB_NAME_COMMIT_VERIFY}; +use crate::{Conceal, DigestExt, MerkleHash, MerkleLeaves, LIB_NAME_COMMIT_VERIFY}; const COMMIT_MAX_LEN: usize = U64MAX; -#[derive(Debug)] +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub enum CommitColType { + List, + Set, + Map { key: TypeFqn }, +} + +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub enum CommitStep { + Serialized(TypeFqn), + Collection(CommitColType, Sizing, TypeFqn), + Hashed(TypeFqn), + Merklized(TypeFqn), + Concealed(TypeFqn), +} + +#[derive(Clone, Debug)] pub struct CommitEngine { finished: bool, hasher: Sha256, - layout: Vec, + layout: TinyVec, +} + +fn commitment_fqn() -> TypeFqn { + TypeFqn::with( + libname!(T::STRICT_LIB_NAME), + T::strict_name().expect("commit encoder can commit only to named types"), + ) } impl CommitEngine { @@ -41,34 +68,145 @@ impl CommitEngine { Self { finished: false, hasher: Sha256::from_tag(tag), - layout: vec![], + layout: empty!(), } } - pub fn commit_to(&mut self, value: &T) { + fn inner_commit_to(&mut self, value: &T) { debug_assert!(!self.finished); - let writer = StreamWriter::new::(&mut self.hasher); + let writer = StreamWriter::new::(&mut self.hasher); let ok = value.strict_write(writer).is_ok(); - let fqn = TypeFqn::with( - libname!(T::STRICT_LIB_NAME), - T::strict_name().expect("commit encoder can commit only to named types"), - ); - self.layout.push(fqn); debug_assert!(ok); } - pub fn as_layout(&mut self) -> &[TypeFqn] { + pub fn commit_to_serialized(&mut self, value: &T) { + let fqn = commitment_fqn::(); + debug_assert!( + Some(&fqn.name) != MerkleHash::strict_name().as_ref() || + fqn.lib.as_str() != MerkleHash::STRICT_LIB_NAME, + "do not use commit_to_serialized for merklized collections, use commit_to_merkle \ + instead" + ); + debug_assert!( + Some(&fqn.name) != StrictHash::strict_name().as_ref() || + fqn.lib.as_str() != StrictHash::STRICT_LIB_NAME, + "do not use commit_to_serialized for StrictHash types, use commit_to_hash instead" + ); + self.layout + .push(CommitStep::Serialized(fqn)) + .expect("too many fields for commitment"); + + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&value); + } + + pub fn commit_to_option(&mut self, value: &Option) { + let fqn = commitment_fqn::(); + self.layout + .push(CommitStep::Serialized(fqn)) + .expect("too many fields for commitment"); + + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&value); + } + + pub fn commit_to_hash + StrictType>( + &mut self, + value: T, + ) { + let fqn = commitment_fqn::(); + self.layout + .push(CommitStep::Hashed(fqn)) + .expect("too many fields for commitment"); + + self.inner_commit_to::<_, 32>(&value.commit_id()); + } + + pub fn commit_to_merkle(&mut self, value: &T) + where T::Leaf: StrictType { + let fqn = commitment_fqn::(); + self.layout + .push(CommitStep::Merklized(fqn)) + .expect("too many fields for commitment"); + + let root = MerkleHash::merklize(value); + self.inner_commit_to::<_, 32>(&root); + } + + pub fn commit_to_concealed(&mut self, value: &T) + where + T: StrictType, + T::Concealed: StrictEncode, + { + let fqn = commitment_fqn::(); + self.layout + .push(CommitStep::Concealed(fqn)) + .expect("too many fields for commitment"); + + let concealed = value.conceal(); + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&concealed); + } + + pub fn commit_to_list( + &mut self, + collection: &Confined, MIN, MAX>, + ) where + T: StrictEncode + StrictDumb, + { + let fqn = commitment_fqn::(); + let step = + CommitStep::Collection(CommitColType::List, Sizing::new(MIN as u64, MAX as u64), fqn); + self.layout + .push(step) + .expect("too many fields for commitment"); + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&collection); + } + + pub fn commit_to_set( + &mut self, + collection: &Confined, MIN, MAX>, + ) where + T: Ord + StrictEncode + StrictDumb, + { + let fqn = commitment_fqn::(); + let step = + CommitStep::Collection(CommitColType::Set, Sizing::new(MIN as u64, MAX as u64), fqn); + self.layout + .push(step) + .expect("too many fields for commitment"); + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&collection); + } + + pub fn commit_to_map( + &mut self, + collection: &Confined, MIN, MAX>, + ) where + K: Ord + Hash + StrictEncode + StrictDumb, + V: StrictEncode + StrictDumb, + { + let key_fqn = commitment_fqn::(); + let val_fqn = commitment_fqn::(); + let step = CommitStep::Collection( + CommitColType::Map { key: key_fqn }, + Sizing::new(MIN as u64, MAX as u64), + val_fqn, + ); + self.layout + .push(step) + .expect("too many fields for commitment"); + self.inner_commit_to::<_, COMMIT_MAX_LEN>(&collection); + } + + pub fn as_layout(&mut self) -> &[CommitStep] { self.finished = true; self.layout.as_ref() } - pub fn into_layout(self) -> Vec { self.layout } + pub fn into_layout(self) -> TinyVec { self.layout } pub fn set_finished(&mut self) { self.finished = true; } pub fn finish(self) -> Sha256 { self.hasher } - pub fn finish_layout(self) -> (Sha256, Vec) { (self.hasher, self.layout) } + pub fn finish_layout(self) -> (Sha256, TinyVec) { (self.hasher, self.layout) } } /// Prepares the data to the *consensus commit* procedure by first running @@ -79,21 +217,50 @@ pub trait CommitEncode { type CommitmentId: CommitmentId; /// Encodes the data for the commitment by writing them directly into a - /// [`io::Write`] writer instance + /// [`std::io::Write`] writer instance fn commit_encode(&self, e: &mut CommitEngine); } #[derive(Getters, Clone, Eq, PartialEq, Hash, Debug)] -pub struct CommitmentLayout { - ty: TypeFqn, +pub struct CommitLayout { + idty: TypeFqn, + #[getter(as_copy)] tag: &'static str, - fields: Vec, + fields: TinyVec, +} + +impl Display for CommitLayout { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(&self.to_vesper().display(), f) + } } pub trait CommitmentId: Copy + Ord + From + StrictType { const TAG: &'static str; } +pub trait CommitmentLayout: CommitEncode { + fn commitment_layout() -> CommitLayout; +} + +impl CommitmentLayout for T +where T: CommitEncode + StrictDumb +{ + fn commitment_layout() -> CommitLayout { + let dumb = Self::strict_dumb(); + let fields = dumb.commit().into_layout(); + CommitLayout { + idty: TypeFqn::with( + libname!(Self::CommitmentId::STRICT_LIB_NAME), + Self::CommitmentId::strict_name() + .expect("commitment types must have explicit type name"), + ), + tag: T::CommitmentId::TAG, + fields, + } + } +} + /// High-level API used in client-side validation for producing a single /// commitment to the data, which includes running all necessary procedures like /// concealment with [`crate::Conceal`], merklization, strict encoding, @@ -107,8 +274,6 @@ pub trait CommitId: CommitEncode { #[doc = hidden] fn commit(&self) -> CommitEngine; - fn commitment_layout(&self) -> CommitmentLayout; - /// Performs commitment to client-side-validated data fn commit_id(&self) -> Self::CommitmentId; } @@ -121,19 +286,6 @@ impl CommitId for T { engine } - fn commitment_layout(&self) -> CommitmentLayout { - let fields = self.commit().into_layout(); - CommitmentLayout { - ty: TypeFqn::with( - libname!(Self::CommitmentId::STRICT_LIB_NAME), - Self::CommitmentId::strict_name() - .expect("commitment types must have explicit type name"), - ), - tag: T::CommitmentId::TAG, - fields, - } - } - fn commit_id(&self) -> Self::CommitmentId { self.commit().finish().into() } } diff --git a/commit_verify/src/lib.rs b/commit_verify/src/lib.rs index 0e0d9945..6e9d0e2c 100644 --- a/commit_verify/src/lib.rs +++ b/commit_verify/src/lib.rs @@ -56,13 +56,17 @@ pub mod stl; pub mod merkle; pub mod mpc; mod digest; +pub mod vesper; pub use commit::{CommitVerify, TryCommitVerify, VerifyError}; pub use conceal::Conceal; pub use convolve::{ConvolveCommit, ConvolveCommitProof, ConvolveVerifyError}; pub use digest::{Digest, DigestExt, Ripemd160, Sha256}; pub use embed::{EmbedCommitProof, EmbedCommitVerify, EmbedVerifyError, VerifyEq}; -pub use id::{CommitEncode, CommitEngine, CommitId, CommitmentId, CommitmentLayout, StrictHash}; +pub use id::{ + CommitColType, CommitEncode, CommitEngine, CommitId, CommitLayout, CommitStep, CommitmentId, + CommitmentLayout, StrictHash, +}; pub use merkle::{MerkleBuoy, MerkleHash, MerkleLeaves, MerkleNode, NodeBranching}; pub const LIB_NAME_COMMIT_VERIFY: &str = "CommitVerify"; diff --git a/commit_verify/src/merkle.rs b/commit_verify/src/merkle.rs index 581412c4..90e87d54 100644 --- a/commit_verify/src/merkle.rs +++ b/commit_verify/src/merkle.rs @@ -154,35 +154,35 @@ impl MerkleHash { /// [LNPBP-81]: https://github.com/LNP-BP/LNPBPs/blob/master/lnpbp-0081.md pub fn merklize(leaves: &impl MerkleLeaves) -> Self { let mut nodes = leaves.merkle_leaves().map(|leaf| leaf.commit_id()); - let len = nodes.len() as u32; - if len == 1 { + let base_width = + u32::try_from(nodes.len()).expect("too many merkle leaves (more than 2^32)"); + if base_width == 1 { // If we have just one leaf, it's MerkleNode value is the root nodes.next().expect("length is 1") } else { - Self::_merklize(nodes, u5::ZERO, len) + Self::_merklize(nodes, u5::ZERO, base_width, base_width) } } - pub fn _merklize( + fn _merklize( mut iter: impl ExactSizeIterator, depth: u5, - width: u32, + branch_width: u32, + base_width: u32, ) -> Self { - let len = iter.len() as u16; - - if len <= 2 { + if branch_width <= 2 { match (iter.next(), iter.next()) { - (None, None) => MerkleHash::void(depth, width), - // Here, a single node means Merkle tree width nonequal to the power of 2, thus we + (None, None) => MerkleHash::void(depth, base_width), + // Here, a single node means Merkle tree width non-equal to the power of 2, thus we // need to process it with a special encoding. - (Some(branch), None) => MerkleHash::single(depth, width, branch), + (Some(branch), None) => MerkleHash::single(depth, base_width, branch), (Some(branch1), Some(branch2)) => { - MerkleHash::branches(depth, width, branch1, branch2) + MerkleHash::branches(depth, base_width, branch1, branch2) } (None, Some(_)) => unreachable!(), } } else { - let div = len / 2 + len % 2; + let div = branch_width / 2 + branch_width % 2; let slice = iter .by_ref() @@ -192,10 +192,10 @@ impl MerkleHash { // TODO: Do this without allocation .collect::>() .into_iter(); - let branch1 = Self::_merklize(slice, depth + 1, width); - let branch2 = Self::_merklize(iter, depth + 1, width); + let branch1 = Self::_merklize(slice, depth + 1, base_width, div); + let branch2 = Self::_merklize(iter, depth + 1, base_width, branch_width - div); - MerkleHash::branches(depth, width, branch1, branch2) + MerkleHash::branches(depth, base_width, branch1, branch2) } } } @@ -244,6 +244,24 @@ where T: CommitId + Copy fn merkle_leaves(&self) -> Self::LeafIter<'_> { self.iter().copied() } } +impl MerkleLeaves for Confined, MIN, { u32::MAX as usize }> +where T: CommitId + Copy +{ + type Leaf = T; + type LeafIter<'tmp> = iter::Copied> where Self: 'tmp; + + fn merkle_leaves(&self) -> Self::LeafIter<'_> { self.iter().copied() } +} + +impl MerkleLeaves for Confined, MIN, { u32::MAX as usize }> +where T: CommitId + Copy +{ + type Leaf = T; + type LeafIter<'tmp> = iter::Copied> where Self: 'tmp; + + fn merkle_leaves(&self) -> Self::LeafIter<'_> { self.iter().copied() } +} + /// Helper struct to track depth when working with Merkle blocks. #[derive(Clone, PartialEq, Eq, Debug, Default)] pub struct MerkleBuoy + Default> { diff --git a/commit_verify/src/mpc/tree.rs b/commit_verify/src/mpc/tree.rs index bcb5e699..a0984c1f 100644 --- a/commit_verify/src/mpc/tree.rs +++ b/commit_verify/src/mpc/tree.rs @@ -19,7 +19,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use amplify::confinement::{MediumOrdMap, SmallVec}; +use amplify::confinement::{LargeVec, MediumOrdMap}; use amplify::num::{u256, u5}; use amplify::Wrapper; @@ -68,7 +68,7 @@ impl MerkleTree { .map(|(protocol, msg)| Leaf::inhabited(*protocol, *msg)) .unwrap_or_else(|| Leaf::entropy(self.entropy, pos)) }); - let leaves = SmallVec::try_from_iter(iter).expect("u16-bound size"); + let leaves = LargeVec::try_from_iter(iter).expect("tree width has u32-bound size"); MerkleHash::merklize(&leaves) } } @@ -281,18 +281,35 @@ mod test { #[test] fn tree_huge() { + // Tree with 8192 protocol-messages: depth 23, cofactor 103. Serialized length + // 1081361 bytes. Takes 71589 msecs to generate + // Root is 58755c63bbcb1a648982956c90a471a3fc79b12ae97867828e2f0ce8c9f7e7db. + // Takes 560735 msecs to compute + + use std::time::Instant; + let count = 1_048_576 / 128; let msgs = make_random_messages(count); + + let start = Instant::now(); let tree = make_random_tree(&msgs); + let elapsed_gen = start.elapsed(); + let mut counter = StreamWriter::counter::<{ usize::MAX }>(); tree.strict_write(&mut counter).unwrap(); eprintln!( - "Tree with {} protocol-messages: depth {}, cofactor {}. Serialized length {} bytes", - count, + "Tree with {count} protocol-messages: depth {}, cofactor {}. Serialized length {} \ + bytes. Takes {} msecs to generate", tree.depth, tree.cofactor, - counter.unconfine().count + counter.unconfine().count, + elapsed_gen.as_millis(), ); + + let start = Instant::now(); + let root = tree.root(); + let elapsed_root = start.elapsed(); + eprintln!("Root is {root}. Takes {} msecs to compute", elapsed_root.as_millis(),); } #[test] diff --git a/commit_verify/src/vesper.rs b/commit_verify/src/vesper.rs new file mode 100644 index 00000000..fbe3a847 --- /dev/null +++ b/commit_verify/src/vesper.rs @@ -0,0 +1,188 @@ +// Client-side-validation foundation libraries. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Written in 2024 by +// Dr. Maxim Orlovsky +// +// Copyright (C) 2024 LNP/BP Standards Association. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use amplify::confinement::{Confined, SmallVec, TinyVec}; +use strict_encoding::Ident; +use strict_types::layout::vesper::LenRange; +use strict_types::typesys::TypeFqn; +use vesper::{AttrVal, Attribute, Expression, Predicate, TExpr}; + +use crate::{CommitColType, CommitLayout, CommitStep}; + +pub type VesperCommit = TExpr; + +#[derive(Clone, Eq, PartialEq, Hash, Debug, Display)] +#[display(lowercase)] +pub enum Pred { + Commitment, + Serialized, + Hashed, + Merklized, + Concealed, + List, + Set, + Element, + Map, + #[display("mapKey")] + MapKey, + #[display("mapValue")] + MapValue, +} + +impl Predicate for Pred { + type Attr = Attr; +} + +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub enum Attr { + Tagged(&'static str), + Concealed(TypeFqn), + LenRange(LenRange), + Hasher, +} +#[derive(Clone, Eq, PartialEq, Hash, Debug, Display)] +#[display(inner)] +pub enum AttrExpr { + Tag(&'static str), + LenRange(LenRange), +} + +impl Expression for AttrExpr {} + +impl Attribute for Attr { + type Expression = AttrExpr; + + fn name(&self) -> Option { + match self { + Attr::Tagged(_) => Some(tn!("tagged")), + Attr::Concealed(_) => Some(tn!("concealed")), + Attr::LenRange(_) => Some(tn!("len")), + Attr::Hasher => Some(tn!("hasher")), + } + } + + fn value(&self) -> AttrVal { + match self { + Attr::Tagged(tag) => AttrVal::Expr(AttrExpr::Tag(tag)), + Attr::Concealed(fqn) => AttrVal::Ident(fqn.name.to_ident()), + Attr::LenRange(range) => AttrVal::Expr(AttrExpr::LenRange(range.clone())), + Attr::Hasher => AttrVal::Ident(tn!("SHA256")), + } + } +} + +impl CommitStep { + fn subject(&self) -> Ident { + match self { + CommitStep::Serialized(fqn) => fqn, + CommitStep::Collection(_, _, fqn) => fqn, + CommitStep::Hashed(fqn) => fqn, + CommitStep::Merklized(fqn) => fqn, + CommitStep::Concealed(fqn) => fqn, + } + .name + .to_ident() + } + + fn predicate(&self) -> Pred { + match self { + CommitStep::Serialized(_) => Pred::Serialized, + CommitStep::Collection(CommitColType::List, _, _) => Pred::List, + CommitStep::Collection(CommitColType::Set, _, _) => Pred::Set, + CommitStep::Collection(CommitColType::Map { .. }, _, _) => Pred::Map, + CommitStep::Hashed(_) => Pred::Hashed, + CommitStep::Merklized(_) => Pred::Merklized, + CommitStep::Concealed(_) => Pred::Concealed, + } + } + + fn attributes(&self) -> SmallVec { + match self { + CommitStep::Collection(_, sizing, _) => small_vec![Attr::LenRange((*sizing).into())], + CommitStep::Concealed(from) => small_vec![Attr::Concealed(from.clone())], + CommitStep::Serialized(_) | CommitStep::Hashed(_) | CommitStep::Merklized(_) => none!(), + } + } + + fn content(&self) -> TinyVec> { + match self { + CommitStep::Collection(CommitColType::List, _, val) | + CommitStep::Collection(CommitColType::Set, _, val) => { + tiny_vec![Box::new(VesperCommit { + subject: val.name.to_ident(), + predicate: Pred::Element, + attributes: none!(), + content: none!(), + comment: None + })] + } + CommitStep::Collection(CommitColType::Map { key }, _, val) => { + tiny_vec![ + Box::new(VesperCommit { + subject: key.name.to_ident(), + predicate: Pred::MapKey, + attributes: none!(), + content: none!(), + comment: None + }), + Box::new(VesperCommit { + subject: val.name.to_ident(), + predicate: Pred::MapValue, + attributes: none!(), + content: none!(), + comment: None + }) + ] + } + CommitStep::Serialized(_) | + CommitStep::Hashed(_) | + CommitStep::Merklized(_) | + CommitStep::Concealed(_) => empty!(), + } + } +} + +impl CommitLayout { + pub fn to_vesper(&self) -> VesperCommit { + let subject = self.idty().name.to_ident(); + + // SecretSeal commitment tagged="" + // BlindSeal rec serialized + + let content = self.fields().iter().map(|field| { + Box::new(VesperCommit { + subject: field.subject(), + predicate: field.predicate(), + attributes: field.attributes(), + content: field.content(), + comment: None, + }) + }); + + VesperCommit { + subject, + predicate: Pred::Commitment, + attributes: confined_vec![Attr::Hasher, Attr::Tagged(self.tag())], + content: Confined::from_iter_unsafe(content), + comment: None, + } + } +}