From 5a8a3dbc45471c7ded74e75e96722bed22560010 Mon Sep 17 00:00:00 2001 From: Liamsi Date: Wed, 20 Jun 2018 15:30:44 -0700 Subject: [PATCH 01/86] mv go-crypto files to crypto dir --- crypto/merkle/README.md | 4 + crypto/merkle/doc.go | 31 ++++++ crypto/merkle/simple_map.go | 88 ++++++++++++++++ crypto/merkle/simple_map_test.go | 54 ++++++++++ crypto/merkle/simple_proof.go | 160 ++++++++++++++++++++++++++++++ crypto/merkle/simple_tree.go | 58 +++++++++++ crypto/merkle/simple_tree_test.go | 88 ++++++++++++++++ crypto/merkle/types.go | 38 +++++++ 8 files changed, 521 insertions(+) create mode 100644 crypto/merkle/README.md create mode 100644 crypto/merkle/doc.go create mode 100644 crypto/merkle/simple_map.go create mode 100644 crypto/merkle/simple_map_test.go create mode 100644 crypto/merkle/simple_proof.go create mode 100644 crypto/merkle/simple_tree.go create mode 100644 crypto/merkle/simple_tree_test.go create mode 100644 crypto/merkle/types.go diff --git a/crypto/merkle/README.md b/crypto/merkle/README.md new file mode 100644 index 0000000..c449783 --- /dev/null +++ b/crypto/merkle/README.md @@ -0,0 +1,4 @@ +## Simple Merkle Tree + +For smaller static data structures that don't require immutable snapshots or mutability; +for instance the transactions and validation signatures of a block can be hashed using this simple merkle tree logic. diff --git a/crypto/merkle/doc.go b/crypto/merkle/doc.go new file mode 100644 index 0000000..da65dd8 --- /dev/null +++ b/crypto/merkle/doc.go @@ -0,0 +1,31 @@ +/* +Package merkle computes a deterministic minimal height Merkle tree hash. +If the number of items is not a power of two, some leaves +will be at different levels. Tries to keep both sides of +the tree the same size, but the left may be one greater. + +Use this for short deterministic trees, such as the validator list. +For larger datasets, use IAVLTree. + +Be aware that the current implementation by itself does not prevent +second pre-image attacks. Hence, use this library with caution. +Otherwise you might run into similar issues as, e.g., in early Bitcoin: +https://bitcointalk.org/?topic=102395 + + * + / \ + / \ + / \ + / \ + * * + / \ / \ + / \ / \ + / \ / \ + * * * h6 + / \ / \ / \ + h0 h1 h2 h3 h4 h5 + +TODO(ismail): add 2nd pre-image protection or clarify further on how we use this and why this secure. + +*/ +package merkle \ No newline at end of file diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go new file mode 100644 index 0000000..2486326 --- /dev/null +++ b/crypto/merkle/simple_map.go @@ -0,0 +1,88 @@ +package merkle + +import ( + "github.com/tendermint/go-crypto/tmhash" + cmn "github.com/tendermint/tmlibs/common" +) + +// Merkle tree from a map. +// Leaves are `hash(key) | hash(value)`. +// Leaves are sorted before Merkle hashing. +type simpleMap struct { + kvs cmn.KVPairs + sorted bool +} + +func newSimpleMap() *simpleMap { + return &simpleMap{ + kvs: nil, + sorted: false, + } +} + +// Set hashes the key and value and appends it to the kv pairs. +func (sm *simpleMap) Set(key string, value Hasher) { + sm.sorted = false + + // The value is hashed, so you can + // check for equality with a cached value (say) + // and make a determination to fetch or not. + vhash := value.Hash() + + sm.kvs = append(sm.kvs, cmn.KVPair{ + Key: []byte(key), + Value: vhash, + }) +} + +// Hash Merkle root hash of items sorted by key +// (UNSTABLE: and by value too if duplicate key). +func (sm *simpleMap) Hash() []byte { + sm.Sort() + return hashKVPairs(sm.kvs) +} + +func (sm *simpleMap) Sort() { + if sm.sorted { + return + } + sm.kvs.Sort() + sm.sorted = true +} + +// Returns a copy of sorted KVPairs. +// NOTE these contain the hashed key and value. +func (sm *simpleMap) KVPairs() cmn.KVPairs { + sm.Sort() + kvs := make(cmn.KVPairs, len(sm.kvs)) + copy(kvs, sm.kvs) + return kvs +} + +//---------------------------------------- + +// A local extension to KVPair that can be hashed. +// Key and value are length prefixed and concatenated, +// then hashed. +type KVPair cmn.KVPair + +func (kv KVPair) Hash() []byte { + hasher := tmhash.New() + err := encodeByteSlice(hasher, kv.Key) + if err != nil { + panic(err) + } + err = encodeByteSlice(hasher, kv.Value) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + +func hashKVPairs(kvs cmn.KVPairs) []byte { + kvsH := make([]Hasher, len(kvs)) + for i, kvp := range kvs { + kvsH[i] = KVPair(kvp) + } + return SimpleHashFromHashers(kvsH) +} diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go new file mode 100644 index 0000000..d9d6351 --- /dev/null +++ b/crypto/merkle/simple_map_test.go @@ -0,0 +1,54 @@ +package merkle + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/go-crypto/tmhash" +) + +type strHasher string + +func (str strHasher) Hash() []byte { + return tmhash.Sum([]byte(str)) +} + +func TestSimpleMap(t *testing.T) { + { + db := newSimpleMap() + db.Set("key1", strHasher("value1")) + assert.Equal(t, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key1", strHasher("value2")) + assert.Equal(t, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key1", strHasher("value1")) + db.Set("key2", strHasher("value2")) + assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key2", strHasher("value2")) // NOTE: out of order + db.Set("key1", strHasher("value1")) + assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key1", strHasher("value1")) + db.Set("key2", strHasher("value2")) + db.Set("key3", strHasher("value3")) + assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key2", strHasher("value2")) // NOTE: out of order + db.Set("key1", strHasher("value1")) + db.Set("key3", strHasher("value3")) + assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } +} diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go new file mode 100644 index 0000000..2541b6d --- /dev/null +++ b/crypto/merkle/simple_proof.go @@ -0,0 +1,160 @@ +package merkle + +import ( + "bytes" + "fmt" +) + +// SimpleProof represents a simple merkle proof. +type SimpleProof struct { + Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. +} + +// SimpleProofsFromHashers computes inclusion proof for given items. +// proofs[0] is the proof for items[0]. +func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) { + trails, rootSPN := trailsFromHashers(items) + rootHash = rootSPN.Hash + proofs = make([]*SimpleProof, len(items)) + for i, trail := range trails { + proofs[i] = &SimpleProof{ + Aunts: trail.FlattenAunts(), + } + } + return +} + +// SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values +// in the underlying key-value pairs. +// The keys are sorted before the proofs are computed. +func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) { + sm := newSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + sm.Sort() + kvs := sm.kvs + kvsH := make([]Hasher, 0, len(kvs)) + for _, kvp := range kvs { + kvsH = append(kvsH, KVPair(kvp)) + } + + rootHash, proofList := SimpleProofsFromHashers(kvsH) + proofs = make(map[string]*SimpleProof) + keys = make([]string, len(proofList)) + for i, kvp := range kvs { + proofs[string(kvp.Key)] = proofList[i] + keys[i] = string(kvp.Key) + } + return +} + +// Verify that leafHash is a leaf hash of the simple-merkle-tree +// which hashes to rootHash. +func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash []byte) bool { + computedHash := computeHashFromAunts(index, total, leafHash, sp.Aunts) + return computedHash != nil && bytes.Equal(computedHash, rootHash) +} + +// String implements the stringer interface for SimpleProof. +// It is a wrapper around StringIndented. +func (sp *SimpleProof) String() string { + return sp.StringIndented("") +} + +// StringIndented generates a canonical string representation of a SimpleProof. +func (sp *SimpleProof) StringIndented(indent string) string { + return fmt.Sprintf(`SimpleProof{ +%s Aunts: %X +%s}`, + indent, sp.Aunts, + indent) +} + +// Use the leafHash and innerHashes to get the root merkle hash. +// If the length of the innerHashes slice isn't exactly correct, the result is nil. +// Recursive impl. +func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][]byte) []byte { + if index >= total || index < 0 || total <= 0 { + return nil + } + switch total { + case 0: + panic("Cannot call computeHashFromAunts() with 0 total") + case 1: + if len(innerHashes) != 0 { + return nil + } + return leafHash + default: + if len(innerHashes) == 0 { + return nil + } + numLeft := (total + 1) / 2 + if index < numLeft { + leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if leftHash == nil { + return nil + } + return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + } + rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if rightHash == nil { + return nil + } + return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) + } +} + +// SimpleProofNode is a helper structure to construct merkle proof. +// The node and the tree is thrown away afterwards. +// Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. +// node.Parent.Hash = hash(node.Hash, node.Right.Hash) or +// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. +type SimpleProofNode struct { + Hash []byte + Parent *SimpleProofNode + Left *SimpleProofNode // Left sibling (only one of Left,Right is set) + Right *SimpleProofNode // Right sibling (only one of Left,Right is set) +} + +// FlattenAunts will return the inner hashes for the item corresponding to the leaf, +// starting from a leaf SimpleProofNode. +func (spn *SimpleProofNode) FlattenAunts() [][]byte { + // Nonrecursive impl. + innerHashes := [][]byte{} + for spn != nil { + if spn.Left != nil { + innerHashes = append(innerHashes, spn.Left.Hash) + } else if spn.Right != nil { + innerHashes = append(innerHashes, spn.Right.Hash) + } else { + break + } + spn = spn.Parent + } + return innerHashes +} + +// trails[0].Hash is the leaf hash for items[0]. +// trails[i].Parent.Parent....Parent == root for all i. +func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) { + // Recursive impl. + switch len(items) { + case 0: + return nil, nil + case 1: + trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil} + return []*SimpleProofNode{trail}, trail + default: + lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2]) + rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:]) + rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) + root := &SimpleProofNode{rootHash, nil, nil, nil} + leftRoot.Parent = root + leftRoot.Right = rightRoot + rightRoot.Parent = root + rightRoot.Left = leftRoot + return append(lefts, rights...), root + } +} diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go new file mode 100644 index 0000000..c23f842 --- /dev/null +++ b/crypto/merkle/simple_tree.go @@ -0,0 +1,58 @@ +package merkle + +import ( + "github.com/tendermint/go-crypto/tmhash" +) + +// SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). +func SimpleHashFromTwoHashes(left, right []byte) []byte { + var hasher = tmhash.New() + err := encodeByteSlice(hasher, left) + if err != nil { + panic(err) + } + err = encodeByteSlice(hasher, right) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + +// SimpleHashFromHashers computes a Merkle tree from items that can be hashed. +func SimpleHashFromHashers(items []Hasher) []byte { + hashes := make([][]byte, len(items)) + for i, item := range items { + hash := item.Hash() + hashes[i] = hash + } + return simpleHashFromHashes(hashes) +} + +// SimpleHashFromMap computes a Merkle tree from sorted map. +// Like calling SimpleHashFromHashers with +// `item = []byte(Hash(key) | Hash(value))`, +// sorted by `item`. +func SimpleHashFromMap(m map[string]Hasher) []byte { + sm := newSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + return sm.Hash() +} + +//---------------------------------------------------------------- + +// Expects hashes! +func simpleHashFromHashes(hashes [][]byte) []byte { + // Recursive impl. + switch len(hashes) { + case 0: + return nil + case 1: + return hashes[0] + default: + left := simpleHashFromHashes(hashes[:(len(hashes)+1)/2]) + right := simpleHashFromHashes(hashes[(len(hashes)+1)/2:]) + return SimpleHashFromTwoHashes(left, right) + } +} diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go new file mode 100644 index 0000000..db8e3d7 --- /dev/null +++ b/crypto/merkle/simple_tree_test.go @@ -0,0 +1,88 @@ +package merkle + +import ( + "bytes" + + cmn "github.com/tendermint/tmlibs/common" + . "github.com/tendermint/tmlibs/test" + + "testing" + "github.com/tendermint/go-crypto/tmhash" +) + +type testItem []byte + +func (tI testItem) Hash() []byte { + return []byte(tI) +} + +func TestSimpleProof(t *testing.T) { + + total := 100 + + items := make([]Hasher, total) + for i := 0; i < total; i++ { + items[i] = testItem(cmn.RandBytes(tmhash.Size)) + } + + rootHash := SimpleHashFromHashers(items) + + rootHash2, proofs := SimpleProofsFromHashers(items) + + if !bytes.Equal(rootHash, rootHash2) { + t.Errorf("Unmatched root hashes: %X vs %X", rootHash, rootHash2) + } + + // For each item, check the trail. + for i, item := range items { + itemHash := item.Hash() + proof := proofs[i] + + // Verify success + ok := proof.Verify(i, total, itemHash, rootHash) + if !ok { + t.Errorf("Verification failed for index %v.", i) + } + + // Wrong item index should make it fail + { + ok = proof.Verify((i+1)%total, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong index %v.", i) + } + } + + // Trail too long should make it fail + origAunts := proof.Aunts + proof.Aunts = append(proof.Aunts, cmn.RandBytes(32)) + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Trail too short should make it fail + proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1] + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Mutating the itemHash should make it fail. + ok = proof.Verify(i, total, MutateByteSlice(itemHash), rootHash) + if ok { + t.Errorf("Expected verification to fail for mutated leaf hash") + } + + // Mutating the rootHash should make it fail. + ok = proof.Verify(i, total, itemHash, MutateByteSlice(rootHash)) + if ok { + t.Errorf("Expected verification to fail for mutated root hash") + } + } +} diff --git a/crypto/merkle/types.go b/crypto/merkle/types.go new file mode 100644 index 0000000..2fcb3f3 --- /dev/null +++ b/crypto/merkle/types.go @@ -0,0 +1,38 @@ +package merkle + +import ( + "io" + + amino "github.com/tendermint/go-amino" +) + +// Tree is a Merkle tree interface. +type Tree interface { + Size() (size int) + Height() (height int8) + Has(key []byte) (has bool) + Proof(key []byte) (value []byte, proof []byte, exists bool) // TODO make it return an index + Get(key []byte) (index int, value []byte, exists bool) + GetByIndex(index int) (key []byte, value []byte) + Set(key []byte, value []byte) (updated bool) + Remove(key []byte) (value []byte, removed bool) + HashWithCount() (hash []byte, count int) + Hash() (hash []byte) + Save() (hash []byte) + Load(hash []byte) + Copy() Tree + Iterate(func(key []byte, value []byte) (stop bool)) (stopped bool) + IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool) +} + +// Hasher represents a hashable piece of data which can be hashed in the Tree. +type Hasher interface { + Hash() []byte +} + +//----------------------------------------------------------------------- + +// Uvarint length prefixed byteslice +func encodeByteSlice(w io.Writer, bz []byte) (err error) { + return amino.EncodeByteSlice(w, bz) +} From 9114c7021ffd494a332ecb27b5e9217dafbe35cd Mon Sep 17 00:00:00 2001 From: Liamsi Date: Wed, 20 Jun 2018 21:05:38 -0700 Subject: [PATCH 02/86] remove go-crypto from go-crypto: use tendermint/crypto :-) --- crypto/merkle/simple_map.go | 2 +- crypto/merkle/simple_map_test.go | 2 +- crypto/merkle/simple_tree.go | 2 +- crypto/merkle/simple_tree_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go index 2486326..86a9bad 100644 --- a/crypto/merkle/simple_map.go +++ b/crypto/merkle/simple_map.go @@ -1,7 +1,7 @@ package merkle import ( - "github.com/tendermint/go-crypto/tmhash" + "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tmlibs/common" ) diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go index d9d6351..34febcf 100644 --- a/crypto/merkle/simple_map_test.go +++ b/crypto/merkle/simple_map_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/tendermint/go-crypto/tmhash" + "github.com/tendermint/tendermint/crypto/tmhash" ) type strHasher string diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index c23f842..35a6eaa 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -1,7 +1,7 @@ package merkle import ( - "github.com/tendermint/go-crypto/tmhash" + "github.com/tendermint/tendermint/crypto/tmhash" ) // SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index db8e3d7..a721bcc 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -7,7 +7,7 @@ import ( . "github.com/tendermint/tmlibs/test" "testing" - "github.com/tendermint/go-crypto/tmhash" + "github.com/tendermint/tendermint/crypto/tmhash" ) type testItem []byte From 00d54ffe4045a22f259ace7384fdca90cb1a0d36 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 29 Jun 2018 12:17:26 +0400 Subject: [PATCH 03/86] revert back to Jae's original payload size limit except now we calculate the max size using the maxPacketMsgSize() function, which frees developers from having to know amino encoding details. plus, 10 additional bytes are added to leave the room for amino upgrades (both making it more efficient / less efficient) --- crypto/merkle/doc.go | 2 +- crypto/merkle/simple_tree.go | 8 ++++---- crypto/merkle/simple_tree_test.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/merkle/doc.go b/crypto/merkle/doc.go index da65dd8..865c302 100644 --- a/crypto/merkle/doc.go +++ b/crypto/merkle/doc.go @@ -28,4 +28,4 @@ https://bitcointalk.org/?topic=102395 TODO(ismail): add 2nd pre-image protection or clarify further on how we use this and why this secure. */ -package merkle \ No newline at end of file +package merkle diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 35a6eaa..46a0759 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -9,12 +9,12 @@ func SimpleHashFromTwoHashes(left, right []byte) []byte { var hasher = tmhash.New() err := encodeByteSlice(hasher, left) if err != nil { - panic(err) - } + panic(err) + } err = encodeByteSlice(hasher, right) if err != nil { - panic(err) - } + panic(err) + } return hasher.Sum(nil) } diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index a721bcc..6eef936 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -6,8 +6,8 @@ import ( cmn "github.com/tendermint/tmlibs/common" . "github.com/tendermint/tmlibs/test" - "testing" "github.com/tendermint/tendermint/crypto/tmhash" + "testing" ) type testItem []byte From 670b39bef13fb0ada7fb1f2fd6cc91b0815f076e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 1 Jul 2018 22:36:49 -0400 Subject: [PATCH 04/86] fix import paths --- crypto/merkle/simple_map.go | 2 +- crypto/merkle/simple_tree_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go index 86a9bad..ba4b930 100644 --- a/crypto/merkle/simple_map.go +++ b/crypto/merkle/simple_map.go @@ -2,7 +2,7 @@ package merkle import ( "github.com/tendermint/tendermint/crypto/tmhash" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) // Merkle tree from a map. diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index 6eef936..488e0c9 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -3,8 +3,8 @@ package merkle import ( "bytes" - cmn "github.com/tendermint/tmlibs/common" - . "github.com/tendermint/tmlibs/test" + cmn "github.com/tendermint/tendermint/libs/common" + . "github.com/tendermint/tendermint/libs/test" "github.com/tendermint/tendermint/crypto/tmhash" "testing" From 031b463c486244e619b47144418eafff8a4b1579 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Tue, 10 Jul 2018 11:12:46 -0400 Subject: [PATCH 05/86] detele everything --- crypto/merkle/README.md | 4 - crypto/merkle/doc.go | 31 ------ crypto/merkle/simple_map.go | 88 ---------------- crypto/merkle/simple_map_test.go | 54 ---------- crypto/merkle/simple_proof.go | 160 ------------------------------ crypto/merkle/simple_tree.go | 58 ----------- crypto/merkle/simple_tree_test.go | 88 ---------------- crypto/merkle/types.go | 38 ------- 8 files changed, 521 deletions(-) delete mode 100644 crypto/merkle/README.md delete mode 100644 crypto/merkle/doc.go delete mode 100644 crypto/merkle/simple_map.go delete mode 100644 crypto/merkle/simple_map_test.go delete mode 100644 crypto/merkle/simple_proof.go delete mode 100644 crypto/merkle/simple_tree.go delete mode 100644 crypto/merkle/simple_tree_test.go delete mode 100644 crypto/merkle/types.go diff --git a/crypto/merkle/README.md b/crypto/merkle/README.md deleted file mode 100644 index c449783..0000000 --- a/crypto/merkle/README.md +++ /dev/null @@ -1,4 +0,0 @@ -## Simple Merkle Tree - -For smaller static data structures that don't require immutable snapshots or mutability; -for instance the transactions and validation signatures of a block can be hashed using this simple merkle tree logic. diff --git a/crypto/merkle/doc.go b/crypto/merkle/doc.go deleted file mode 100644 index 865c302..0000000 --- a/crypto/merkle/doc.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Package merkle computes a deterministic minimal height Merkle tree hash. -If the number of items is not a power of two, some leaves -will be at different levels. Tries to keep both sides of -the tree the same size, but the left may be one greater. - -Use this for short deterministic trees, such as the validator list. -For larger datasets, use IAVLTree. - -Be aware that the current implementation by itself does not prevent -second pre-image attacks. Hence, use this library with caution. -Otherwise you might run into similar issues as, e.g., in early Bitcoin: -https://bitcointalk.org/?topic=102395 - - * - / \ - / \ - / \ - / \ - * * - / \ / \ - / \ / \ - / \ / \ - * * * h6 - / \ / \ / \ - h0 h1 h2 h3 h4 h5 - -TODO(ismail): add 2nd pre-image protection or clarify further on how we use this and why this secure. - -*/ -package merkle diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go deleted file mode 100644 index ba4b930..0000000 --- a/crypto/merkle/simple_map.go +++ /dev/null @@ -1,88 +0,0 @@ -package merkle - -import ( - "github.com/tendermint/tendermint/crypto/tmhash" - cmn "github.com/tendermint/tendermint/libs/common" -) - -// Merkle tree from a map. -// Leaves are `hash(key) | hash(value)`. -// Leaves are sorted before Merkle hashing. -type simpleMap struct { - kvs cmn.KVPairs - sorted bool -} - -func newSimpleMap() *simpleMap { - return &simpleMap{ - kvs: nil, - sorted: false, - } -} - -// Set hashes the key and value and appends it to the kv pairs. -func (sm *simpleMap) Set(key string, value Hasher) { - sm.sorted = false - - // The value is hashed, so you can - // check for equality with a cached value (say) - // and make a determination to fetch or not. - vhash := value.Hash() - - sm.kvs = append(sm.kvs, cmn.KVPair{ - Key: []byte(key), - Value: vhash, - }) -} - -// Hash Merkle root hash of items sorted by key -// (UNSTABLE: and by value too if duplicate key). -func (sm *simpleMap) Hash() []byte { - sm.Sort() - return hashKVPairs(sm.kvs) -} - -func (sm *simpleMap) Sort() { - if sm.sorted { - return - } - sm.kvs.Sort() - sm.sorted = true -} - -// Returns a copy of sorted KVPairs. -// NOTE these contain the hashed key and value. -func (sm *simpleMap) KVPairs() cmn.KVPairs { - sm.Sort() - kvs := make(cmn.KVPairs, len(sm.kvs)) - copy(kvs, sm.kvs) - return kvs -} - -//---------------------------------------- - -// A local extension to KVPair that can be hashed. -// Key and value are length prefixed and concatenated, -// then hashed. -type KVPair cmn.KVPair - -func (kv KVPair) Hash() []byte { - hasher := tmhash.New() - err := encodeByteSlice(hasher, kv.Key) - if err != nil { - panic(err) - } - err = encodeByteSlice(hasher, kv.Value) - if err != nil { - panic(err) - } - return hasher.Sum(nil) -} - -func hashKVPairs(kvs cmn.KVPairs) []byte { - kvsH := make([]Hasher, len(kvs)) - for i, kvp := range kvs { - kvsH[i] = KVPair(kvp) - } - return SimpleHashFromHashers(kvsH) -} diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go deleted file mode 100644 index 34febcf..0000000 --- a/crypto/merkle/simple_map_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package merkle - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tendermint/tendermint/crypto/tmhash" -) - -type strHasher string - -func (str strHasher) Hash() []byte { - return tmhash.Sum([]byte(str)) -} - -func TestSimpleMap(t *testing.T) { - { - db := newSimpleMap() - db.Set("key1", strHasher("value1")) - assert.Equal(t, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key1", strHasher("value2")) - assert.Equal(t, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key1", strHasher("value1")) - db.Set("key2", strHasher("value2")) - assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key2", strHasher("value2")) // NOTE: out of order - db.Set("key1", strHasher("value1")) - assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key1", strHasher("value1")) - db.Set("key2", strHasher("value2")) - db.Set("key3", strHasher("value3")) - assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key2", strHasher("value2")) // NOTE: out of order - db.Set("key1", strHasher("value1")) - db.Set("key3", strHasher("value3")) - assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } -} diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go deleted file mode 100644 index 2541b6d..0000000 --- a/crypto/merkle/simple_proof.go +++ /dev/null @@ -1,160 +0,0 @@ -package merkle - -import ( - "bytes" - "fmt" -) - -// SimpleProof represents a simple merkle proof. -type SimpleProof struct { - Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. -} - -// SimpleProofsFromHashers computes inclusion proof for given items. -// proofs[0] is the proof for items[0]. -func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) { - trails, rootSPN := trailsFromHashers(items) - rootHash = rootSPN.Hash - proofs = make([]*SimpleProof, len(items)) - for i, trail := range trails { - proofs[i] = &SimpleProof{ - Aunts: trail.FlattenAunts(), - } - } - return -} - -// SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values -// in the underlying key-value pairs. -// The keys are sorted before the proofs are computed. -func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) { - sm := newSimpleMap() - for k, v := range m { - sm.Set(k, v) - } - sm.Sort() - kvs := sm.kvs - kvsH := make([]Hasher, 0, len(kvs)) - for _, kvp := range kvs { - kvsH = append(kvsH, KVPair(kvp)) - } - - rootHash, proofList := SimpleProofsFromHashers(kvsH) - proofs = make(map[string]*SimpleProof) - keys = make([]string, len(proofList)) - for i, kvp := range kvs { - proofs[string(kvp.Key)] = proofList[i] - keys[i] = string(kvp.Key) - } - return -} - -// Verify that leafHash is a leaf hash of the simple-merkle-tree -// which hashes to rootHash. -func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash []byte) bool { - computedHash := computeHashFromAunts(index, total, leafHash, sp.Aunts) - return computedHash != nil && bytes.Equal(computedHash, rootHash) -} - -// String implements the stringer interface for SimpleProof. -// It is a wrapper around StringIndented. -func (sp *SimpleProof) String() string { - return sp.StringIndented("") -} - -// StringIndented generates a canonical string representation of a SimpleProof. -func (sp *SimpleProof) StringIndented(indent string) string { - return fmt.Sprintf(`SimpleProof{ -%s Aunts: %X -%s}`, - indent, sp.Aunts, - indent) -} - -// Use the leafHash and innerHashes to get the root merkle hash. -// If the length of the innerHashes slice isn't exactly correct, the result is nil. -// Recursive impl. -func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][]byte) []byte { - if index >= total || index < 0 || total <= 0 { - return nil - } - switch total { - case 0: - panic("Cannot call computeHashFromAunts() with 0 total") - case 1: - if len(innerHashes) != 0 { - return nil - } - return leafHash - default: - if len(innerHashes) == 0 { - return nil - } - numLeft := (total + 1) / 2 - if index < numLeft { - leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) - if leftHash == nil { - return nil - } - return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) - } - rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) - if rightHash == nil { - return nil - } - return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) - } -} - -// SimpleProofNode is a helper structure to construct merkle proof. -// The node and the tree is thrown away afterwards. -// Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. -// node.Parent.Hash = hash(node.Hash, node.Right.Hash) or -// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. -type SimpleProofNode struct { - Hash []byte - Parent *SimpleProofNode - Left *SimpleProofNode // Left sibling (only one of Left,Right is set) - Right *SimpleProofNode // Right sibling (only one of Left,Right is set) -} - -// FlattenAunts will return the inner hashes for the item corresponding to the leaf, -// starting from a leaf SimpleProofNode. -func (spn *SimpleProofNode) FlattenAunts() [][]byte { - // Nonrecursive impl. - innerHashes := [][]byte{} - for spn != nil { - if spn.Left != nil { - innerHashes = append(innerHashes, spn.Left.Hash) - } else if spn.Right != nil { - innerHashes = append(innerHashes, spn.Right.Hash) - } else { - break - } - spn = spn.Parent - } - return innerHashes -} - -// trails[0].Hash is the leaf hash for items[0]. -// trails[i].Parent.Parent....Parent == root for all i. -func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) { - // Recursive impl. - switch len(items) { - case 0: - return nil, nil - case 1: - trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil} - return []*SimpleProofNode{trail}, trail - default: - lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2]) - rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:]) - rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) - root := &SimpleProofNode{rootHash, nil, nil, nil} - leftRoot.Parent = root - leftRoot.Right = rightRoot - rightRoot.Parent = root - rightRoot.Left = leftRoot - return append(lefts, rights...), root - } -} diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go deleted file mode 100644 index 46a0759..0000000 --- a/crypto/merkle/simple_tree.go +++ /dev/null @@ -1,58 +0,0 @@ -package merkle - -import ( - "github.com/tendermint/tendermint/crypto/tmhash" -) - -// SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). -func SimpleHashFromTwoHashes(left, right []byte) []byte { - var hasher = tmhash.New() - err := encodeByteSlice(hasher, left) - if err != nil { - panic(err) - } - err = encodeByteSlice(hasher, right) - if err != nil { - panic(err) - } - return hasher.Sum(nil) -} - -// SimpleHashFromHashers computes a Merkle tree from items that can be hashed. -func SimpleHashFromHashers(items []Hasher) []byte { - hashes := make([][]byte, len(items)) - for i, item := range items { - hash := item.Hash() - hashes[i] = hash - } - return simpleHashFromHashes(hashes) -} - -// SimpleHashFromMap computes a Merkle tree from sorted map. -// Like calling SimpleHashFromHashers with -// `item = []byte(Hash(key) | Hash(value))`, -// sorted by `item`. -func SimpleHashFromMap(m map[string]Hasher) []byte { - sm := newSimpleMap() - for k, v := range m { - sm.Set(k, v) - } - return sm.Hash() -} - -//---------------------------------------------------------------- - -// Expects hashes! -func simpleHashFromHashes(hashes [][]byte) []byte { - // Recursive impl. - switch len(hashes) { - case 0: - return nil - case 1: - return hashes[0] - default: - left := simpleHashFromHashes(hashes[:(len(hashes)+1)/2]) - right := simpleHashFromHashes(hashes[(len(hashes)+1)/2:]) - return SimpleHashFromTwoHashes(left, right) - } -} diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go deleted file mode 100644 index 488e0c9..0000000 --- a/crypto/merkle/simple_tree_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package merkle - -import ( - "bytes" - - cmn "github.com/tendermint/tendermint/libs/common" - . "github.com/tendermint/tendermint/libs/test" - - "github.com/tendermint/tendermint/crypto/tmhash" - "testing" -) - -type testItem []byte - -func (tI testItem) Hash() []byte { - return []byte(tI) -} - -func TestSimpleProof(t *testing.T) { - - total := 100 - - items := make([]Hasher, total) - for i := 0; i < total; i++ { - items[i] = testItem(cmn.RandBytes(tmhash.Size)) - } - - rootHash := SimpleHashFromHashers(items) - - rootHash2, proofs := SimpleProofsFromHashers(items) - - if !bytes.Equal(rootHash, rootHash2) { - t.Errorf("Unmatched root hashes: %X vs %X", rootHash, rootHash2) - } - - // For each item, check the trail. - for i, item := range items { - itemHash := item.Hash() - proof := proofs[i] - - // Verify success - ok := proof.Verify(i, total, itemHash, rootHash) - if !ok { - t.Errorf("Verification failed for index %v.", i) - } - - // Wrong item index should make it fail - { - ok = proof.Verify((i+1)%total, total, itemHash, rootHash) - if ok { - t.Errorf("Expected verification to fail for wrong index %v.", i) - } - } - - // Trail too long should make it fail - origAunts := proof.Aunts - proof.Aunts = append(proof.Aunts, cmn.RandBytes(32)) - { - ok = proof.Verify(i, total, itemHash, rootHash) - if ok { - t.Errorf("Expected verification to fail for wrong trail length.") - } - } - proof.Aunts = origAunts - - // Trail too short should make it fail - proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1] - { - ok = proof.Verify(i, total, itemHash, rootHash) - if ok { - t.Errorf("Expected verification to fail for wrong trail length.") - } - } - proof.Aunts = origAunts - - // Mutating the itemHash should make it fail. - ok = proof.Verify(i, total, MutateByteSlice(itemHash), rootHash) - if ok { - t.Errorf("Expected verification to fail for mutated leaf hash") - } - - // Mutating the rootHash should make it fail. - ok = proof.Verify(i, total, itemHash, MutateByteSlice(rootHash)) - if ok { - t.Errorf("Expected verification to fail for mutated root hash") - } - } -} diff --git a/crypto/merkle/types.go b/crypto/merkle/types.go deleted file mode 100644 index 2fcb3f3..0000000 --- a/crypto/merkle/types.go +++ /dev/null @@ -1,38 +0,0 @@ -package merkle - -import ( - "io" - - amino "github.com/tendermint/go-amino" -) - -// Tree is a Merkle tree interface. -type Tree interface { - Size() (size int) - Height() (height int8) - Has(key []byte) (has bool) - Proof(key []byte) (value []byte, proof []byte, exists bool) // TODO make it return an index - Get(key []byte) (index int, value []byte, exists bool) - GetByIndex(index int) (key []byte, value []byte) - Set(key []byte, value []byte) (updated bool) - Remove(key []byte) (value []byte, removed bool) - HashWithCount() (hash []byte, count int) - Hash() (hash []byte) - Save() (hash []byte) - Load(hash []byte) - Copy() Tree - Iterate(func(key []byte, value []byte) (stop bool)) (stopped bool) - IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool) -} - -// Hasher represents a hashable piece of data which can be hashed in the Tree. -type Hasher interface { - Hash() []byte -} - -//----------------------------------------------------------------------- - -// Uvarint length prefixed byteslice -func encodeByteSlice(w io.Writer, bz []byte) (err error) { - return amino.EncodeByteSlice(w, bz) -} From 056593f1ff5e0e77da8147981e606cba8fadfc99 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Tue, 10 Jul 2018 11:22:25 -0400 Subject: [PATCH 06/86] Revert "detele everything" This reverts commit 031b463c486244e619b47144418eafff8a4b1579. --- crypto/merkle/README.md | 4 + crypto/merkle/doc.go | 31 ++++++ crypto/merkle/simple_map.go | 88 ++++++++++++++++ crypto/merkle/simple_map_test.go | 54 ++++++++++ crypto/merkle/simple_proof.go | 160 ++++++++++++++++++++++++++++++ crypto/merkle/simple_tree.go | 58 +++++++++++ crypto/merkle/simple_tree_test.go | 88 ++++++++++++++++ crypto/merkle/types.go | 38 +++++++ 8 files changed, 521 insertions(+) create mode 100644 crypto/merkle/README.md create mode 100644 crypto/merkle/doc.go create mode 100644 crypto/merkle/simple_map.go create mode 100644 crypto/merkle/simple_map_test.go create mode 100644 crypto/merkle/simple_proof.go create mode 100644 crypto/merkle/simple_tree.go create mode 100644 crypto/merkle/simple_tree_test.go create mode 100644 crypto/merkle/types.go diff --git a/crypto/merkle/README.md b/crypto/merkle/README.md new file mode 100644 index 0000000..c449783 --- /dev/null +++ b/crypto/merkle/README.md @@ -0,0 +1,4 @@ +## Simple Merkle Tree + +For smaller static data structures that don't require immutable snapshots or mutability; +for instance the transactions and validation signatures of a block can be hashed using this simple merkle tree logic. diff --git a/crypto/merkle/doc.go b/crypto/merkle/doc.go new file mode 100644 index 0000000..865c302 --- /dev/null +++ b/crypto/merkle/doc.go @@ -0,0 +1,31 @@ +/* +Package merkle computes a deterministic minimal height Merkle tree hash. +If the number of items is not a power of two, some leaves +will be at different levels. Tries to keep both sides of +the tree the same size, but the left may be one greater. + +Use this for short deterministic trees, such as the validator list. +For larger datasets, use IAVLTree. + +Be aware that the current implementation by itself does not prevent +second pre-image attacks. Hence, use this library with caution. +Otherwise you might run into similar issues as, e.g., in early Bitcoin: +https://bitcointalk.org/?topic=102395 + + * + / \ + / \ + / \ + / \ + * * + / \ / \ + / \ / \ + / \ / \ + * * * h6 + / \ / \ / \ + h0 h1 h2 h3 h4 h5 + +TODO(ismail): add 2nd pre-image protection or clarify further on how we use this and why this secure. + +*/ +package merkle diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go new file mode 100644 index 0000000..ba4b930 --- /dev/null +++ b/crypto/merkle/simple_map.go @@ -0,0 +1,88 @@ +package merkle + +import ( + "github.com/tendermint/tendermint/crypto/tmhash" + cmn "github.com/tendermint/tendermint/libs/common" +) + +// Merkle tree from a map. +// Leaves are `hash(key) | hash(value)`. +// Leaves are sorted before Merkle hashing. +type simpleMap struct { + kvs cmn.KVPairs + sorted bool +} + +func newSimpleMap() *simpleMap { + return &simpleMap{ + kvs: nil, + sorted: false, + } +} + +// Set hashes the key and value and appends it to the kv pairs. +func (sm *simpleMap) Set(key string, value Hasher) { + sm.sorted = false + + // The value is hashed, so you can + // check for equality with a cached value (say) + // and make a determination to fetch or not. + vhash := value.Hash() + + sm.kvs = append(sm.kvs, cmn.KVPair{ + Key: []byte(key), + Value: vhash, + }) +} + +// Hash Merkle root hash of items sorted by key +// (UNSTABLE: and by value too if duplicate key). +func (sm *simpleMap) Hash() []byte { + sm.Sort() + return hashKVPairs(sm.kvs) +} + +func (sm *simpleMap) Sort() { + if sm.sorted { + return + } + sm.kvs.Sort() + sm.sorted = true +} + +// Returns a copy of sorted KVPairs. +// NOTE these contain the hashed key and value. +func (sm *simpleMap) KVPairs() cmn.KVPairs { + sm.Sort() + kvs := make(cmn.KVPairs, len(sm.kvs)) + copy(kvs, sm.kvs) + return kvs +} + +//---------------------------------------- + +// A local extension to KVPair that can be hashed. +// Key and value are length prefixed and concatenated, +// then hashed. +type KVPair cmn.KVPair + +func (kv KVPair) Hash() []byte { + hasher := tmhash.New() + err := encodeByteSlice(hasher, kv.Key) + if err != nil { + panic(err) + } + err = encodeByteSlice(hasher, kv.Value) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + +func hashKVPairs(kvs cmn.KVPairs) []byte { + kvsH := make([]Hasher, len(kvs)) + for i, kvp := range kvs { + kvsH[i] = KVPair(kvp) + } + return SimpleHashFromHashers(kvsH) +} diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go new file mode 100644 index 0000000..34febcf --- /dev/null +++ b/crypto/merkle/simple_map_test.go @@ -0,0 +1,54 @@ +package merkle + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/tendermint/crypto/tmhash" +) + +type strHasher string + +func (str strHasher) Hash() []byte { + return tmhash.Sum([]byte(str)) +} + +func TestSimpleMap(t *testing.T) { + { + db := newSimpleMap() + db.Set("key1", strHasher("value1")) + assert.Equal(t, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key1", strHasher("value2")) + assert.Equal(t, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key1", strHasher("value1")) + db.Set("key2", strHasher("value2")) + assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key2", strHasher("value2")) // NOTE: out of order + db.Set("key1", strHasher("value1")) + assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key1", strHasher("value1")) + db.Set("key2", strHasher("value2")) + db.Set("key3", strHasher("value3")) + assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := newSimpleMap() + db.Set("key2", strHasher("value2")) // NOTE: out of order + db.Set("key1", strHasher("value1")) + db.Set("key3", strHasher("value3")) + assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } +} diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go new file mode 100644 index 0000000..2541b6d --- /dev/null +++ b/crypto/merkle/simple_proof.go @@ -0,0 +1,160 @@ +package merkle + +import ( + "bytes" + "fmt" +) + +// SimpleProof represents a simple merkle proof. +type SimpleProof struct { + Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. +} + +// SimpleProofsFromHashers computes inclusion proof for given items. +// proofs[0] is the proof for items[0]. +func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) { + trails, rootSPN := trailsFromHashers(items) + rootHash = rootSPN.Hash + proofs = make([]*SimpleProof, len(items)) + for i, trail := range trails { + proofs[i] = &SimpleProof{ + Aunts: trail.FlattenAunts(), + } + } + return +} + +// SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values +// in the underlying key-value pairs. +// The keys are sorted before the proofs are computed. +func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) { + sm := newSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + sm.Sort() + kvs := sm.kvs + kvsH := make([]Hasher, 0, len(kvs)) + for _, kvp := range kvs { + kvsH = append(kvsH, KVPair(kvp)) + } + + rootHash, proofList := SimpleProofsFromHashers(kvsH) + proofs = make(map[string]*SimpleProof) + keys = make([]string, len(proofList)) + for i, kvp := range kvs { + proofs[string(kvp.Key)] = proofList[i] + keys[i] = string(kvp.Key) + } + return +} + +// Verify that leafHash is a leaf hash of the simple-merkle-tree +// which hashes to rootHash. +func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash []byte) bool { + computedHash := computeHashFromAunts(index, total, leafHash, sp.Aunts) + return computedHash != nil && bytes.Equal(computedHash, rootHash) +} + +// String implements the stringer interface for SimpleProof. +// It is a wrapper around StringIndented. +func (sp *SimpleProof) String() string { + return sp.StringIndented("") +} + +// StringIndented generates a canonical string representation of a SimpleProof. +func (sp *SimpleProof) StringIndented(indent string) string { + return fmt.Sprintf(`SimpleProof{ +%s Aunts: %X +%s}`, + indent, sp.Aunts, + indent) +} + +// Use the leafHash and innerHashes to get the root merkle hash. +// If the length of the innerHashes slice isn't exactly correct, the result is nil. +// Recursive impl. +func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][]byte) []byte { + if index >= total || index < 0 || total <= 0 { + return nil + } + switch total { + case 0: + panic("Cannot call computeHashFromAunts() with 0 total") + case 1: + if len(innerHashes) != 0 { + return nil + } + return leafHash + default: + if len(innerHashes) == 0 { + return nil + } + numLeft := (total + 1) / 2 + if index < numLeft { + leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if leftHash == nil { + return nil + } + return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + } + rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if rightHash == nil { + return nil + } + return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) + } +} + +// SimpleProofNode is a helper structure to construct merkle proof. +// The node and the tree is thrown away afterwards. +// Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. +// node.Parent.Hash = hash(node.Hash, node.Right.Hash) or +// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. +type SimpleProofNode struct { + Hash []byte + Parent *SimpleProofNode + Left *SimpleProofNode // Left sibling (only one of Left,Right is set) + Right *SimpleProofNode // Right sibling (only one of Left,Right is set) +} + +// FlattenAunts will return the inner hashes for the item corresponding to the leaf, +// starting from a leaf SimpleProofNode. +func (spn *SimpleProofNode) FlattenAunts() [][]byte { + // Nonrecursive impl. + innerHashes := [][]byte{} + for spn != nil { + if spn.Left != nil { + innerHashes = append(innerHashes, spn.Left.Hash) + } else if spn.Right != nil { + innerHashes = append(innerHashes, spn.Right.Hash) + } else { + break + } + spn = spn.Parent + } + return innerHashes +} + +// trails[0].Hash is the leaf hash for items[0]. +// trails[i].Parent.Parent....Parent == root for all i. +func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) { + // Recursive impl. + switch len(items) { + case 0: + return nil, nil + case 1: + trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil} + return []*SimpleProofNode{trail}, trail + default: + lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2]) + rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:]) + rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) + root := &SimpleProofNode{rootHash, nil, nil, nil} + leftRoot.Parent = root + leftRoot.Right = rightRoot + rightRoot.Parent = root + rightRoot.Left = leftRoot + return append(lefts, rights...), root + } +} diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go new file mode 100644 index 0000000..46a0759 --- /dev/null +++ b/crypto/merkle/simple_tree.go @@ -0,0 +1,58 @@ +package merkle + +import ( + "github.com/tendermint/tendermint/crypto/tmhash" +) + +// SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). +func SimpleHashFromTwoHashes(left, right []byte) []byte { + var hasher = tmhash.New() + err := encodeByteSlice(hasher, left) + if err != nil { + panic(err) + } + err = encodeByteSlice(hasher, right) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + +// SimpleHashFromHashers computes a Merkle tree from items that can be hashed. +func SimpleHashFromHashers(items []Hasher) []byte { + hashes := make([][]byte, len(items)) + for i, item := range items { + hash := item.Hash() + hashes[i] = hash + } + return simpleHashFromHashes(hashes) +} + +// SimpleHashFromMap computes a Merkle tree from sorted map. +// Like calling SimpleHashFromHashers with +// `item = []byte(Hash(key) | Hash(value))`, +// sorted by `item`. +func SimpleHashFromMap(m map[string]Hasher) []byte { + sm := newSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + return sm.Hash() +} + +//---------------------------------------------------------------- + +// Expects hashes! +func simpleHashFromHashes(hashes [][]byte) []byte { + // Recursive impl. + switch len(hashes) { + case 0: + return nil + case 1: + return hashes[0] + default: + left := simpleHashFromHashes(hashes[:(len(hashes)+1)/2]) + right := simpleHashFromHashes(hashes[(len(hashes)+1)/2:]) + return SimpleHashFromTwoHashes(left, right) + } +} diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go new file mode 100644 index 0000000..488e0c9 --- /dev/null +++ b/crypto/merkle/simple_tree_test.go @@ -0,0 +1,88 @@ +package merkle + +import ( + "bytes" + + cmn "github.com/tendermint/tendermint/libs/common" + . "github.com/tendermint/tendermint/libs/test" + + "github.com/tendermint/tendermint/crypto/tmhash" + "testing" +) + +type testItem []byte + +func (tI testItem) Hash() []byte { + return []byte(tI) +} + +func TestSimpleProof(t *testing.T) { + + total := 100 + + items := make([]Hasher, total) + for i := 0; i < total; i++ { + items[i] = testItem(cmn.RandBytes(tmhash.Size)) + } + + rootHash := SimpleHashFromHashers(items) + + rootHash2, proofs := SimpleProofsFromHashers(items) + + if !bytes.Equal(rootHash, rootHash2) { + t.Errorf("Unmatched root hashes: %X vs %X", rootHash, rootHash2) + } + + // For each item, check the trail. + for i, item := range items { + itemHash := item.Hash() + proof := proofs[i] + + // Verify success + ok := proof.Verify(i, total, itemHash, rootHash) + if !ok { + t.Errorf("Verification failed for index %v.", i) + } + + // Wrong item index should make it fail + { + ok = proof.Verify((i+1)%total, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong index %v.", i) + } + } + + // Trail too long should make it fail + origAunts := proof.Aunts + proof.Aunts = append(proof.Aunts, cmn.RandBytes(32)) + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Trail too short should make it fail + proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1] + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Mutating the itemHash should make it fail. + ok = proof.Verify(i, total, MutateByteSlice(itemHash), rootHash) + if ok { + t.Errorf("Expected verification to fail for mutated leaf hash") + } + + // Mutating the rootHash should make it fail. + ok = proof.Verify(i, total, itemHash, MutateByteSlice(rootHash)) + if ok { + t.Errorf("Expected verification to fail for mutated root hash") + } + } +} diff --git a/crypto/merkle/types.go b/crypto/merkle/types.go new file mode 100644 index 0000000..2fcb3f3 --- /dev/null +++ b/crypto/merkle/types.go @@ -0,0 +1,38 @@ +package merkle + +import ( + "io" + + amino "github.com/tendermint/go-amino" +) + +// Tree is a Merkle tree interface. +type Tree interface { + Size() (size int) + Height() (height int8) + Has(key []byte) (has bool) + Proof(key []byte) (value []byte, proof []byte, exists bool) // TODO make it return an index + Get(key []byte) (index int, value []byte, exists bool) + GetByIndex(index int) (key []byte, value []byte) + Set(key []byte, value []byte) (updated bool) + Remove(key []byte) (value []byte, removed bool) + HashWithCount() (hash []byte, count int) + Hash() (hash []byte) + Save() (hash []byte) + Load(hash []byte) + Copy() Tree + Iterate(func(key []byte, value []byte) (stop bool)) (stopped bool) + IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool) +} + +// Hasher represents a hashable piece of data which can be hashed in the Tree. +type Hasher interface { + Hash() []byte +} + +//----------------------------------------------------------------------- + +// Uvarint length prefixed byteslice +func encodeByteSlice(w io.Writer, bz []byte) (err error) { + return amino.EncodeByteSlice(w, bz) +} From 8353a8e332228f947e81e2cc8ed4e982db8247d0 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Fri, 10 Aug 2018 00:25:57 -0500 Subject: [PATCH 07/86] libs: Remove usage of custom Fmt, in favor of fmt.Sprintf (#2199) * libs: Remove usage of custom Fmt, in favor of fmt.Sprintf Closes #2193 * Fix bug that was masked by custom Fmt! --- crypto/merkle/simple_tree_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index 488e0c9..e2dccd3 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -6,8 +6,9 @@ import ( cmn "github.com/tendermint/tendermint/libs/common" . "github.com/tendermint/tendermint/libs/test" - "github.com/tendermint/tendermint/crypto/tmhash" "testing" + + "github.com/tendermint/tendermint/crypto/tmhash" ) type testItem []byte From 46002bb3d672a410f36a59caeb0d23338ded5bce Mon Sep 17 00:00:00 2001 From: Joon Date: Sat, 29 Sep 2018 09:03:19 +0900 Subject: [PATCH 08/86] General Merkle Proof (#2298) * first commit finalize rebase add protoc_merkle to Makefile * in progress * fix kvstore * fix tests * remove iavl dependency * fix tx_test * fix test_abci_cli fix test_apps * fix test_apps * fix test_cover * rm rebase residue * address comment in progress * finalize rebase --- crypto/merkle/compile.sh | 6 + crypto/merkle/merkle.pb.go | 792 +++++++++++++++++++++++++++ crypto/merkle/merkle.proto | 30 + crypto/merkle/proof.go | 132 +++++ crypto/merkle/proof_key_path.go | 107 ++++ crypto/merkle/proof_key_path_test.go | 41 ++ crypto/merkle/proof_simple_value.go | 91 +++ crypto/merkle/simple_proof.go | 53 +- crypto/merkle/simple_tree_test.go | 59 +- crypto/merkle/wire.go | 12 + 10 files changed, 1277 insertions(+), 46 deletions(-) create mode 100644 crypto/merkle/compile.sh create mode 100644 crypto/merkle/merkle.pb.go create mode 100644 crypto/merkle/merkle.proto create mode 100644 crypto/merkle/proof.go create mode 100644 crypto/merkle/proof_key_path.go create mode 100644 crypto/merkle/proof_key_path_test.go create mode 100644 crypto/merkle/proof_simple_value.go create mode 100644 crypto/merkle/wire.go diff --git a/crypto/merkle/compile.sh b/crypto/merkle/compile.sh new file mode 100644 index 0000000..8e4c739 --- /dev/null +++ b/crypto/merkle/compile.sh @@ -0,0 +1,6 @@ +#! /bin/bash + +protoc --gogo_out=. -I $GOPATH/src/ -I . -I $GOPATH/src/github.com/gogo/protobuf/protobuf merkle.proto +echo "--> adding nolint declarations to protobuf generated files" +awk '/package merkle/ { print "//nolint: gas"; print; next }1' merkle.pb.go > merkle.pb.go.new +mv merkle.pb.go.new merkle.pb.go diff --git a/crypto/merkle/merkle.pb.go b/crypto/merkle/merkle.pb.go new file mode 100644 index 0000000..75e1b08 --- /dev/null +++ b/crypto/merkle/merkle.pb.go @@ -0,0 +1,792 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: crypto/merkle/merkle.proto + +package merkle + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" + +import bytes "bytes" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +// ProofOp defines an operation used for calculating Merkle root +// The data could be arbitrary format, providing nessecary data +// for example neighbouring node hash +type ProofOp struct { + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ProofOp) Reset() { *m = ProofOp{} } +func (m *ProofOp) String() string { return proto.CompactTextString(m) } +func (*ProofOp) ProtoMessage() {} +func (*ProofOp) Descriptor() ([]byte, []int) { + return fileDescriptor_merkle_5d3f6051907285da, []int{0} +} +func (m *ProofOp) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ProofOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ProofOp.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *ProofOp) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProofOp.Merge(dst, src) +} +func (m *ProofOp) XXX_Size() int { + return m.Size() +} +func (m *ProofOp) XXX_DiscardUnknown() { + xxx_messageInfo_ProofOp.DiscardUnknown(m) +} + +var xxx_messageInfo_ProofOp proto.InternalMessageInfo + +func (m *ProofOp) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *ProofOp) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *ProofOp) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +// Proof is Merkle proof defined by the list of ProofOps +type Proof struct { + Ops []ProofOp `protobuf:"bytes,1,rep,name=ops" json:"ops"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Proof) Reset() { *m = Proof{} } +func (m *Proof) String() string { return proto.CompactTextString(m) } +func (*Proof) ProtoMessage() {} +func (*Proof) Descriptor() ([]byte, []int) { + return fileDescriptor_merkle_5d3f6051907285da, []int{1} +} +func (m *Proof) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Proof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Proof.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *Proof) XXX_Merge(src proto.Message) { + xxx_messageInfo_Proof.Merge(dst, src) +} +func (m *Proof) XXX_Size() int { + return m.Size() +} +func (m *Proof) XXX_DiscardUnknown() { + xxx_messageInfo_Proof.DiscardUnknown(m) +} + +var xxx_messageInfo_Proof proto.InternalMessageInfo + +func (m *Proof) GetOps() []ProofOp { + if m != nil { + return m.Ops + } + return nil +} + +func init() { + proto.RegisterType((*ProofOp)(nil), "merkle.ProofOp") + proto.RegisterType((*Proof)(nil), "merkle.Proof") +} +func (this *ProofOp) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ProofOp) + if !ok { + that2, ok := that.(ProofOp) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Type != that1.Type { + return false + } + if !bytes.Equal(this.Key, that1.Key) { + return false + } + if !bytes.Equal(this.Data, that1.Data) { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *Proof) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Proof) + if !ok { + that2, ok := that.(Proof) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if len(this.Ops) != len(that1.Ops) { + return false + } + for i := range this.Ops { + if !this.Ops[i].Equal(&that1.Ops[i]) { + return false + } + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (m *ProofOp) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ProofOp) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Type) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintMerkle(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) + } + if len(m.Key) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintMerkle(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } + if len(m.Data) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintMerkle(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *Proof) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Proof) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Ops) > 0 { + for _, msg := range m.Ops { + dAtA[i] = 0xa + i++ + i = encodeVarintMerkle(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func encodeVarintMerkle(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func NewPopulatedProofOp(r randyMerkle, easy bool) *ProofOp { + this := &ProofOp{} + this.Type = string(randStringMerkle(r)) + v1 := r.Intn(100) + this.Key = make([]byte, v1) + for i := 0; i < v1; i++ { + this.Key[i] = byte(r.Intn(256)) + } + v2 := r.Intn(100) + this.Data = make([]byte, v2) + for i := 0; i < v2; i++ { + this.Data[i] = byte(r.Intn(256)) + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedMerkle(r, 4) + } + return this +} + +func NewPopulatedProof(r randyMerkle, easy bool) *Proof { + this := &Proof{} + if r.Intn(10) != 0 { + v3 := r.Intn(5) + this.Ops = make([]ProofOp, v3) + for i := 0; i < v3; i++ { + v4 := NewPopulatedProofOp(r, easy) + this.Ops[i] = *v4 + } + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedMerkle(r, 2) + } + return this +} + +type randyMerkle interface { + Float32() float32 + Float64() float64 + Int63() int64 + Int31() int32 + Uint32() uint32 + Intn(n int) int +} + +func randUTF8RuneMerkle(r randyMerkle) rune { + ru := r.Intn(62) + if ru < 10 { + return rune(ru + 48) + } else if ru < 36 { + return rune(ru + 55) + } + return rune(ru + 61) +} +func randStringMerkle(r randyMerkle) string { + v5 := r.Intn(100) + tmps := make([]rune, v5) + for i := 0; i < v5; i++ { + tmps[i] = randUTF8RuneMerkle(r) + } + return string(tmps) +} +func randUnrecognizedMerkle(r randyMerkle, maxFieldNumber int) (dAtA []byte) { + l := r.Intn(5) + for i := 0; i < l; i++ { + wire := r.Intn(4) + if wire == 3 { + wire = 5 + } + fieldNumber := maxFieldNumber + r.Intn(100) + dAtA = randFieldMerkle(dAtA, r, fieldNumber, wire) + } + return dAtA +} +func randFieldMerkle(dAtA []byte, r randyMerkle, fieldNumber int, wire int) []byte { + key := uint32(fieldNumber)<<3 | uint32(wire) + switch wire { + case 0: + dAtA = encodeVarintPopulateMerkle(dAtA, uint64(key)) + v6 := r.Int63() + if r.Intn(2) == 0 { + v6 *= -1 + } + dAtA = encodeVarintPopulateMerkle(dAtA, uint64(v6)) + case 1: + dAtA = encodeVarintPopulateMerkle(dAtA, uint64(key)) + dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) + case 2: + dAtA = encodeVarintPopulateMerkle(dAtA, uint64(key)) + ll := r.Intn(100) + dAtA = encodeVarintPopulateMerkle(dAtA, uint64(ll)) + for j := 0; j < ll; j++ { + dAtA = append(dAtA, byte(r.Intn(256))) + } + default: + dAtA = encodeVarintPopulateMerkle(dAtA, uint64(key)) + dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) + } + return dAtA +} +func encodeVarintPopulateMerkle(dAtA []byte, v uint64) []byte { + for v >= 1<<7 { + dAtA = append(dAtA, uint8(uint64(v)&0x7f|0x80)) + v >>= 7 + } + dAtA = append(dAtA, uint8(v)) + return dAtA +} +func (m *ProofOp) Size() (n int) { + var l int + _ = l + l = len(m.Type) + if l > 0 { + n += 1 + l + sovMerkle(uint64(l)) + } + l = len(m.Key) + if l > 0 { + n += 1 + l + sovMerkle(uint64(l)) + } + l = len(m.Data) + if l > 0 { + n += 1 + l + sovMerkle(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Proof) Size() (n int) { + var l int + _ = l + if len(m.Ops) > 0 { + for _, e := range m.Ops { + l = e.Size() + n += 1 + l + sovMerkle(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovMerkle(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozMerkle(x uint64) (n int) { + return sovMerkle(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *ProofOp) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMerkle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ProofOp: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ProofOp: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMerkle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMerkle + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Type = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMerkle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMerkle + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) + if m.Key == nil { + m.Key = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMerkle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMerkle + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMerkle(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMerkle + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Proof) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMerkle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Proof: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Proof: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMerkle + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMerkle + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ops = append(m.Ops, ProofOp{}) + if err := m.Ops[len(m.Ops)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMerkle(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMerkle + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipMerkle(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMerkle + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMerkle + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMerkle + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthMerkle + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMerkle + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipMerkle(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthMerkle = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowMerkle = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_merkle_5d3f6051907285da) } + +var fileDescriptor_merkle_5d3f6051907285da = []byte{ + // 200 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c, + 0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25, + 0xf9, 0x42, 0x6c, 0x10, 0x9e, 0x94, 0x6e, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, + 0xae, 0x7e, 0x7a, 0x7e, 0x7a, 0xbe, 0x3e, 0x58, 0x3a, 0xa9, 0x34, 0x0d, 0xcc, 0x03, 0x73, 0xc0, + 0x2c, 0x88, 0x36, 0x25, 0x67, 0x2e, 0xf6, 0x80, 0xa2, 0xfc, 0xfc, 0x34, 0xff, 0x02, 0x21, 0x21, + 0x2e, 0x96, 0x92, 0xca, 0x82, 0x54, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, 0x20, 0x30, 0x5b, 0x48, + 0x80, 0x8b, 0x39, 0x3b, 0xb5, 0x52, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xc4, 0x04, 0xa9, + 0x4a, 0x49, 0x2c, 0x49, 0x94, 0x60, 0x06, 0x0b, 0x81, 0xd9, 0x4a, 0x06, 0x5c, 0xac, 0x60, 0x43, + 0x84, 0xd4, 0xb9, 0x98, 0xf3, 0x0b, 0x8a, 0x25, 0x18, 0x15, 0x98, 0x35, 0xb8, 0x8d, 0xf8, 0xf5, + 0xa0, 0x0e, 0x84, 0x5a, 0xe0, 0xc4, 0x72, 0xe2, 0x9e, 0x3c, 0x43, 0x10, 0x48, 0x85, 0x93, 0xc8, + 0x8f, 0x87, 0x72, 0x8c, 0x2b, 0x1e, 0xc9, 0x31, 0x9e, 0x78, 0x24, 0xc7, 0x78, 0xe1, 0x91, 0x1c, + 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x49, 0x6c, 0x60, 0x37, 0x19, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, + 0xb9, 0x2b, 0x0f, 0xd1, 0xe8, 0x00, 0x00, 0x00, +} diff --git a/crypto/merkle/merkle.proto b/crypto/merkle/merkle.proto new file mode 100644 index 0000000..8a6c467 --- /dev/null +++ b/crypto/merkle/merkle.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; +package merkle; + +// For more information on gogo.proto, see: +// https://github.com/gogo/protobuf/blob/master/extensions.md +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (gogoproto.sizer_all) = true; + +option (gogoproto.populate_all) = true; +option (gogoproto.equal_all) = true; + +//---------------------------------------- +// Message types + +// ProofOp defines an operation used for calculating Merkle root +// The data could be arbitrary format, providing nessecary data +// for example neighbouring node hash +message ProofOp { + string type = 1; + bytes key = 2; + bytes data = 3; +} + +// Proof is Merkle proof defined by the list of ProofOps +message Proof { + repeated ProofOp ops = 1 [(gogoproto.nullable)=false]; +} diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go new file mode 100644 index 0000000..7da8949 --- /dev/null +++ b/crypto/merkle/proof.go @@ -0,0 +1,132 @@ +package merkle + +import ( + "bytes" + + cmn "github.com/tendermint/tmlibs/common" +) + +//---------------------------------------- +// ProofOp gets converted to an instance of ProofOperator: + +// ProofOperator is a layer for calculating intermediate Merkle root +// Run() takes a list of bytes because it can be more than one +// for example in range proofs +// ProofOp() defines custom encoding which can be decoded later with +// OpDecoder +type ProofOperator interface { + Run([][]byte) ([][]byte, error) + GetKey() []byte + ProofOp() ProofOp +} + +//---------------------------------------- +// Operations on a list of ProofOperators + +// ProofOperators is a slice of ProofOperator(s) +// Each operator will be applied to the input value sequencially +// and the last Merkle root will be verified with already known data +type ProofOperators []ProofOperator + +func (poz ProofOperators) VerifyValue(root []byte, keypath string, value []byte) (err error) { + return poz.Verify(root, keypath, [][]byte{value}) +} + +func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (err error) { + keys, err := KeyPathToKeys(keypath) + if err != nil { + return + } + + for i, op := range poz { + key := op.GetKey() + if len(key) != 0 { + if !bytes.Equal(keys[0], key) { + return cmn.NewError("Key mismatch on operation #%d: expected %+v but %+v", i, []byte(keys[0]), []byte(key)) + } + keys = keys[1:] + } + args, err = op.Run(args) + if err != nil { + return + } + } + if !bytes.Equal(root, args[0]) { + return cmn.NewError("Calculated root hash is invalid: expected %+v but %+v", root, args[0]) + } + if len(keys) != 0 { + return cmn.NewError("Keypath not consumed all") + } + return nil +} + +//---------------------------------------- +// ProofRuntime - main entrypoint + +type OpDecoder func(ProofOp) (ProofOperator, error) + +type ProofRuntime struct { + decoders map[string]OpDecoder +} + +func NewProofRuntime() *ProofRuntime { + return &ProofRuntime{ + decoders: make(map[string]OpDecoder), + } +} + +func (prt *ProofRuntime) RegisterOpDecoder(typ string, dec OpDecoder) { + _, ok := prt.decoders[typ] + if ok { + panic("already registered for type " + typ) + } + prt.decoders[typ] = dec +} + +func (prt *ProofRuntime) Decode(pop ProofOp) (ProofOperator, error) { + decoder := prt.decoders[pop.Type] + if decoder == nil { + return nil, cmn.NewError("unrecognized proof type %v", pop.Type) + } + return decoder(pop) +} + +func (prt *ProofRuntime) DecodeProof(proof *Proof) (poz ProofOperators, err error) { + poz = ProofOperators(nil) + for _, pop := range proof.Ops { + operator, err := prt.Decode(pop) + if err != nil { + return nil, cmn.ErrorWrap(err, "decoding a proof operator") + } + poz = append(poz, operator) + } + return +} + +func (prt *ProofRuntime) VerifyValue(proof *Proof, root []byte, keypath string, value []byte) (err error) { + return prt.Verify(proof, root, keypath, [][]byte{value}) +} + +// TODO In the long run we'll need a method of classifcation of ops, +// whether existence or absence or perhaps a third? +func (prt *ProofRuntime) VerifyAbsence(proof *Proof, root []byte, keypath string) (err error) { + return prt.Verify(proof, root, keypath, nil) +} + +func (prt *ProofRuntime) Verify(proof *Proof, root []byte, keypath string, args [][]byte) (err error) { + poz, err := prt.DecodeProof(proof) + if err != nil { + return cmn.ErrorWrap(err, "decoding proof") + } + return poz.Verify(root, keypath, args) +} + +// DefaultProofRuntime only knows about Simple value +// proofs. +// To use e.g. IAVL proofs, register op-decoders as +// defined in the IAVL package. +func DefaultProofRuntime() (prt *ProofRuntime) { + prt = NewProofRuntime() + prt.RegisterOpDecoder(ProofOpSimpleValue, SimpleValueOpDecoder) + return +} diff --git a/crypto/merkle/proof_key_path.go b/crypto/merkle/proof_key_path.go new file mode 100644 index 0000000..d74dac5 --- /dev/null +++ b/crypto/merkle/proof_key_path.go @@ -0,0 +1,107 @@ +package merkle + +import ( + "encoding/hex" + "fmt" + "net/url" + "strings" + + cmn "github.com/tendermint/tendermint/libs/common" +) + +/* + + For generalized Merkle proofs, each layer of the proof may require an + optional key. The key may be encoded either by URL-encoding or + (upper-case) hex-encoding. + TODO: In the future, more encodings may be supported, like base32 (e.g. + /32:) + + For example, for a Cosmos-SDK application where the first two proof layers + are SimpleValueOps, and the third proof layer is an IAVLValueOp, the keys + might look like: + + 0: []byte("App") + 1: []byte("IBC") + 2: []byte{0x01, 0x02, 0x03} + + Assuming that we know that the first two layers are always ASCII texts, we + probably want to use URLEncoding for those, whereas the third layer will + require HEX encoding for efficient representation. + + kp := new(KeyPath) + kp.AppendKey([]byte("App"), KeyEncodingURL) + kp.AppendKey([]byte("IBC"), KeyEncodingURL) + kp.AppendKey([]byte{0x01, 0x02, 0x03}, KeyEncodingURL) + kp.String() // Should return "/App/IBC/x:010203" + + NOTE: All encodings *MUST* work compatibly, such that you can choose to use + whatever encoding, and the decoded keys will always be the same. In other + words, it's just as good to encode all three keys using URL encoding or HEX + encoding... it just wouldn't be optimal in terms of readability or space + efficiency. + + NOTE: Punycode will never be supported here, because not all values can be + decoded. For example, no string decodes to the string "xn--blah" in + Punycode. + +*/ + +type keyEncoding int + +const ( + KeyEncodingURL keyEncoding = iota + KeyEncodingHex + KeyEncodingMax +) + +type Key struct { + name []byte + enc keyEncoding +} + +type KeyPath []Key + +func (pth KeyPath) AppendKey(key []byte, enc keyEncoding) KeyPath { + return append(pth, Key{key, enc}) +} + +func (pth KeyPath) String() string { + res := "" + for _, key := range pth { + switch key.enc { + case KeyEncodingURL: + res += "/" + url.PathEscape(string(key.name)) + case KeyEncodingHex: + res += "/x:" + fmt.Sprintf("%X", key.name) + default: + panic("unexpected key encoding type") + } + } + return res +} + +func KeyPathToKeys(path string) (keys [][]byte, err error) { + if path == "" || path[0] != '/' { + return nil, cmn.NewError("key path string must start with a forward slash '/'") + } + parts := strings.Split(path[1:], "/") + keys = make([][]byte, len(parts)) + for i, part := range parts { + if strings.HasPrefix(part, "x:") { + hexPart := part[2:] + key, err := hex.DecodeString(hexPart) + if err != nil { + return nil, cmn.ErrorWrap(err, "decoding hex-encoded part #%d: /%s", i, part) + } + keys[i] = key + } else { + key, err := url.PathUnescape(part) + if err != nil { + return nil, cmn.ErrorWrap(err, "decoding url-encoded part #%d: /%s", i, part) + } + keys[i] = []byte(key) // TODO Test this with random bytes, I'm not sure that it works for arbitrary bytes... + } + } + return keys, nil +} diff --git a/crypto/merkle/proof_key_path_test.go b/crypto/merkle/proof_key_path_test.go new file mode 100644 index 0000000..48fda30 --- /dev/null +++ b/crypto/merkle/proof_key_path_test.go @@ -0,0 +1,41 @@ +package merkle + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestKeyPath(t *testing.T) { + var path KeyPath + keys := make([][]byte, 10) + alphanum := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + + for d := 0; d < 1e4; d++ { + path = nil + + for i := range keys { + enc := keyEncoding(rand.Intn(int(KeyEncodingMax))) + keys[i] = make([]byte, rand.Uint32()%20) + switch enc { + case KeyEncodingURL: + for j := range keys[i] { + keys[i][j] = alphanum[rand.Intn(len(alphanum))] + } + case KeyEncodingHex: + rand.Read(keys[i]) + default: + panic("Unexpected encoding") + } + path = path.AppendKey(keys[i], enc) + } + + res, err := KeyPathToKeys(path.String()) + require.Nil(t, err) + + for i, key := range keys { + require.Equal(t, key, res[i]) + } + } +} diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_simple_value.go new file mode 100644 index 0000000..28935e2 --- /dev/null +++ b/crypto/merkle/proof_simple_value.go @@ -0,0 +1,91 @@ +package merkle + +import ( + "bytes" + "fmt" + + "github.com/tendermint/tendermint/crypto/tmhash" + cmn "github.com/tendermint/tendermint/libs/common" +) + +const ProofOpSimpleValue = "simple:v" + +// SimpleValueOp takes a key and a single value as argument and +// produces the root hash. The corresponding tree structure is +// the SimpleMap tree. SimpleMap takes a Hasher, and currently +// Tendermint uses aminoHasher. SimpleValueOp should support +// the hash function as used in aminoHasher. TODO support +// additional hash functions here as options/args to this +// operator. +// +// If the produced root hash matches the expected hash, the +// proof is good. +type SimpleValueOp struct { + // Encoded in ProofOp.Key. + key []byte + + // To encode in ProofOp.Data + Proof *SimpleProof `json:"simple-proof"` +} + +var _ ProofOperator = SimpleValueOp{} + +func NewSimpleValueOp(key []byte, proof *SimpleProof) SimpleValueOp { + return SimpleValueOp{ + key: key, + Proof: proof, + } +} + +func SimpleValueOpDecoder(pop ProofOp) (ProofOperator, error) { + if pop.Type != ProofOpSimpleValue { + return nil, cmn.NewError("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpSimpleValue) + } + var op SimpleValueOp // a bit strange as we'll discard this, but it works. + err := cdc.UnmarshalBinary(pop.Data, &op) + if err != nil { + return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") + } + return NewSimpleValueOp(pop.Key, op.Proof), nil +} + +func (op SimpleValueOp) ProofOp() ProofOp { + bz := cdc.MustMarshalBinary(op) + return ProofOp{ + Type: ProofOpSimpleValue, + Key: op.key, + Data: bz, + } +} + +func (op SimpleValueOp) String() string { + return fmt.Sprintf("SimpleValueOp{%v}", op.GetKey()) +} + +func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { + if len(args) != 1 { + return nil, cmn.NewError("expected 1 arg, got %v", len(args)) + } + value := args[0] + hasher := tmhash.New() + hasher.Write(value) // does not error + vhash := hasher.Sum(nil) + + // Wrap to hash the KVPair. + hasher = tmhash.New() + encodeByteSlice(hasher, []byte(op.key)) // does not error + encodeByteSlice(hasher, []byte(vhash)) // does not error + kvhash := hasher.Sum(nil) + + if !bytes.Equal(kvhash, op.Proof.LeafHash) { + return nil, cmn.NewError("leaf hash mismatch: want %X got %X", op.Proof.LeafHash, kvhash) + } + + return [][]byte{ + op.Proof.ComputeRootHash(), + }, nil +} + +func (op SimpleValueOp) GetKey() []byte { + return op.key +} diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 2541b6d..306505f 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -2,12 +2,24 @@ package merkle import ( "bytes" + "errors" "fmt" + + cmn "github.com/tendermint/tendermint/libs/common" ) -// SimpleProof represents a simple merkle proof. +// SimpleProof represents a simple Merkle proof. +// NOTE: The convention for proofs is to include leaf hashes but to +// exclude the root hash. +// This convention is implemented across IAVL range proofs as well. +// Keep this consistent unless there's a very good reason to change +// everything. This also affects the generalized proof system as +// well. type SimpleProof struct { - Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. + Total int `json:"total"` // Total number of items. + Index int `json:"index"` // Index of item to prove. + LeafHash []byte `json:"leaf_hash"` // Hash of item value. + Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. } // SimpleProofsFromHashers computes inclusion proof for given items. @@ -18,7 +30,10 @@ func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleP proofs = make([]*SimpleProof, len(items)) for i, trail := range trails { proofs[i] = &SimpleProof{ - Aunts: trail.FlattenAunts(), + Total: len(items), + Index: i, + LeafHash: trail.Hash, + Aunts: trail.FlattenAunts(), } } return @@ -49,11 +64,33 @@ func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[strin return } -// Verify that leafHash is a leaf hash of the simple-merkle-tree -// which hashes to rootHash. -func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash []byte) bool { - computedHash := computeHashFromAunts(index, total, leafHash, sp.Aunts) - return computedHash != nil && bytes.Equal(computedHash, rootHash) +// Verify that the SimpleProof proves the root hash. +// Check sp.Index/sp.Total manually if needed +func (sp *SimpleProof) Verify(rootHash []byte, leafHash []byte) error { + if sp.Total < 0 { + return errors.New("Proof total must be positive") + } + if sp.Index < 0 { + return errors.New("Proof index cannot be negative") + } + if !bytes.Equal(sp.LeafHash, leafHash) { + return cmn.NewError("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) + } + computedHash := sp.ComputeRootHash() + if !bytes.Equal(computedHash, rootHash) { + return cmn.NewError("invalid root hash: wanted %X got %X", rootHash, computedHash) + } + return nil +} + +// Compute the root hash given a leaf hash. Does not verify the result. +func (sp *SimpleProof) ComputeRootHash() []byte { + return computeHashFromAunts( + sp.Index, + sp.Total, + sp.LeafHash, + sp.Aunts, + ) } // String implements the stringer interface for SimpleProof. diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index e2dccd3..b299aba 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -1,13 +1,13 @@ package merkle import ( - "bytes" + "testing" + + "github.com/stretchr/testify/require" cmn "github.com/tendermint/tendermint/libs/common" . "github.com/tendermint/tendermint/libs/test" - "testing" - "github.com/tendermint/tendermint/crypto/tmhash" ) @@ -30,60 +30,43 @@ func TestSimpleProof(t *testing.T) { rootHash2, proofs := SimpleProofsFromHashers(items) - if !bytes.Equal(rootHash, rootHash2) { - t.Errorf("Unmatched root hashes: %X vs %X", rootHash, rootHash2) - } + require.Equal(t, rootHash, rootHash2, "Unmatched root hashes: %X vs %X", rootHash, rootHash2) // For each item, check the trail. for i, item := range items { itemHash := item.Hash() proof := proofs[i] + // Check total/index + require.Equal(t, proof.Index, i, "Unmatched indicies: %d vs %d", proof.Index, i) + + require.Equal(t, proof.Total, total, "Unmatched totals: %d vs %d", proof.Total, total) + // Verify success - ok := proof.Verify(i, total, itemHash, rootHash) - if !ok { - t.Errorf("Verification failed for index %v.", i) - } - - // Wrong item index should make it fail - { - ok = proof.Verify((i+1)%total, total, itemHash, rootHash) - if ok { - t.Errorf("Expected verification to fail for wrong index %v.", i) - } - } + err := proof.Verify(rootHash, itemHash) + require.NoError(t, err, "Verificatior failed: %v.", err) // Trail too long should make it fail origAunts := proof.Aunts proof.Aunts = append(proof.Aunts, cmn.RandBytes(32)) - { - ok = proof.Verify(i, total, itemHash, rootHash) - if ok { - t.Errorf("Expected verification to fail for wrong trail length.") - } - } + err = proof.Verify(rootHash, itemHash) + require.Error(t, err, "Expected verification to fail for wrong trail length") + proof.Aunts = origAunts // Trail too short should make it fail proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1] - { - ok = proof.Verify(i, total, itemHash, rootHash) - if ok { - t.Errorf("Expected verification to fail for wrong trail length.") - } - } + err = proof.Verify(rootHash, itemHash) + require.Error(t, err, "Expected verification to fail for wrong trail length") + proof.Aunts = origAunts // Mutating the itemHash should make it fail. - ok = proof.Verify(i, total, MutateByteSlice(itemHash), rootHash) - if ok { - t.Errorf("Expected verification to fail for mutated leaf hash") - } + err = proof.Verify(rootHash, MutateByteSlice(itemHash)) + require.Error(t, err, "Expected verification to fail for mutated leaf hash") // Mutating the rootHash should make it fail. - ok = proof.Verify(i, total, itemHash, MutateByteSlice(rootHash)) - if ok { - t.Errorf("Expected verification to fail for mutated root hash") - } + err = proof.Verify(MutateByteSlice(rootHash), itemHash) + require.Error(t, err, "Expected verification to fail for mutated root hash") } } diff --git a/crypto/merkle/wire.go b/crypto/merkle/wire.go new file mode 100644 index 0000000..c20ec9a --- /dev/null +++ b/crypto/merkle/wire.go @@ -0,0 +1,12 @@ +package merkle + +import ( + "github.com/tendermint/go-amino" +) + +var cdc *amino.Codec + +func init() { + cdc = amino.NewCodec() + cdc.Seal() +} From dc43d99e3bfbd93ccbb4f75eaa94c41fb5ea99ca Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 28 Sep 2018 23:32:13 -0400 Subject: [PATCH 09/86] General Merkle Follow Up (#2510) * tmlibs -> libs * update changelog * address some comments from review of #2298 --- crypto/merkle/proof.go | 24 +++++++++++++----------- crypto/merkle/proof_key_path.go | 6 +++++- crypto/merkle/proof_simple_value.go | 2 +- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 7da8949..3059ed3 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -3,17 +3,19 @@ package merkle import ( "bytes" - cmn "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tendermint/libs/common" ) //---------------------------------------- // ProofOp gets converted to an instance of ProofOperator: -// ProofOperator is a layer for calculating intermediate Merkle root -// Run() takes a list of bytes because it can be more than one -// for example in range proofs -// ProofOp() defines custom encoding which can be decoded later with -// OpDecoder +// ProofOperator is a layer for calculating intermediate Merkle roots +// when a series of Merkle trees are chained together. +// Run() takes leaf values from a tree and returns the Merkle +// root for the corresponding tree. It takes and returns a list of bytes +// to allow multiple leaves to be part of a single proof, for instance in a range proof. +// ProofOp() encodes the ProofOperator in a generic way so it can later be +// decoded with OpDecoder. type ProofOperator interface { Run([][]byte) ([][]byte, error) GetKey() []byte @@ -23,8 +25,8 @@ type ProofOperator interface { //---------------------------------------- // Operations on a list of ProofOperators -// ProofOperators is a slice of ProofOperator(s) -// Each operator will be applied to the input value sequencially +// ProofOperators is a slice of ProofOperator(s). +// Each operator will be applied to the input value sequentially // and the last Merkle root will be verified with already known data type ProofOperators []ProofOperator @@ -91,8 +93,8 @@ func (prt *ProofRuntime) Decode(pop ProofOp) (ProofOperator, error) { return decoder(pop) } -func (prt *ProofRuntime) DecodeProof(proof *Proof) (poz ProofOperators, err error) { - poz = ProofOperators(nil) +func (prt *ProofRuntime) DecodeProof(proof *Proof) (ProofOperators, error) { + var poz ProofOperators for _, pop := range proof.Ops { operator, err := prt.Decode(pop) if err != nil { @@ -100,7 +102,7 @@ func (prt *ProofRuntime) DecodeProof(proof *Proof) (poz ProofOperators, err erro } poz = append(poz, operator) } - return + return poz, nil } func (prt *ProofRuntime) VerifyValue(proof *Proof, root []byte, keypath string, value []byte) (err error) { diff --git a/crypto/merkle/proof_key_path.go b/crypto/merkle/proof_key_path.go index d74dac5..aec93e8 100644 --- a/crypto/merkle/proof_key_path.go +++ b/crypto/merkle/proof_key_path.go @@ -35,6 +35,8 @@ import ( kp.AppendKey([]byte{0x01, 0x02, 0x03}, KeyEncodingURL) kp.String() // Should return "/App/IBC/x:010203" + NOTE: Key paths must begin with a `/`. + NOTE: All encodings *MUST* work compatibly, such that you can choose to use whatever encoding, and the decoded keys will always be the same. In other words, it's just as good to encode all three keys using URL encoding or HEX @@ -52,7 +54,7 @@ type keyEncoding int const ( KeyEncodingURL keyEncoding = iota KeyEncodingHex - KeyEncodingMax + KeyEncodingMax // Number of known encodings. Used for testing ) type Key struct { @@ -81,6 +83,8 @@ func (pth KeyPath) String() string { return res } +// Decode a path to a list of keys. Path must begin with `/`. +// Each key must use a known encoding. func KeyPathToKeys(path string) (keys [][]byte, err error) { if path == "" || path[0] != '/' { return nil, cmn.NewError("key path string must start with a forward slash '/'") diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_simple_value.go index 28935e2..5b7b523 100644 --- a/crypto/merkle/proof_simple_value.go +++ b/crypto/merkle/proof_simple_value.go @@ -25,7 +25,7 @@ type SimpleValueOp struct { key []byte // To encode in ProofOp.Data - Proof *SimpleProof `json:"simple-proof"` + Proof *SimpleProof `json:"simple_proof"` } var _ ProofOperator = SimpleValueOp{} From 32666bac20edc8e3873a9f8ce7ed1e04e4a74292 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Wed, 10 Oct 2018 09:46:09 -0700 Subject: [PATCH 10/86] crypto/merkle: Remove byter in favor of plain byte slices (#2595) * crypto/merkle: Remove byter in favor of plain byte slices This PR is fully backwards compatible in terms of function output! (The Go API differs though) The only test case changes was to refactor it to be table driven. * Update godocs per review comments --- crypto/merkle/simple_map.go | 28 +++++++++------ crypto/merkle/simple_map_test.go | 59 ++++++++++--------------------- crypto/merkle/simple_proof.go | 25 ++++++------- crypto/merkle/simple_tree.go | 9 ++--- crypto/merkle/simple_tree_test.go | 8 ++--- crypto/merkle/types.go | 5 --- 6 files changed, 58 insertions(+), 76 deletions(-) diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go index ba4b930..bde4422 100644 --- a/crypto/merkle/simple_map.go +++ b/crypto/merkle/simple_map.go @@ -1,6 +1,9 @@ package merkle import ( + "bytes" + + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -20,14 +23,15 @@ func newSimpleMap() *simpleMap { } } -// Set hashes the key and value and appends it to the kv pairs. -func (sm *simpleMap) Set(key string, value Hasher) { +// Set creates a kv pair of the key and the hash of the value, +// and then appends it to simpleMap's kv pairs. +func (sm *simpleMap) Set(key string, value []byte) { sm.sorted = false // The value is hashed, so you can // check for equality with a cached value (say) // and make a determination to fetch or not. - vhash := value.Hash() + vhash := tmhash.Sum(value) sm.kvs = append(sm.kvs, cmn.KVPair{ Key: []byte(key), @@ -66,23 +70,25 @@ func (sm *simpleMap) KVPairs() cmn.KVPairs { // then hashed. type KVPair cmn.KVPair -func (kv KVPair) Hash() []byte { - hasher := tmhash.New() - err := encodeByteSlice(hasher, kv.Key) +// Bytes returns key || value, with both the +// key and value length prefixed. +func (kv KVPair) Bytes() []byte { + var b bytes.Buffer + err := amino.EncodeByteSlice(&b, kv.Key) if err != nil { panic(err) } - err = encodeByteSlice(hasher, kv.Value) + err = amino.EncodeByteSlice(&b, kv.Value) if err != nil { panic(err) } - return hasher.Sum(nil) + return b.Bytes() } func hashKVPairs(kvs cmn.KVPairs) []byte { - kvsH := make([]Hasher, len(kvs)) + kvsH := make([][]byte, len(kvs)) for i, kvp := range kvs { - kvsH[i] = KVPair(kvp) + kvsH[i] = KVPair(kvp).Bytes() } - return SimpleHashFromHashers(kvsH) + return SimpleHashFromByteSlices(kvsH) } diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go index 34febcf..bc095c0 100644 --- a/crypto/merkle/simple_map_test.go +++ b/crypto/merkle/simple_map_test.go @@ -5,50 +5,29 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/tendermint/tendermint/crypto/tmhash" ) -type strHasher string - -func (str strHasher) Hash() []byte { - return tmhash.Sum([]byte(str)) -} - func TestSimpleMap(t *testing.T) { - { - db := newSimpleMap() - db.Set("key1", strHasher("value1")) - assert.Equal(t, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key1", strHasher("value2")) - assert.Equal(t, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key1", strHasher("value1")) - db.Set("key2", strHasher("value2")) - assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key2", strHasher("value2")) // NOTE: out of order - db.Set("key1", strHasher("value1")) - assert.Equal(t, "eff12d1c703a1022ab509287c0f196130123d786", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") - } - { - db := newSimpleMap() - db.Set("key1", strHasher("value1")) - db.Set("key2", strHasher("value2")) - db.Set("key3", strHasher("value3")) - assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + tests := []struct { + keys []string + values []string // each string gets converted to []byte in test + want string + }{ + {[]string{"key1"}, []string{"value1"}, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f"}, + {[]string{"key1"}, []string{"value2"}, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8"}, + // swap order with 2 keys + {[]string{"key1", "key2"}, []string{"value1", "value2"}, "eff12d1c703a1022ab509287c0f196130123d786"}, + {[]string{"key2", "key1"}, []string{"value2", "value1"}, "eff12d1c703a1022ab509287c0f196130123d786"}, + // swap order with 3 keys + {[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"}, + {[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"}, } - { + for i, tc := range tests { db := newSimpleMap() - db.Set("key2", strHasher("value2")) // NOTE: out of order - db.Set("key1", strHasher("value1")) - db.Set("key3", strHasher("value3")) - assert.Equal(t, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + for i := 0; i < len(tc.keys); i++ { + db.Set(tc.keys[i], []byte(tc.values[i])) + } + got := db.Hash() + assert.Equal(t, tc.want, fmt.Sprintf("%x", got), "Hash didn't match on tc %d", i) } } diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 306505f..d2cbb12 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -22,10 +23,10 @@ type SimpleProof struct { Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. } -// SimpleProofsFromHashers computes inclusion proof for given items. +// SimpleProofsFromByteSlices computes inclusion proof for given items. // proofs[0] is the proof for items[0]. -func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) { - trails, rootSPN := trailsFromHashers(items) +func SimpleProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*SimpleProof) { + trails, rootSPN := trailsFromByteSlices(items) rootHash = rootSPN.Hash proofs = make([]*SimpleProof, len(items)) for i, trail := range trails { @@ -42,19 +43,19 @@ func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleP // SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values // in the underlying key-value pairs. // The keys are sorted before the proofs are computed. -func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) { +func SimpleProofsFromMap(m map[string][]byte) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) { sm := newSimpleMap() for k, v := range m { sm.Set(k, v) } sm.Sort() kvs := sm.kvs - kvsH := make([]Hasher, 0, len(kvs)) - for _, kvp := range kvs { - kvsH = append(kvsH, KVPair(kvp)) + kvsBytes := make([][]byte, len(kvs)) + for i, kvp := range kvs { + kvsBytes[i] = KVPair(kvp).Bytes() } - rootHash, proofList := SimpleProofsFromHashers(kvsH) + rootHash, proofList := SimpleProofsFromByteSlices(kvsBytes) proofs = make(map[string]*SimpleProof) keys = make([]string, len(proofList)) for i, kvp := range kvs { @@ -175,17 +176,17 @@ func (spn *SimpleProofNode) FlattenAunts() [][]byte { // trails[0].Hash is the leaf hash for items[0]. // trails[i].Parent.Parent....Parent == root for all i. -func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) { +func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *SimpleProofNode) { // Recursive impl. switch len(items) { case 0: return nil, nil case 1: - trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil} + trail := &SimpleProofNode{tmhash.Sum(items[0]), nil, nil, nil} return []*SimpleProofNode{trail}, trail default: - lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2]) - rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:]) + lefts, leftRoot := trailsFromByteSlices(items[:(len(items)+1)/2]) + rights, rightRoot := trailsFromByteSlices(items[(len(items)+1)/2:]) rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) root := &SimpleProofNode{rootHash, nil, nil, nil} leftRoot.Parent = root diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 46a0759..9677aef 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -18,11 +18,12 @@ func SimpleHashFromTwoHashes(left, right []byte) []byte { return hasher.Sum(nil) } -// SimpleHashFromHashers computes a Merkle tree from items that can be hashed. -func SimpleHashFromHashers(items []Hasher) []byte { +// SimpleHashFromByteSlices computes a Merkle tree where the leaves are the byte slice, +// in the provided order. +func SimpleHashFromByteSlices(items [][]byte) []byte { hashes := make([][]byte, len(items)) for i, item := range items { - hash := item.Hash() + hash := tmhash.Sum(item) hashes[i] = hash } return simpleHashFromHashes(hashes) @@ -32,7 +33,7 @@ func SimpleHashFromHashers(items []Hasher) []byte { // Like calling SimpleHashFromHashers with // `item = []byte(Hash(key) | Hash(value))`, // sorted by `item`. -func SimpleHashFromMap(m map[string]Hasher) []byte { +func SimpleHashFromMap(m map[string][]byte) []byte { sm := newSimpleMap() for k, v := range m { sm.Set(k, v) diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index b299aba..32edc65 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -21,20 +21,20 @@ func TestSimpleProof(t *testing.T) { total := 100 - items := make([]Hasher, total) + items := make([][]byte, total) for i := 0; i < total; i++ { items[i] = testItem(cmn.RandBytes(tmhash.Size)) } - rootHash := SimpleHashFromHashers(items) + rootHash := SimpleHashFromByteSlices(items) - rootHash2, proofs := SimpleProofsFromHashers(items) + rootHash2, proofs := SimpleProofsFromByteSlices(items) require.Equal(t, rootHash, rootHash2, "Unmatched root hashes: %X vs %X", rootHash, rootHash2) // For each item, check the trail. for i, item := range items { - itemHash := item.Hash() + itemHash := tmhash.Sum(item) proof := proofs[i] // Check total/index diff --git a/crypto/merkle/types.go b/crypto/merkle/types.go index 2fcb3f3..97a4787 100644 --- a/crypto/merkle/types.go +++ b/crypto/merkle/types.go @@ -25,11 +25,6 @@ type Tree interface { IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool) } -// Hasher represents a hashable piece of data which can be hashed in the Tree. -type Hasher interface { - Hash() []byte -} - //----------------------------------------------------------------------- // Uvarint length prefixed byteslice From f371b5b0750da6285d6a87e4013170ec1a155caf Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Fri, 12 Oct 2018 14:48:00 -0700 Subject: [PATCH 11/86] Remove unnecessary layer of indirection / unnecessary allocation of hashes (#2620) --- crypto/merkle/simple_tree.go | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 9677aef..45e0c5c 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -21,12 +21,16 @@ func SimpleHashFromTwoHashes(left, right []byte) []byte { // SimpleHashFromByteSlices computes a Merkle tree where the leaves are the byte slice, // in the provided order. func SimpleHashFromByteSlices(items [][]byte) []byte { - hashes := make([][]byte, len(items)) - for i, item := range items { - hash := tmhash.Sum(item) - hashes[i] = hash + switch len(items) { + case 0: + return nil + case 1: + return tmhash.Sum(items[0]) + default: + left := SimpleHashFromByteSlices(items[:(len(items)+1)/2]) + right := SimpleHashFromByteSlices(items[(len(items)+1)/2:]) + return SimpleHashFromTwoHashes(left, right) } - return simpleHashFromHashes(hashes) } // SimpleHashFromMap computes a Merkle tree from sorted map. @@ -40,20 +44,3 @@ func SimpleHashFromMap(m map[string][]byte) []byte { } return sm.Hash() } - -//---------------------------------------------------------------- - -// Expects hashes! -func simpleHashFromHashes(hashes [][]byte) []byte { - // Recursive impl. - switch len(hashes) { - case 0: - return nil - case 1: - return hashes[0] - default: - left := simpleHashFromHashes(hashes[:(len(hashes)+1)/2]) - right := simpleHashFromHashes(hashes[(len(hashes)+1)/2:]) - return SimpleHashFromTwoHashes(left, right) - } -} From d394042f65bae700693329f9af88ddc29dec81b3 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Mon, 15 Oct 2018 15:42:47 -0500 Subject: [PATCH 12/86] Make txs and evidencelist use merkle.SimpleHashFromBytes to create hash (#2635) This is a performance regression, but will also spare the types directory from knowing about RFC 6962, which is a more correct abstraction. For txs this performance hit will be fixed soon with #2603. For evidence, the performance impact is negligible due to it being capped at a small number. --- crypto/merkle/simple_proof.go | 6 +++--- crypto/merkle/simple_tree.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index d2cbb12..fd6d07b 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -134,13 +134,13 @@ func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][ if leftHash == nil { return nil } - return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + return simpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) } rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) if rightHash == nil { return nil } - return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) + return simpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) } } @@ -187,7 +187,7 @@ func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *Simp default: lefts, leftRoot := trailsFromByteSlices(items[:(len(items)+1)/2]) rights, rightRoot := trailsFromByteSlices(items[(len(items)+1)/2:]) - rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) + rootHash := simpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) root := &SimpleProofNode{rootHash, nil, nil, nil} leftRoot.Parent = root leftRoot.Right = rightRoot diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 45e0c5c..7aacb08 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -4,8 +4,8 @@ import ( "github.com/tendermint/tendermint/crypto/tmhash" ) -// SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). -func SimpleHashFromTwoHashes(left, right []byte) []byte { +// simpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). +func simpleHashFromTwoHashes(left, right []byte) []byte { var hasher = tmhash.New() err := encodeByteSlice(hasher, left) if err != nil { @@ -29,7 +29,7 @@ func SimpleHashFromByteSlices(items [][]byte) []byte { default: left := SimpleHashFromByteSlices(items[:(len(items)+1)/2]) right := SimpleHashFromByteSlices(items[(len(items)+1)/2:]) - return SimpleHashFromTwoHashes(left, right) + return simpleHashFromTwoHashes(left, right) } } From 3d8a26dd228da968f9d021703aba988236cd805d Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Thu, 25 Oct 2018 03:34:01 +0200 Subject: [PATCH 13/86] Catch up with amino 0.13.0 (#2690) * catch up with amino changes in https://github.com/tendermint/go-amino/pull/222 * WIP: update to amino v0.13.0 * update to fixed amino release --- crypto/merkle/proof_simple_value.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_simple_value.go index 5b7b523..904b6e5 100644 --- a/crypto/merkle/proof_simple_value.go +++ b/crypto/merkle/proof_simple_value.go @@ -42,7 +42,7 @@ func SimpleValueOpDecoder(pop ProofOp) (ProofOperator, error) { return nil, cmn.NewError("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpSimpleValue) } var op SimpleValueOp // a bit strange as we'll discard this, but it works. - err := cdc.UnmarshalBinary(pop.Data, &op) + err := cdc.UnmarshalBinaryLengthPrefixed(pop.Data, &op) if err != nil { return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") } @@ -50,7 +50,7 @@ func SimpleValueOpDecoder(pop ProofOp) (ProofOperator, error) { } func (op SimpleValueOp) ProofOp() ProofOp { - bz := cdc.MustMarshalBinary(op) + bz := cdc.MustMarshalBinaryLengthPrefixed(op) return ProofOp{ Type: ProofOpSimpleValue, Key: op.key, From 8261d0760afcb1ed75f8d82063db27019a30aeb0 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 31 Oct 2018 08:02:13 -0700 Subject: [PATCH 14/86] Fix general merkle keypath to start w/ last op's key (#2733) * Fix general merkle keypath to start w/ last op's key * Update CHANGELOG_PENDING.md --- crypto/merkle/proof.go | 9 +-- crypto/merkle/proof_test.go | 136 ++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 crypto/merkle/proof_test.go diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 3059ed3..5705c96 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -43,10 +43,11 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er for i, op := range poz { key := op.GetKey() if len(key) != 0 { - if !bytes.Equal(keys[0], key) { - return cmn.NewError("Key mismatch on operation #%d: expected %+v but %+v", i, []byte(keys[0]), []byte(key)) + lastKey := keys[len(keys)-1] + if !bytes.Equal(lastKey, key) { + return cmn.NewError("Key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) } - keys = keys[1:] + keys = keys[:len(keys)-1] } args, err = op.Run(args) if err != nil { @@ -54,7 +55,7 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er } } if !bytes.Equal(root, args[0]) { - return cmn.NewError("Calculated root hash is invalid: expected %+v but %+v", root, args[0]) + return cmn.NewError("Calculated root hash is invalid: expected %+v but got %+v", root, args[0]) } if len(keys) != 0 { return cmn.NewError("Keypath not consumed all") diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go new file mode 100644 index 0000000..cc208e9 --- /dev/null +++ b/crypto/merkle/proof_test.go @@ -0,0 +1,136 @@ +package merkle + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/go-amino" + cmn "github.com/tendermint/tendermint/libs/common" +) + +const ProofOpDomino = "test:domino" + +// Expects given input, produces given output. +// Like the game dominos. +type DominoOp struct { + key string // unexported, may be empty + Input string + Output string +} + +func NewDominoOp(key, input, output string) DominoOp { + return DominoOp{ + key: key, + Input: input, + Output: output, + } +} + +func DominoOpDecoder(pop ProofOp) (ProofOperator, error) { + if pop.Type != ProofOpDomino { + panic("unexpected proof op type") + } + var op DominoOp // a bit strange as we'll discard this, but it works. + err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) + if err != nil { + return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") + } + return NewDominoOp(string(pop.Key), op.Input, op.Output), nil +} + +func (dop DominoOp) ProofOp() ProofOp { + bz := amino.MustMarshalBinaryLengthPrefixed(dop) + return ProofOp{ + Type: ProofOpDomino, + Key: []byte(dop.key), + Data: bz, + } +} + +func (dop DominoOp) Run(input [][]byte) (output [][]byte, err error) { + if len(input) != 1 { + return nil, cmn.NewError("Expected input of length 1") + } + if string(input[0]) != dop.Input { + return nil, cmn.NewError("Expected input %v, got %v", + dop.Input, string(input[0])) + } + return [][]byte{[]byte(dop.Output)}, nil +} + +func (dop DominoOp) GetKey() []byte { + return []byte(dop.key) +} + +//---------------------------------------- + +func TestProofOperators(t *testing.T) { + var err error + + // ProofRuntime setup + // TODO test this somehow. + // prt := NewProofRuntime() + // prt.RegisterOpDecoder(ProofOpDomino, DominoOpDecoder) + + // ProofOperators setup + op1 := NewDominoOp("KEY1", "INPUT1", "INPUT2") + op2 := NewDominoOp("KEY2", "INPUT2", "INPUT3") + op3 := NewDominoOp("", "INPUT3", "INPUT4") + op4 := NewDominoOp("KEY4", "INPUT4", "OUTPUT4") + + // Good + popz := ProofOperators([]ProofOperator{op1, op2, op3, op4}) + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.Nil(t, err) + err = popz.VerifyValue(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", bz("INPUT1")) + assert.Nil(t, err) + + // BAD INPUT + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1_WRONG")}) + assert.NotNil(t, err) + err = popz.VerifyValue(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", bz("INPUT1_WRONG")) + assert.NotNil(t, err) + + // BAD KEY 1 + err = popz.Verify(bz("OUTPUT4"), "/KEY3/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD KEY 2 + err = popz.Verify(bz("OUTPUT4"), "KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD KEY 3 + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1/", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD KEY 4 + err = popz.Verify(bz("OUTPUT4"), "//KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD OUTPUT 1 + err = popz.Verify(bz("OUTPUT4_WRONG"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD OUTPUT 2 + err = popz.Verify(bz(""), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD POPZ 1 + popz = []ProofOperator{op1, op2, op4} + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD POPZ 2 + popz = []ProofOperator{op4, op3, op2, op1} + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD POPZ 3 + popz = []ProofOperator{} + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) +} + +func bz(s string) []byte { + return []byte(s) +} From a7f602f13214c31456983cdf992febad36fd9470 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 31 Oct 2018 12:42:05 -0400 Subject: [PATCH 15/86] TMHASH is 32 bytes. Closes #1990 (#2732) * tmhash is fully 32 bytes. closes #1990 * AddressSize * fix tests * fix max sizes --- crypto/merkle/simple_map_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go index bc095c0..7abde11 100644 --- a/crypto/merkle/simple_map_test.go +++ b/crypto/merkle/simple_map_test.go @@ -13,14 +13,14 @@ func TestSimpleMap(t *testing.T) { values []string // each string gets converted to []byte in test want string }{ - {[]string{"key1"}, []string{"value1"}, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f"}, - {[]string{"key1"}, []string{"value2"}, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8"}, + {[]string{"key1"}, []string{"value1"}, "321d150de16dceb51c72981b432b115045383259b1a550adf8dc80f927508967"}, + {[]string{"key1"}, []string{"value2"}, "2a9e4baf321eac99f6eecc3406603c14bc5e85bb7b80483cbfc75b3382d24a2f"}, // swap order with 2 keys - {[]string{"key1", "key2"}, []string{"value1", "value2"}, "eff12d1c703a1022ab509287c0f196130123d786"}, - {[]string{"key2", "key1"}, []string{"value2", "value1"}, "eff12d1c703a1022ab509287c0f196130123d786"}, + {[]string{"key1", "key2"}, []string{"value1", "value2"}, "c4d8913ab543ba26aa970646d4c99a150fd641298e3367cf68ca45fb45a49881"}, + {[]string{"key2", "key1"}, []string{"value2", "value1"}, "c4d8913ab543ba26aa970646d4c99a150fd641298e3367cf68ca45fb45a49881"}, // swap order with 3 keys - {[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"}, - {[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"}, + {[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "b23cef00eda5af4548a213a43793f2752d8d9013b3f2b64bc0523a4791196268"}, + {[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "b23cef00eda5af4548a213a43793f2752d8d9013b3f2b64bc0523a4791196268"}, } for i, tc := range tests { db := newSimpleMap() From 80dd63434ca748580bd79e30a0791d11b3a30d0d Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 5 Nov 2018 22:53:44 -0800 Subject: [PATCH 16/86] =?UTF-8?q?Fix=20crypto/merkle=20ProofOperators.Veri?= =?UTF-8?q?fy=20to=20check=20bounds=20on=20keypath=20pa=E2=80=A6=20(#2756)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix crypto/merkle ProofOperators.Verify to check bounds on keypath parts. * Update PENDING --- crypto/merkle/proof.go | 3 +++ crypto/merkle/proof_test.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 5705c96..8f8b460 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -43,6 +43,9 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er for i, op := range poz { key := op.GetKey() if len(key) != 0 { + if len(keys) == 0 { + return cmn.NewError("Key path has insufficient # of parts: expected no more keys but got %+v", string(key)) + } lastKey := keys[len(keys)-1] if !bytes.Equal(lastKey, key) { return cmn.NewError("Key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index cc208e9..320b918 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -107,6 +107,10 @@ func TestProofOperators(t *testing.T) { err = popz.Verify(bz("OUTPUT4"), "//KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) assert.NotNil(t, err) + // BAD KEY 5 + err = popz.Verify(bz("OUTPUT4"), "/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + // BAD OUTPUT 1 err = popz.Verify(bz("OUTPUT4_WRONG"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) assert.NotNil(t, err) From 0ff494acb0a6cf02e9a534d90e55f76218a5d8d2 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Sun, 13 Jan 2019 17:02:38 -0600 Subject: [PATCH 17/86] Simple merkle rfc compatibility (#2713) * Begin simple merkle compatibility PR * Fix query_test * Use trillian test vectors * Change the split point per RFC 6962 * update spec * refactor innerhash to match spec * Update changelog * Address @liamsi's comments * Write the comment requested by @liamsi --- crypto/merkle/hash.go | 21 +++++++ crypto/merkle/proof_simple_value.go | 8 +-- crypto/merkle/rfc6962_test.go | 97 +++++++++++++++++++++++++++++ crypto/merkle/simple_map_test.go | 12 ++-- crypto/merkle/simple_proof.go | 19 +++--- crypto/merkle/simple_tree.go | 38 +++++------ crypto/merkle/simple_tree_test.go | 36 ++++++++--- 7 files changed, 186 insertions(+), 45 deletions(-) create mode 100644 crypto/merkle/hash.go create mode 100644 crypto/merkle/rfc6962_test.go diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go new file mode 100644 index 0000000..4e24046 --- /dev/null +++ b/crypto/merkle/hash.go @@ -0,0 +1,21 @@ +package merkle + +import ( + "github.com/tendermint/tendermint/crypto/tmhash" +) + +// TODO: make these have a large predefined capacity +var ( + leafPrefix = []byte{0} + innerPrefix = []byte{1} +) + +// returns tmhash(0x00 || leaf) +func leafHash(leaf []byte) []byte { + return tmhash.Sum(append(leafPrefix, leaf...)) +} + +// returns tmhash(0x01 || left || right) +func innerHash(left []byte, right []byte) []byte { + return tmhash.Sum(append(innerPrefix, append(left, right...)...)) +} diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_simple_value.go index 904b6e5..247921a 100644 --- a/crypto/merkle/proof_simple_value.go +++ b/crypto/merkle/proof_simple_value.go @@ -71,11 +71,11 @@ func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { hasher.Write(value) // does not error vhash := hasher.Sum(nil) + bz := new(bytes.Buffer) // Wrap to hash the KVPair. - hasher = tmhash.New() - encodeByteSlice(hasher, []byte(op.key)) // does not error - encodeByteSlice(hasher, []byte(vhash)) // does not error - kvhash := hasher.Sum(nil) + encodeByteSlice(bz, []byte(op.key)) // does not error + encodeByteSlice(bz, []byte(vhash)) // does not error + kvhash := leafHash(bz.Bytes()) if !bytes.Equal(kvhash, op.Proof.LeafHash) { return nil, cmn.NewError("leaf hash mismatch: want %X got %X", op.Proof.LeafHash, kvhash) diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go new file mode 100644 index 0000000..b6413b4 --- /dev/null +++ b/crypto/merkle/rfc6962_test.go @@ -0,0 +1,97 @@ +package merkle + +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// These tests were taken from https://github.com/google/trillian/blob/master/merkle/rfc6962/rfc6962_test.go, +// and consequently fall under the above license. +import ( + "bytes" + "encoding/hex" + "testing" + + "github.com/tendermint/tendermint/crypto/tmhash" +) + +func TestRFC6962Hasher(t *testing.T) { + _, leafHashTrail := trailsFromByteSlices([][]byte{[]byte("L123456")}) + leafHash := leafHashTrail.Hash + _, leafHashTrail = trailsFromByteSlices([][]byte{[]byte{}}) + emptyLeafHash := leafHashTrail.Hash + for _, tc := range []struct { + desc string + got []byte + want string + }{ + // Since creating a merkle tree of no leaves is unsupported here, we skip + // the corresponding trillian test vector. + + // Check that the empty hash is not the same as the hash of an empty leaf. + // echo -n 00 | xxd -r -p | sha256sum + { + desc: "RFC6962 Empty Leaf", + want: "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d"[:tmhash.Size*2], + got: emptyLeafHash, + }, + // echo -n 004C313233343536 | xxd -r -p | sha256sum + { + desc: "RFC6962 Leaf", + want: "395aa064aa4c29f7010acfe3f25db9485bbd4b91897b6ad7ad547639252b4d56"[:tmhash.Size*2], + got: leafHash, + }, + // echo -n 014E3132334E343536 | xxd -r -p | sha256sum + { + desc: "RFC6962 Node", + want: "aa217fe888e47007fa15edab33c2b492a722cb106c64667fc2b044444de66bbb"[:tmhash.Size*2], + got: innerHash([]byte("N123"), []byte("N456")), + }, + } { + t.Run(tc.desc, func(t *testing.T) { + wantBytes, err := hex.DecodeString(tc.want) + if err != nil { + t.Fatalf("hex.DecodeString(%x): %v", tc.want, err) + } + if got, want := tc.got, wantBytes; !bytes.Equal(got, want) { + t.Errorf("got %x, want %x", got, want) + } + }) + } +} + +func TestRFC6962HasherCollisions(t *testing.T) { + // Check that different leaves have different hashes. + leaf1, leaf2 := []byte("Hello"), []byte("World") + _, leafHashTrail := trailsFromByteSlices([][]byte{leaf1}) + hash1 := leafHashTrail.Hash + _, leafHashTrail = trailsFromByteSlices([][]byte{leaf2}) + hash2 := leafHashTrail.Hash + if bytes.Equal(hash1, hash2) { + t.Errorf("Leaf hashes should differ, but both are %x", hash1) + } + // Compute an intermediate subtree hash. + _, subHash1Trail := trailsFromByteSlices([][]byte{hash1, hash2}) + subHash1 := subHash1Trail.Hash + // Check that this is not the same as a leaf hash of their concatenation. + preimage := append(hash1, hash2...) + _, forgedHashTrail := trailsFromByteSlices([][]byte{preimage}) + forgedHash := forgedHashTrail.Hash + if bytes.Equal(subHash1, forgedHash) { + t.Errorf("Hasher is not second-preimage resistant") + } + // Swap the order of nodes and check that the hash is different. + _, subHash2Trail := trailsFromByteSlices([][]byte{hash2, hash1}) + subHash2 := subHash2Trail.Hash + if bytes.Equal(subHash1, subHash2) { + t.Errorf("Subtree hash does not depend on the order of leaves") + } +} diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go index 7abde11..366d9f3 100644 --- a/crypto/merkle/simple_map_test.go +++ b/crypto/merkle/simple_map_test.go @@ -13,14 +13,14 @@ func TestSimpleMap(t *testing.T) { values []string // each string gets converted to []byte in test want string }{ - {[]string{"key1"}, []string{"value1"}, "321d150de16dceb51c72981b432b115045383259b1a550adf8dc80f927508967"}, - {[]string{"key1"}, []string{"value2"}, "2a9e4baf321eac99f6eecc3406603c14bc5e85bb7b80483cbfc75b3382d24a2f"}, + {[]string{"key1"}, []string{"value1"}, "a44d3cc7daba1a4600b00a2434b30f8b970652169810d6dfa9fb1793a2189324"}, + {[]string{"key1"}, []string{"value2"}, "0638e99b3445caec9d95c05e1a3fc1487b4ddec6a952ff337080360b0dcc078c"}, // swap order with 2 keys - {[]string{"key1", "key2"}, []string{"value1", "value2"}, "c4d8913ab543ba26aa970646d4c99a150fd641298e3367cf68ca45fb45a49881"}, - {[]string{"key2", "key1"}, []string{"value2", "value1"}, "c4d8913ab543ba26aa970646d4c99a150fd641298e3367cf68ca45fb45a49881"}, + {[]string{"key1", "key2"}, []string{"value1", "value2"}, "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3"}, + {[]string{"key2", "key1"}, []string{"value2", "value1"}, "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3"}, // swap order with 3 keys - {[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "b23cef00eda5af4548a213a43793f2752d8d9013b3f2b64bc0523a4791196268"}, - {[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "b23cef00eda5af4548a213a43793f2752d8d9013b3f2b64bc0523a4791196268"}, + {[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc"}, + {[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc"}, } for i, tc := range tests { db := newSimpleMap() diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index fd6d07b..f01dcdc 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -67,7 +66,8 @@ func SimpleProofsFromMap(m map[string][]byte) (rootHash []byte, proofs map[strin // Verify that the SimpleProof proves the root hash. // Check sp.Index/sp.Total manually if needed -func (sp *SimpleProof) Verify(rootHash []byte, leafHash []byte) error { +func (sp *SimpleProof) Verify(rootHash []byte, leaf []byte) error { + leafHash := leafHash(leaf) if sp.Total < 0 { return errors.New("Proof total must be positive") } @@ -128,19 +128,19 @@ func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][ if len(innerHashes) == 0 { return nil } - numLeft := (total + 1) / 2 + numLeft := getSplitPoint(total) if index < numLeft { leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) if leftHash == nil { return nil } - return simpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + return innerHash(leftHash, innerHashes[len(innerHashes)-1]) } rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) if rightHash == nil { return nil } - return simpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) + return innerHash(innerHashes[len(innerHashes)-1], rightHash) } } @@ -182,12 +182,13 @@ func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *Simp case 0: return nil, nil case 1: - trail := &SimpleProofNode{tmhash.Sum(items[0]), nil, nil, nil} + trail := &SimpleProofNode{leafHash(items[0]), nil, nil, nil} return []*SimpleProofNode{trail}, trail default: - lefts, leftRoot := trailsFromByteSlices(items[:(len(items)+1)/2]) - rights, rightRoot := trailsFromByteSlices(items[(len(items)+1)/2:]) - rootHash := simpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) + k := getSplitPoint(len(items)) + lefts, leftRoot := trailsFromByteSlices(items[:k]) + rights, rightRoot := trailsFromByteSlices(items[k:]) + rootHash := innerHash(leftRoot.Hash, rightRoot.Hash) root := &SimpleProofNode{rootHash, nil, nil, nil} leftRoot.Parent = root leftRoot.Right = rightRoot diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 7aacb08..e150c0d 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -1,23 +1,9 @@ package merkle import ( - "github.com/tendermint/tendermint/crypto/tmhash" + "math/bits" ) -// simpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). -func simpleHashFromTwoHashes(left, right []byte) []byte { - var hasher = tmhash.New() - err := encodeByteSlice(hasher, left) - if err != nil { - panic(err) - } - err = encodeByteSlice(hasher, right) - if err != nil { - panic(err) - } - return hasher.Sum(nil) -} - // SimpleHashFromByteSlices computes a Merkle tree where the leaves are the byte slice, // in the provided order. func SimpleHashFromByteSlices(items [][]byte) []byte { @@ -25,11 +11,12 @@ func SimpleHashFromByteSlices(items [][]byte) []byte { case 0: return nil case 1: - return tmhash.Sum(items[0]) + return leafHash(items[0]) default: - left := SimpleHashFromByteSlices(items[:(len(items)+1)/2]) - right := SimpleHashFromByteSlices(items[(len(items)+1)/2:]) - return simpleHashFromTwoHashes(left, right) + k := getSplitPoint(len(items)) + left := SimpleHashFromByteSlices(items[:k]) + right := SimpleHashFromByteSlices(items[k:]) + return innerHash(left, right) } } @@ -44,3 +31,16 @@ func SimpleHashFromMap(m map[string][]byte) []byte { } return sm.Hash() } + +func getSplitPoint(length int) int { + if length < 1 { + panic("Trying to split a tree with size < 1") + } + uLength := uint(length) + bitlen := bits.Len(uLength) + k := 1 << uint(bitlen-1) + if k == length { + k >>= 1 + } + return k +} diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index 32edc65..9abe321 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -34,7 +34,6 @@ func TestSimpleProof(t *testing.T) { // For each item, check the trail. for i, item := range items { - itemHash := tmhash.Sum(item) proof := proofs[i] // Check total/index @@ -43,30 +42,53 @@ func TestSimpleProof(t *testing.T) { require.Equal(t, proof.Total, total, "Unmatched totals: %d vs %d", proof.Total, total) // Verify success - err := proof.Verify(rootHash, itemHash) - require.NoError(t, err, "Verificatior failed: %v.", err) + err := proof.Verify(rootHash, item) + require.NoError(t, err, "Verification failed: %v.", err) // Trail too long should make it fail origAunts := proof.Aunts proof.Aunts = append(proof.Aunts, cmn.RandBytes(32)) - err = proof.Verify(rootHash, itemHash) + err = proof.Verify(rootHash, item) require.Error(t, err, "Expected verification to fail for wrong trail length") proof.Aunts = origAunts // Trail too short should make it fail proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1] - err = proof.Verify(rootHash, itemHash) + err = proof.Verify(rootHash, item) require.Error(t, err, "Expected verification to fail for wrong trail length") proof.Aunts = origAunts // Mutating the itemHash should make it fail. - err = proof.Verify(rootHash, MutateByteSlice(itemHash)) + err = proof.Verify(rootHash, MutateByteSlice(item)) require.Error(t, err, "Expected verification to fail for mutated leaf hash") // Mutating the rootHash should make it fail. - err = proof.Verify(MutateByteSlice(rootHash), itemHash) + err = proof.Verify(MutateByteSlice(rootHash), item) require.Error(t, err, "Expected verification to fail for mutated root hash") } } + +func Test_getSplitPoint(t *testing.T) { + tests := []struct { + length int + want int + }{ + {1, 0}, + {2, 1}, + {3, 2}, + {4, 2}, + {5, 4}, + {10, 8}, + {20, 16}, + {100, 64}, + {255, 128}, + {256, 128}, + {257, 256}, + } + for _, tt := range tests { + got := getSplitPoint(tt.length) + require.Equal(t, tt.want, got, "getSplitPoint(%d) = %v, want %v", tt.length, got, tt.want) + } +} From e65a788e7853356772f1b559a100ef052e2c5388 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Wed, 16 Jan 2019 15:03:19 -0600 Subject: [PATCH 18/86] Add comment to simple_merkle get_split_point (#3136) * Add comment to simple_merkle get_split_point * fix grammar error --- crypto/merkle/simple_tree.go | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index e150c0d..5de514b 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -32,6 +32,7 @@ func SimpleHashFromMap(m map[string][]byte) []byte { return sm.Hash() } +// getSplitPoint returns the largest power of 2 less than length func getSplitPoint(length int) int { if length < 1 { panic("Trying to split a tree with size < 1") From 522981e2563d3950b532c2615d659128bf2ddd39 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Mon, 28 Jan 2019 14:13:17 +0200 Subject: [PATCH 19/86] alias amino imports (#3219) As per conversation here: https://github.com/tendermint/tendermint/pull/3218#discussion_r251364041 This is the result of running the following code on the repo: ```bash find . -name '*.go' | grep -v 'vendor/' | xargs -n 1 goimports -w ``` --- crypto/merkle/proof_test.go | 2 +- crypto/merkle/wire.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 320b918..2a0bdcc 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cmn "github.com/tendermint/tendermint/libs/common" ) diff --git a/crypto/merkle/wire.go b/crypto/merkle/wire.go index c20ec9a..2b6ee35 100644 --- a/crypto/merkle/wire.go +++ b/crypto/merkle/wire.go @@ -1,7 +1,7 @@ package merkle import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" ) var cdc *amino.Codec From be50841812d8dcfceb52ee2b1d9b3a447c1444ab Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 30 Jan 2019 12:24:26 +0400 Subject: [PATCH 20/86] update gometalinter to 3.0.0 (#3233) in the attempt to fix https://circleci.com/gh/tendermint/tendermint/43165 also code is simplified by running gofmt -s . remove unused vars enable linters we're currently passing remove deprecated linters --- crypto/merkle/rfc6962_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go index b6413b4..52eab42 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/crypto/merkle/rfc6962_test.go @@ -26,7 +26,7 @@ import ( func TestRFC6962Hasher(t *testing.T) { _, leafHashTrail := trailsFromByteSlices([][]byte{[]byte("L123456")}) leafHash := leafHashTrail.Hash - _, leafHashTrail = trailsFromByteSlices([][]byte{[]byte{}}) + _, leafHashTrail = trailsFromByteSlices([][]byte{{}}) emptyLeafHash := leafHashTrail.Hash for _, tc := range []struct { desc string From 97aaab9be966b4b7bf80405ce847c5e918c2742b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 6 Feb 2019 15:16:38 +0400 Subject: [PATCH 21/86] remove or comment out unused code --- crypto/merkle/proof_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 2a0bdcc..5041562 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -26,17 +26,17 @@ func NewDominoOp(key, input, output string) DominoOp { } } -func DominoOpDecoder(pop ProofOp) (ProofOperator, error) { - if pop.Type != ProofOpDomino { - panic("unexpected proof op type") - } - var op DominoOp // a bit strange as we'll discard this, but it works. - err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) - if err != nil { - return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") - } - return NewDominoOp(string(pop.Key), op.Input, op.Output), nil -} +// func DominoOpDecoder(pop ProofOp) (ProofOperator, error) { +// if pop.Type != ProofOpDomino { +// panic("unexpected proof op type") +// } +// var op DominoOp // a bit strange as we'll discard this, but it works. +// err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) +// if err != nil { +// return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") +// } +// return NewDominoOp(string(pop.Key), op.Input, op.Output), nil +// } func (dop DominoOp) ProofOp() ProofOp { bz := amino.MustMarshalBinaryLengthPrefixed(dop) From a0e986895f05157fc7d82b9e872f005ff69fb168 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 6 Feb 2019 15:24:54 +0400 Subject: [PATCH 22/86] preallocating memory when we can --- crypto/merkle/proof.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 8f8b460..5e2a3ab 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -98,7 +98,7 @@ func (prt *ProofRuntime) Decode(pop ProofOp) (ProofOperator, error) { } func (prt *ProofRuntime) DecodeProof(proof *Proof) (ProofOperators, error) { - var poz ProofOperators + poz := make(ProofOperators, 0, len(proof.Ops)) for _, pop := range proof.Ops { operator, err := prt.Decode(pop) if err != nil { From 6268084f80af45a48c9862f577fb6836a45554f9 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 6 Feb 2019 18:20:10 +0400 Subject: [PATCH 23/86] use nolint label instead of commenting --- crypto/merkle/proof_test.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 5041562..4de3246 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -26,17 +26,18 @@ func NewDominoOp(key, input, output string) DominoOp { } } -// func DominoOpDecoder(pop ProofOp) (ProofOperator, error) { -// if pop.Type != ProofOpDomino { -// panic("unexpected proof op type") -// } -// var op DominoOp // a bit strange as we'll discard this, but it works. -// err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) -// if err != nil { -// return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") -// } -// return NewDominoOp(string(pop.Key), op.Input, op.Output), nil -// } +//nolint:unused +func DominoOpDecoder(pop ProofOp) (ProofOperator, error) { + if pop.Type != ProofOpDomino { + panic("unexpected proof op type") + } + var op DominoOp // a bit strange as we'll discard this, but it works. + err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) + if err != nil { + return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") + } + return NewDominoOp(string(pop.Key), op.Input, op.Output), nil +} func (dop DominoOp) ProofOp() ProofOp { bz := amino.MustMarshalBinaryLengthPrefixed(dop) From ba635fb7459da680cc6394ce444efa845b28829f Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Tue, 12 Feb 2019 05:54:12 +0100 Subject: [PATCH 24/86] make gosec linter pass (#3294) * not related to linter: remove obsolete constants: - `Insecure` and `Secure` and type `Security` are not used anywhere * not related to linter: update example - NewInsecure was deleted; change example to NewRemoteDB * address: Binds to all network interfaces (gosec): - bind to localhost instead of 0.0.0.0 - regenerate test key and cert for this purpose (was valid for ::) and otherwise we would see: transport: authentication handshake failed: x509: certificate is valid for ::, not 127.0.0.1\" (used https://github.com/google/keytransparency/blob/master/scripts/gen_server_keys.sh to regenerate certs) * use sha256 in tests instead of md5; time difference is negligible * nolint usage of math/rand in test and add comment on its import - crypto/rand is slower and we do not need sth more secure in tests * enable linter in circle-ci * another nolint math/rand in test * replace another occurrence of md5 * consistent comment about importing math/rand --- crypto/merkle/proof_key_path_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crypto/merkle/proof_key_path_test.go b/crypto/merkle/proof_key_path_test.go index 48fda30..34c918f 100644 --- a/crypto/merkle/proof_key_path_test.go +++ b/crypto/merkle/proof_key_path_test.go @@ -1,6 +1,8 @@ package merkle import ( + // it is ok to use math/rand here: we do not need a cryptographically secure random + // number generator here and we can run the tests a bit faster "math/rand" "testing" @@ -24,7 +26,7 @@ func TestKeyPath(t *testing.T) { keys[i][j] = alphanum[rand.Intn(len(alphanum))] } case KeyEncodingHex: - rand.Read(keys[i]) + rand.Read(keys[i]) //nolint: gosec default: panic("Unexpected encoding") } From 344f0892a58e3ff547aa702f26deebe5ae6e9991 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 4 Mar 2019 12:17:38 +0400 Subject: [PATCH 25/86] deps: update gogo/protobuf from 1.1.1 to 1.2.1 and golang/protobuf from 1.1.0 to 1.3.0 (#3357) * deps: update gogo/proto from 1.1.1 to 1.2.1 - verified changes manually git diff 636bf030~ ba06b47c --stat -- ':!*.pb.go' ':!test' * deps: update golang/protobuf from 1.1.0 to 1.3.0 - verified changes manually git diff b4deda0~ c823c79 -- ':!*.pb.go' ':!test' --- crypto/merkle/merkle.pb.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crypto/merkle/merkle.pb.go b/crypto/merkle/merkle.pb.go index 75e1b08..5b7e15c 100644 --- a/crypto/merkle/merkle.pb.go +++ b/crypto/merkle/merkle.pb.go @@ -39,7 +39,7 @@ func (m *ProofOp) Reset() { *m = ProofOp{} } func (m *ProofOp) String() string { return proto.CompactTextString(m) } func (*ProofOp) ProtoMessage() {} func (*ProofOp) Descriptor() ([]byte, []int) { - return fileDescriptor_merkle_5d3f6051907285da, []int{0} + return fileDescriptor_merkle_24be8bc4e405ac66, []int{0} } func (m *ProofOp) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -101,7 +101,7 @@ func (m *Proof) Reset() { *m = Proof{} } func (m *Proof) String() string { return proto.CompactTextString(m) } func (*Proof) ProtoMessage() {} func (*Proof) Descriptor() ([]byte, []int) { - return fileDescriptor_merkle_5d3f6051907285da, []int{1} + return fileDescriptor_merkle_24be8bc4e405ac66, []int{1} } func (m *Proof) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -395,6 +395,9 @@ func encodeVarintPopulateMerkle(dAtA []byte, v uint64) []byte { return dAtA } func (m *ProofOp) Size() (n int) { + if m == nil { + return 0 + } var l int _ = l l = len(m.Type) @@ -416,6 +419,9 @@ func (m *ProofOp) Size() (n int) { } func (m *Proof) Size() (n int) { + if m == nil { + return 0 + } var l int _ = l if len(m.Ops) > 0 { @@ -772,9 +778,9 @@ var ( ErrIntOverflowMerkle = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_merkle_5d3f6051907285da) } +func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_merkle_24be8bc4e405ac66) } -var fileDescriptor_merkle_5d3f6051907285da = []byte{ +var fileDescriptor_merkle_24be8bc4e405ac66 = []byte{ // 200 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c, 0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25, From e7de59ec50f53af2e6ebbd3f3647cf9503ff06df Mon Sep 17 00:00:00 2001 From: Sean Braithwaite Date: Thu, 18 Apr 2019 17:31:36 +0200 Subject: [PATCH 26/86] crypto: Proof of Concept for iterative version of SimpleHashFromByteSlices (#2611) (#3530) (#2611) had suggested that an iterative version of SimpleHashFromByteSlice would be faster, presumably because we can envision some overhead accumulating from stack frames and function calls. Additionally, a recursive algorithm risks hitting the stack limit and causing a stack overflow should the tree be too large. Provided here is an iterative alternative, a simple test to assert correctness and a benchmark. On the performance side, there appears to be no overall difference: ``` BenchmarkSimpleHashAlternatives/recursive-4 20000 77677 ns/op BenchmarkSimpleHashAlternatives/iterative-4 20000 76802 ns/op ``` On the surface it might seem that the additional overhead is due to the different allocation patterns of the implementations. The recursive version uses a single `[][]byte` slices which it then re-slices at each level of the tree. The iterative version reproduces `[][]byte` once within the function and then rewrites sub-slices of that array at each level of the tree. Eexperimenting by modifying the code to simply calculate the hash and not store the result show little to no difference in performance. These preliminary results suggest: 1. The performance of the current implementation is pretty good 2. Go has low overhead for recursive functions 3. The performance of the SimpleHashFromByteSlice routine is dominated by the actual hashing of data Although this work is in no way exhaustive, point #3 suggests that optimizations of this routine would need to take an alternative approach to make significant improvements on the current performance. Finally, considering that the recursive implementation is easier to read, it might not be worthwhile to switch to a less intuitive implementation for so little benefit. * re-add slice re-writing * [crypto] Document SimpleHashFromByteSlicesIterative --- crypto/merkle/simple_tree.go | 71 +++++++++++++++++++++++++++++++ crypto/merkle/simple_tree_test.go | 36 ++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 5de514b..03dc9d9 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -20,6 +20,77 @@ func SimpleHashFromByteSlices(items [][]byte) []byte { } } +// SimpleHashFromByteSliceIterative is an iterative alternative to +// SimpleHashFromByteSlice motivated by potential performance improvements. +// (#2611) had suggested that an iterative version of +// SimpleHashFromByteSlice would be faster, presumably because +// we can envision some overhead accumulating from stack +// frames and function calls. Additionally, a recursive algorithm risks +// hitting the stack limit and causing a stack overflow should the tree +// be too large. +// +// Provided here is an iterative alternative, a simple test to assert +// correctness and a benchmark. On the performance side, there appears to +// be no overall difference: +// +// BenchmarkSimpleHashAlternatives/recursive-4 20000 77677 ns/op +// BenchmarkSimpleHashAlternatives/iterative-4 20000 76802 ns/op +// +// On the surface it might seem that the additional overhead is due to +// the different allocation patterns of the implementations. The recursive +// version uses a single [][]byte slices which it then re-slices at each level of the tree. +// The iterative version reproduces [][]byte once within the function and +// then rewrites sub-slices of that array at each level of the tree. +// +// Experimenting by modifying the code to simply calculate the +// hash and not store the result show little to no difference in performance. +// +// These preliminary results suggest: +// +// 1. The performance of the SimpleHashFromByteSlice is pretty good +// 2. Go has low overhead for recursive functions +// 3. The performance of the SimpleHashFromByteSlice routine is dominated +// by the actual hashing of data +// +// Although this work is in no way exhaustive, point #3 suggests that +// optimization of this routine would need to take an alternative +// approach to make significant improvements on the current performance. +// +// Finally, considering that the recursive implementation is easier to +// read, it might not be worthwhile to switch to a less intuitive +// implementation for so little benefit. +func SimpleHashFromByteSlicesIterative(input [][]byte) []byte { + items := make([][]byte, len(input)) + + for i, leaf := range input { + items[i] = leafHash(leaf) + } + + size := len(items) + for { + switch size { + case 0: + return nil + case 1: + return items[0] + default: + rp := 0 // read position + wp := 0 // write position + for rp < size { + if rp+1 < size { + items[wp] = innerHash(items[rp], items[rp+1]) + rp += 2 + } else { + items[wp] = items[rp] + rp += 1 + } + wp += 1 + } + size = wp + } + } +} + // SimpleHashFromMap computes a Merkle tree from sorted map. // Like calling SimpleHashFromHashers with // `item = []byte(Hash(key) | Hash(value))`, diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index 9abe321..5bbe294 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -70,6 +70,42 @@ func TestSimpleProof(t *testing.T) { } } +func TestSimpleHashAlternatives(t *testing.T) { + + total := 100 + + items := make([][]byte, total) + for i := 0; i < total; i++ { + items[i] = testItem(cmn.RandBytes(tmhash.Size)) + } + + rootHash1 := SimpleHashFromByteSlicesIterative(items) + rootHash2 := SimpleHashFromByteSlices(items) + require.Equal(t, rootHash1, rootHash2, "Unmatched root hashes: %X vs %X", rootHash1, rootHash2) +} + +func BenchmarkSimpleHashAlternatives(b *testing.B) { + total := 100 + + items := make([][]byte, total) + for i := 0; i < total; i++ { + items[i] = testItem(cmn.RandBytes(tmhash.Size)) + } + + b.ResetTimer() + b.Run("recursive", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = SimpleHashFromByteSlices(items) + } + }) + + b.Run("iterative", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = SimpleHashFromByteSlicesIterative(items) + } + }) +} + func Test_getSplitPoint(t *testing.T) { tests := []struct { length int From 3741e5d10457df127f1e859e9b0e5c786393f8e2 Mon Sep 17 00:00:00 2001 From: Marko Date: Tue, 23 Jul 2019 15:35:36 +0200 Subject: [PATCH 27/86] Renamed wire.go to codec.go (#3827) * Renamed wire.go to codec.go - Wire was the previous name of amino - Codec describes the file better than `wire` & `amino` Signed-off-by: Marko Baricevic * ide error * rename amino.go to codec.go --- crypto/merkle/{wire.go => codec.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crypto/merkle/{wire.go => codec.go} (100%) diff --git a/crypto/merkle/wire.go b/crypto/merkle/codec.go similarity index 100% rename from crypto/merkle/wire.go rename to crypto/merkle/codec.go From 12206cafb3218b740d98d7b6e74499247d496ce4 Mon Sep 17 00:00:00 2001 From: Marko Date: Fri, 2 Aug 2019 08:53:52 +0200 Subject: [PATCH 28/86] gocritic (2/2) (#3864) Refs #3262 --- crypto/merkle/simple_proof.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index f01dcdc..d3be5d7 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -162,11 +162,12 @@ func (spn *SimpleProofNode) FlattenAunts() [][]byte { // Nonrecursive impl. innerHashes := [][]byte{} for spn != nil { - if spn.Left != nil { + switch { + case spn.Left != nil: innerHashes = append(innerHashes, spn.Left.Hash) - } else if spn.Right != nil { + case spn.Right != nil: innerHashes = append(innerHashes, spn.Right.Hash) - } else { + default: break } spn = spn.Parent From 6a21f3d8f1afb18b7920488b8282a4348374e2eb Mon Sep 17 00:00:00 2001 From: Marko Date: Thu, 8 Aug 2019 12:31:13 +0200 Subject: [PATCH 29/86] replace errors.go with github.com/pkg/errors (1/2) (#3888) * (1/2) of replace errors.go with github.com/pkg/errors ref #3862 - step one in removing instances of errors.go in favor of github.com/pkg/errors Signed-off-by: Marko Baricevic * gofmt * add in /store --- crypto/merkle/proof.go | 16 ++++++++-------- crypto/merkle/proof_key_path.go | 8 ++++---- crypto/merkle/proof_simple_value.go | 11 ++++++----- crypto/merkle/proof_test.go | 8 ++++---- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 5e2a3ab..ad101d9 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -3,7 +3,7 @@ package merkle import ( "bytes" - cmn "github.com/tendermint/tendermint/libs/common" + "github.com/pkg/errors" ) //---------------------------------------- @@ -44,11 +44,11 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er key := op.GetKey() if len(key) != 0 { if len(keys) == 0 { - return cmn.NewError("Key path has insufficient # of parts: expected no more keys but got %+v", string(key)) + return errors.Errorf("Key path has insufficient # of parts: expected no more keys but got %+v", string(key)) } lastKey := keys[len(keys)-1] if !bytes.Equal(lastKey, key) { - return cmn.NewError("Key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) + return errors.Errorf("Key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) } keys = keys[:len(keys)-1] } @@ -58,10 +58,10 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er } } if !bytes.Equal(root, args[0]) { - return cmn.NewError("Calculated root hash is invalid: expected %+v but got %+v", root, args[0]) + return errors.Errorf("Calculated root hash is invalid: expected %+v but got %+v", root, args[0]) } if len(keys) != 0 { - return cmn.NewError("Keypath not consumed all") + return errors.New("Keypath not consumed all") } return nil } @@ -92,7 +92,7 @@ func (prt *ProofRuntime) RegisterOpDecoder(typ string, dec OpDecoder) { func (prt *ProofRuntime) Decode(pop ProofOp) (ProofOperator, error) { decoder := prt.decoders[pop.Type] if decoder == nil { - return nil, cmn.NewError("unrecognized proof type %v", pop.Type) + return nil, errors.Errorf("unrecognized proof type %v", pop.Type) } return decoder(pop) } @@ -102,7 +102,7 @@ func (prt *ProofRuntime) DecodeProof(proof *Proof) (ProofOperators, error) { for _, pop := range proof.Ops { operator, err := prt.Decode(pop) if err != nil { - return nil, cmn.ErrorWrap(err, "decoding a proof operator") + return nil, errors.Wrap(err, "decoding a proof operator") } poz = append(poz, operator) } @@ -122,7 +122,7 @@ func (prt *ProofRuntime) VerifyAbsence(proof *Proof, root []byte, keypath string func (prt *ProofRuntime) Verify(proof *Proof, root []byte, keypath string, args [][]byte) (err error) { poz, err := prt.DecodeProof(proof) if err != nil { - return cmn.ErrorWrap(err, "decoding proof") + return errors.Wrap(err, "decoding proof") } return poz.Verify(root, keypath, args) } diff --git a/crypto/merkle/proof_key_path.go b/crypto/merkle/proof_key_path.go index aec93e8..7ea6785 100644 --- a/crypto/merkle/proof_key_path.go +++ b/crypto/merkle/proof_key_path.go @@ -6,7 +6,7 @@ import ( "net/url" "strings" - cmn "github.com/tendermint/tendermint/libs/common" + "github.com/pkg/errors" ) /* @@ -87,7 +87,7 @@ func (pth KeyPath) String() string { // Each key must use a known encoding. func KeyPathToKeys(path string) (keys [][]byte, err error) { if path == "" || path[0] != '/' { - return nil, cmn.NewError("key path string must start with a forward slash '/'") + return nil, errors.New("key path string must start with a forward slash '/'") } parts := strings.Split(path[1:], "/") keys = make([][]byte, len(parts)) @@ -96,13 +96,13 @@ func KeyPathToKeys(path string) (keys [][]byte, err error) { hexPart := part[2:] key, err := hex.DecodeString(hexPart) if err != nil { - return nil, cmn.ErrorWrap(err, "decoding hex-encoded part #%d: /%s", i, part) + return nil, errors.Wrapf(err, "decoding hex-encoded part #%d: /%s", i, part) } keys[i] = key } else { key, err := url.PathUnescape(part) if err != nil { - return nil, cmn.ErrorWrap(err, "decoding url-encoded part #%d: /%s", i, part) + return nil, errors.Wrapf(err, "decoding url-encoded part #%d: /%s", i, part) } keys[i] = []byte(key) // TODO Test this with random bytes, I'm not sure that it works for arbitrary bytes... } diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_simple_value.go index 247921a..2c89bb5 100644 --- a/crypto/merkle/proof_simple_value.go +++ b/crypto/merkle/proof_simple_value.go @@ -4,8 +4,9 @@ import ( "bytes" "fmt" + "github.com/pkg/errors" + "github.com/tendermint/tendermint/crypto/tmhash" - cmn "github.com/tendermint/tendermint/libs/common" ) const ProofOpSimpleValue = "simple:v" @@ -39,12 +40,12 @@ func NewSimpleValueOp(key []byte, proof *SimpleProof) SimpleValueOp { func SimpleValueOpDecoder(pop ProofOp) (ProofOperator, error) { if pop.Type != ProofOpSimpleValue { - return nil, cmn.NewError("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpSimpleValue) + return nil, errors.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpSimpleValue) } var op SimpleValueOp // a bit strange as we'll discard this, but it works. err := cdc.UnmarshalBinaryLengthPrefixed(pop.Data, &op) if err != nil { - return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") + return nil, errors.Wrap(err, "decoding ProofOp.Data into SimpleValueOp") } return NewSimpleValueOp(pop.Key, op.Proof), nil } @@ -64,7 +65,7 @@ func (op SimpleValueOp) String() string { func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { if len(args) != 1 { - return nil, cmn.NewError("expected 1 arg, got %v", len(args)) + return nil, errors.Errorf("expected 1 arg, got %v", len(args)) } value := args[0] hasher := tmhash.New() @@ -78,7 +79,7 @@ func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { kvhash := leafHash(bz.Bytes()) if !bytes.Equal(kvhash, op.Proof.LeafHash) { - return nil, cmn.NewError("leaf hash mismatch: want %X got %X", op.Proof.LeafHash, kvhash) + return nil, errors.Errorf("leaf hash mismatch: want %X got %X", op.Proof.LeafHash, kvhash) } return [][]byte{ diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 4de3246..4dc916a 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -3,9 +3,9 @@ package merkle import ( "testing" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" amino "github.com/tendermint/go-amino" - cmn "github.com/tendermint/tendermint/libs/common" ) const ProofOpDomino = "test:domino" @@ -34,7 +34,7 @@ func DominoOpDecoder(pop ProofOp) (ProofOperator, error) { var op DominoOp // a bit strange as we'll discard this, but it works. err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) if err != nil { - return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") + return nil, errors.Wrap(err, "decoding ProofOp.Data into SimpleValueOp") } return NewDominoOp(string(pop.Key), op.Input, op.Output), nil } @@ -50,10 +50,10 @@ func (dop DominoOp) ProofOp() ProofOp { func (dop DominoOp) Run(input [][]byte) (output [][]byte, err error) { if len(input) != 1 { - return nil, cmn.NewError("Expected input of length 1") + return nil, errors.New("Expected input of length 1") } if string(input[0]) != dop.Input { - return nil, cmn.NewError("Expected input %v, got %v", + return nil, errors.Errorf("Expected input %v, got %v", dop.Input, string(input[0])) } return [][]byte{[]byte(dop.Output)}, nil From 0f7b3ff3b26802b3c0a921b5e19019fe032a3209 Mon Sep 17 00:00:00 2001 From: Marko Date: Sun, 11 Aug 2019 19:03:40 +0200 Subject: [PATCH 30/86] replace errors.go with github.com/pkg/errors (2/2) (#3890) * init of (2/2) common errors * Remove instances of cmn.Error (2/2) - Replace usage of cmnError and errorWrap - ref #3862 Signed-off-by: Marko Baricevic * comment wording * simplify IsErrXXX functions * log panic along with stopping the MConnection --- crypto/merkle/simple_proof.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index d3be5d7..da32157 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -2,10 +2,9 @@ package merkle import ( "bytes" - "errors" "fmt" - cmn "github.com/tendermint/tendermint/libs/common" + "github.com/pkg/errors" ) // SimpleProof represents a simple Merkle proof. @@ -75,11 +74,11 @@ func (sp *SimpleProof) Verify(rootHash []byte, leaf []byte) error { return errors.New("Proof index cannot be negative") } if !bytes.Equal(sp.LeafHash, leafHash) { - return cmn.NewError("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) + return errors.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) } computedHash := sp.ComputeRootHash() if !bytes.Equal(computedHash, rootHash) { - return cmn.NewError("invalid root hash: wanted %X got %X", rootHash, computedHash) + return errors.Errorf("invalid root hash: wanted %X got %X", rootHash, computedHash) } return nil } From 8c797a07099ff9d390a52eae4625a38c8dd58d70 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Tue, 10 Sep 2019 03:31:44 -0400 Subject: [PATCH 31/86] fix linter errors thrown by `unconvert`, `goconst`, and `nakedret` (#3960) * Remove unnecessary type conversions * Consolidate repeated strings into consts * Clothe return statements * Update blockchain/v1/reactor_fsm_test.go Co-Authored-By: Anton Kaliaev This PR repairs linter errors seen when running the following commands: golangci-lint run --no-config --disable-all=true --enable=unconvert golangci-lint run --no-config --disable-all=true --enable=goconst golangci-lint run --no-config --disable-all=true --enable=nakedret Contributes to #3262 --- crypto/merkle/proof_simple_value.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_simple_value.go index 2c89bb5..55337b7 100644 --- a/crypto/merkle/proof_simple_value.go +++ b/crypto/merkle/proof_simple_value.go @@ -74,8 +74,8 @@ func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { bz := new(bytes.Buffer) // Wrap to hash the KVPair. - encodeByteSlice(bz, []byte(op.key)) // does not error - encodeByteSlice(bz, []byte(vhash)) // does not error + encodeByteSlice(bz, op.key) // does not error + encodeByteSlice(bz, vhash) // does not error kvhash := leafHash(bz.Bytes()) if !bytes.Equal(kvhash, op.Proof.LeafHash) { From 21bd52b461eb77b6647ecd2b0039439a41ac2d56 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Wed, 11 Sep 2019 01:15:18 -0400 Subject: [PATCH 32/86] linters: enable scopelint (#3963) * Pin range scope vars * Don't disable scopelint This PR repairs linter errors seen when running the following commands: golangci-lint run --no-config --disable-all=true --enable=scopelint Contributes to #3262 --- crypto/merkle/rfc6962_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go index 52eab42..e2fe7f6 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/crypto/merkle/rfc6962_test.go @@ -56,6 +56,7 @@ func TestRFC6962Hasher(t *testing.T) { got: innerHash([]byte("N123"), []byte("N456")), }, } { + tc := tc t.Run(tc.desc, func(t *testing.T) { wantBytes, err := hex.DecodeString(tc.want) if err != nil { From 48299a847d62c8a0e48b3adf89fdd2c84695ad35 Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 18 Sep 2019 11:32:50 +0200 Subject: [PATCH 33/86] custom marshallers for proto types, which EmitDefaults (#3889) * Remove omitempty from *pb.go - remove omitempty from *pb.go files - added command to makefile for everytime `make protoc_all` is run - open question: - Do we want to further remove omitempty from other places - https://github.com/tendermint/tendermint/blob/master/rpc/lib/types/types.go#L151 - and other places ref #3882 Signed-off-by: Marko Baricevic * bring back omitempty to *pb.go * Update types/tx.go * custom marshlers * undo benchmark `omitepmty` * golangci lint fix * cleanup comments * changelog_pending entry --- crypto/merkle/result.go | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 crypto/merkle/result.go diff --git a/crypto/merkle/result.go b/crypto/merkle/result.go new file mode 100644 index 0000000..c7bbb57 --- /dev/null +++ b/crypto/merkle/result.go @@ -0,0 +1,53 @@ +// nolint: dupl +package merkle + +import ( + "bytes" + "encoding/json" + + "github.com/gogo/protobuf/jsonpb" +) + +//--------------------------------------------------------------------------- +// override JSON marshalling so we emit defaults (ie. disable omitempty) + +var ( + jsonpbMarshaller = jsonpb.Marshaler{ + EnumsAsInts: true, + EmitDefaults: true, + } + jsonpbUnmarshaller = jsonpb.Unmarshaler{} +) + +func (r *ProofOp) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *ProofOp) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +func (r *Proof) MarshalJSON() ([]byte, error) { + s, err := jsonpbMarshaller.MarshalToString(r) + return []byte(s), err +} + +func (r *Proof) UnmarshalJSON(b []byte) error { + reader := bytes.NewBuffer(b) + return jsonpbUnmarshaller.Unmarshal(reader, r) +} + +// Some compile time assertions to ensure we don't +// have accidental runtime surprises later on. +// jsonEncodingRoundTripper ensures that asserted +// interfaces implement both MarshalJSON and UnmarshalJSON + +type jsonRoundTripper interface { + json.Marshaler + json.Unmarshaler +} + +var _ jsonRoundTripper = (*ProofOp)(nil) +var _ jsonRoundTripper = (*Proof)(nil) From fa8abc47189c277d812093d354b2429b6c201b7f Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 19 Sep 2019 11:41:57 +0300 Subject: [PATCH 34/86] regenerate protobuf files with newer gogo version (#3998) protoc 3.7.0 gogo v1.3.0 --- crypto/merkle/merkle.pb.go | 212 ++++++++++++++++++++++--------------- 1 file changed, 125 insertions(+), 87 deletions(-) diff --git a/crypto/merkle/merkle.pb.go b/crypto/merkle/merkle.pb.go index 5b7e15c..2ebbee1 100644 --- a/crypto/merkle/merkle.pb.go +++ b/crypto/merkle/merkle.pb.go @@ -3,14 +3,15 @@ package merkle -import proto "github.com/gogo/protobuf/proto" -import fmt "fmt" -import math "math" -import _ "github.com/gogo/protobuf/gogoproto" - -import bytes "bytes" - -import io "io" +import ( + bytes "bytes" + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -21,7 +22,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package // ProofOp defines an operation used for calculating Merkle root // The data could be arbitrary format, providing nessecary data @@ -39,7 +40,7 @@ func (m *ProofOp) Reset() { *m = ProofOp{} } func (m *ProofOp) String() string { return proto.CompactTextString(m) } func (*ProofOp) ProtoMessage() {} func (*ProofOp) Descriptor() ([]byte, []int) { - return fileDescriptor_merkle_24be8bc4e405ac66, []int{0} + return fileDescriptor_9c1c2162d560d38e, []int{0} } func (m *ProofOp) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -49,15 +50,15 @@ func (m *ProofOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ProofOp.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *ProofOp) XXX_Merge(src proto.Message) { - xxx_messageInfo_ProofOp.Merge(dst, src) +func (m *ProofOp) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProofOp.Merge(m, src) } func (m *ProofOp) XXX_Size() int { return m.Size() @@ -91,7 +92,7 @@ func (m *ProofOp) GetData() []byte { // Proof is Merkle proof defined by the list of ProofOps type Proof struct { - Ops []ProofOp `protobuf:"bytes,1,rep,name=ops" json:"ops"` + Ops []ProofOp `protobuf:"bytes,1,rep,name=ops,proto3" json:"ops"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -101,7 +102,7 @@ func (m *Proof) Reset() { *m = Proof{} } func (m *Proof) String() string { return proto.CompactTextString(m) } func (*Proof) ProtoMessage() {} func (*Proof) Descriptor() ([]byte, []int) { - return fileDescriptor_merkle_24be8bc4e405ac66, []int{1} + return fileDescriptor_9c1c2162d560d38e, []int{1} } func (m *Proof) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -111,15 +112,15 @@ func (m *Proof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Proof.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } -func (dst *Proof) XXX_Merge(src proto.Message) { - xxx_messageInfo_Proof.Merge(dst, src) +func (m *Proof) XXX_Merge(src proto.Message) { + xxx_messageInfo_Proof.Merge(m, src) } func (m *Proof) XXX_Size() int { return m.Size() @@ -141,6 +142,26 @@ func init() { proto.RegisterType((*ProofOp)(nil), "merkle.ProofOp") proto.RegisterType((*Proof)(nil), "merkle.Proof") } + +func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_9c1c2162d560d38e) } + +var fileDescriptor_9c1c2162d560d38e = []byte{ + // 200 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c, + 0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25, + 0xf9, 0x42, 0x6c, 0x10, 0x9e, 0x94, 0x6e, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, + 0xae, 0x7e, 0x7a, 0x7e, 0x7a, 0xbe, 0x3e, 0x58, 0x3a, 0xa9, 0x34, 0x0d, 0xcc, 0x03, 0x73, 0xc0, + 0x2c, 0x88, 0x36, 0x25, 0x67, 0x2e, 0xf6, 0x80, 0xa2, 0xfc, 0xfc, 0x34, 0xff, 0x02, 0x21, 0x21, + 0x2e, 0x96, 0x92, 0xca, 0x82, 0x54, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, 0x20, 0x30, 0x5b, 0x48, + 0x80, 0x8b, 0x39, 0x3b, 0xb5, 0x52, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xc4, 0x04, 0xa9, + 0x4a, 0x49, 0x2c, 0x49, 0x94, 0x60, 0x06, 0x0b, 0x81, 0xd9, 0x4a, 0x06, 0x5c, 0xac, 0x60, 0x43, + 0x84, 0xd4, 0xb9, 0x98, 0xf3, 0x0b, 0x8a, 0x25, 0x18, 0x15, 0x98, 0x35, 0xb8, 0x8d, 0xf8, 0xf5, + 0xa0, 0x0e, 0x84, 0x5a, 0xe0, 0xc4, 0x72, 0xe2, 0x9e, 0x3c, 0x43, 0x10, 0x48, 0x85, 0x93, 0xc8, + 0x8f, 0x87, 0x72, 0x8c, 0x2b, 0x1e, 0xc9, 0x31, 0x9e, 0x78, 0x24, 0xc7, 0x78, 0xe1, 0x91, 0x1c, + 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x49, 0x6c, 0x60, 0x37, 0x19, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, + 0xb9, 0x2b, 0x0f, 0xd1, 0xe8, 0x00, 0x00, 0x00, +} + func (this *ProofOp) Equal(that interface{}) bool { if that == nil { return this == nil @@ -209,7 +230,7 @@ func (this *Proof) Equal(that interface{}) bool { func (m *ProofOp) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -217,38 +238,47 @@ func (m *ProofOp) Marshal() (dAtA []byte, err error) { } func (m *ProofOp) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ProofOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Type) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintMerkle(dAtA, i, uint64(len(m.Type))) - i += copy(dAtA[i:], m.Type) - } - if len(m.Key) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintMerkle(dAtA, i, uint64(len(m.Key))) - i += copy(dAtA[i:], m.Key) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Data) > 0 { - dAtA[i] = 0x1a - i++ + i -= len(m.Data) + copy(dAtA[i:], m.Data) i = encodeVarintMerkle(dAtA, i, uint64(len(m.Data))) - i += copy(dAtA[i:], m.Data) + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Key) > 0 { + i -= len(m.Key) + copy(dAtA[i:], m.Key) + i = encodeVarintMerkle(dAtA, i, uint64(len(m.Key))) + i-- + dAtA[i] = 0x12 + } + if len(m.Type) > 0 { + i -= len(m.Type) + copy(dAtA[i:], m.Type) + i = encodeVarintMerkle(dAtA, i, uint64(len(m.Type))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *Proof) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -256,36 +286,46 @@ func (m *Proof) Marshal() (dAtA []byte, err error) { } func (m *Proof) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Proof) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } if len(m.Ops) > 0 { - for _, msg := range m.Ops { - dAtA[i] = 0xa - i++ - i = encodeVarintMerkle(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.Ops) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Ops[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMerkle(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0xa } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func encodeVarintMerkle(dAtA []byte, offset int, v uint64) int { + offset -= sovMerkle(v) + base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) - return offset + 1 + return base } func NewPopulatedProofOp(r randyMerkle, easy bool) *ProofOp { this := &ProofOp{} @@ -308,7 +348,7 @@ func NewPopulatedProofOp(r randyMerkle, easy bool) *ProofOp { func NewPopulatedProof(r randyMerkle, easy bool) *Proof { this := &Proof{} - if r.Intn(10) != 0 { + if r.Intn(5) != 0 { v3 := r.Intn(5) this.Ops = make([]ProofOp, v3) for i := 0; i < v3; i++ { @@ -437,14 +477,7 @@ func (m *Proof) Size() (n int) { } func sovMerkle(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n + return (math_bits.Len64(x|1) + 6) / 7 } func sozMerkle(x uint64) (n int) { return sovMerkle(uint64((x << 1) ^ uint64((int64(x) >> 63)))) @@ -464,7 +497,7 @@ func (m *ProofOp) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -492,7 +525,7 @@ func (m *ProofOp) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -502,6 +535,9 @@ func (m *ProofOp) Unmarshal(dAtA []byte) error { return ErrInvalidLengthMerkle } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMerkle + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -521,7 +557,7 @@ func (m *ProofOp) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= (int(b) & 0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -530,6 +566,9 @@ func (m *ProofOp) Unmarshal(dAtA []byte) error { return ErrInvalidLengthMerkle } postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMerkle + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -552,7 +591,7 @@ func (m *ProofOp) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= (int(b) & 0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -561,6 +600,9 @@ func (m *ProofOp) Unmarshal(dAtA []byte) error { return ErrInvalidLengthMerkle } postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMerkle + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -578,6 +620,9 @@ func (m *ProofOp) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthMerkle } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMerkle + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -606,7 +651,7 @@ func (m *Proof) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -634,7 +679,7 @@ func (m *Proof) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -643,6 +688,9 @@ func (m *Proof) Unmarshal(dAtA []byte) error { return ErrInvalidLengthMerkle } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMerkle + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -660,6 +708,9 @@ func (m *Proof) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthMerkle } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMerkle + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -727,10 +778,13 @@ func skipMerkle(dAtA []byte) (n int, err error) { break } } - iNdEx += length if length < 0 { return 0, ErrInvalidLengthMerkle } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthMerkle + } return iNdEx, nil case 3: for { @@ -759,6 +813,9 @@ func skipMerkle(dAtA []byte) (n int, err error) { return 0, err } iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthMerkle + } } return iNdEx, nil case 4: @@ -777,22 +834,3 @@ var ( ErrInvalidLengthMerkle = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowMerkle = fmt.Errorf("proto: integer overflow") ) - -func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_merkle_24be8bc4e405ac66) } - -var fileDescriptor_merkle_24be8bc4e405ac66 = []byte{ - // 200 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c, - 0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25, - 0xf9, 0x42, 0x6c, 0x10, 0x9e, 0x94, 0x6e, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, - 0xae, 0x7e, 0x7a, 0x7e, 0x7a, 0xbe, 0x3e, 0x58, 0x3a, 0xa9, 0x34, 0x0d, 0xcc, 0x03, 0x73, 0xc0, - 0x2c, 0x88, 0x36, 0x25, 0x67, 0x2e, 0xf6, 0x80, 0xa2, 0xfc, 0xfc, 0x34, 0xff, 0x02, 0x21, 0x21, - 0x2e, 0x96, 0x92, 0xca, 0x82, 0x54, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, 0x20, 0x30, 0x5b, 0x48, - 0x80, 0x8b, 0x39, 0x3b, 0xb5, 0x52, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xc4, 0x04, 0xa9, - 0x4a, 0x49, 0x2c, 0x49, 0x94, 0x60, 0x06, 0x0b, 0x81, 0xd9, 0x4a, 0x06, 0x5c, 0xac, 0x60, 0x43, - 0x84, 0xd4, 0xb9, 0x98, 0xf3, 0x0b, 0x8a, 0x25, 0x18, 0x15, 0x98, 0x35, 0xb8, 0x8d, 0xf8, 0xf5, - 0xa0, 0x0e, 0x84, 0x5a, 0xe0, 0xc4, 0x72, 0xe2, 0x9e, 0x3c, 0x43, 0x10, 0x48, 0x85, 0x93, 0xc8, - 0x8f, 0x87, 0x72, 0x8c, 0x2b, 0x1e, 0xc9, 0x31, 0x9e, 0x78, 0x24, 0xc7, 0x78, 0xe1, 0x91, 0x1c, - 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x49, 0x6c, 0x60, 0x37, 0x19, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, - 0xb9, 0x2b, 0x0f, 0xd1, 0xe8, 0x00, 0x00, 0x00, -} From 0a35432b3325780420c6d3a5ac837a6bd9980109 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Thu, 17 Oct 2019 04:42:28 -0400 Subject: [PATCH 35/86] Fix linter errors thrown by `lll` (#3970) * Fix long line errors in abci, crypto, and libs packages * Fix long lines in p2p and rpc packages * Fix long lines in abci, state, and tools packages * Fix long lines in behaviour and blockchain packages * Fix long lines in cmd and config packages * Begin fixing long lines in consensus package * Finish fixing long lines in consensus package * Add lll exclusion for lines containing URLs * Fix long lines in crypto package * Fix long lines in evidence package * Fix long lines in mempool and node packages * Fix long lines in libs package * Fix long lines in lite package * Fix new long line in node package * Fix long lines in p2p package * Ignore gocritic warning * Fix long lines in privval package * Fix long lines in rpc package * Fix long lines in scripts package * Fix long lines in state package * Fix long lines in tools package * Fix long lines in types package * Enable lll linter --- crypto/merkle/simple_map_test.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go index 366d9f3..20868a7 100644 --- a/crypto/merkle/simple_map_test.go +++ b/crypto/merkle/simple_map_test.go @@ -16,11 +16,27 @@ func TestSimpleMap(t *testing.T) { {[]string{"key1"}, []string{"value1"}, "a44d3cc7daba1a4600b00a2434b30f8b970652169810d6dfa9fb1793a2189324"}, {[]string{"key1"}, []string{"value2"}, "0638e99b3445caec9d95c05e1a3fc1487b4ddec6a952ff337080360b0dcc078c"}, // swap order with 2 keys - {[]string{"key1", "key2"}, []string{"value1", "value2"}, "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3"}, - {[]string{"key2", "key1"}, []string{"value2", "value1"}, "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3"}, + { + []string{"key1", "key2"}, + []string{"value1", "value2"}, + "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3", + }, + { + []string{"key2", "key1"}, + []string{"value2", "value1"}, + "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3", + }, // swap order with 3 keys - {[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc"}, - {[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc"}, + { + []string{"key1", "key2", "key3"}, + []string{"value1", "value2", "value3"}, + "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc", + }, + { + []string{"key1", "key3", "key2"}, + []string{"value1", "value3", "value2"}, + "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc", + }, } for i, tc := range tests { db := newSimpleMap() From ee91397eb2aa14f8c3a4d4a0b96ab4e13d277d97 Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 30 Oct 2019 12:25:58 -0700 Subject: [PATCH 36/86] Update master (#4087) * cs: panic only when WAL#WriteSync fails - modify WAL#Write and WAL#WriteSync to return an error * fix test * types: validate Part#Proof add ValidateBasic to crypto/merkle/SimpleProof * cs: limit max bit array size and block parts count * cs: test new limits * cs: only assert important stuff * update changelog and bump version to 0.32.7 * fixes after Ethan's review * align max wal msg and max consensus msg sizes * fix tests * fix test * add change log for 31.11 --- crypto/merkle/simple_proof.go | 29 +++++++++++++++++++++++ crypto/merkle/simple_proof_test.go | 38 ++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 crypto/merkle/simple_proof_test.go diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index da32157..93b6f9f 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -5,6 +5,11 @@ import ( "fmt" "github.com/pkg/errors" + "github.com/tendermint/tendermint/crypto/tmhash" +) + +const ( + maxAunts = 100 ) // SimpleProof represents a simple Merkle proof. @@ -108,6 +113,30 @@ func (sp *SimpleProof) StringIndented(indent string) string { indent) } +// ValidateBasic performs basic validation. +// NOTE: - it expects LeafHash and Aunts of tmhash.Size size +// - it expects no more than 100 aunts +func (sp *SimpleProof) ValidateBasic() error { + if sp.Total < 0 { + return errors.New("negative Total") + } + if sp.Index < 0 { + return errors.New("negative Index") + } + if len(sp.LeafHash) != tmhash.Size { + return errors.Errorf("expected LeafHash size to be %d, got %d", tmhash.Size, len(sp.LeafHash)) + } + if len(sp.Aunts) > maxAunts { + return errors.Errorf("expected no more than %d aunts, got %d", maxAunts, len(sp.Aunts)) + } + for i, auntHash := range sp.Aunts { + if len(auntHash) != tmhash.Size { + return errors.Errorf("expected Aunts#%d size to be %d, got %d", i, tmhash.Size, len(auntHash)) + } + } + return nil +} + // Use the leafHash and innerHashes to get the root merkle hash. // If the length of the innerHashes slice isn't exactly correct, the result is nil. // Recursive impl. diff --git a/crypto/merkle/simple_proof_test.go b/crypto/merkle/simple_proof_test.go new file mode 100644 index 0000000..1a51790 --- /dev/null +++ b/crypto/merkle/simple_proof_test.go @@ -0,0 +1,38 @@ +package merkle + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSimpleProofValidateBasic(t *testing.T) { + testCases := []struct { + testName string + malleateProof func(*SimpleProof) + errStr string + }{ + {"Good", func(sp *SimpleProof) {}, ""}, + {"Negative Total", func(sp *SimpleProof) { sp.Total = -1 }, "negative Total"}, + {"Negative Index", func(sp *SimpleProof) { sp.Index = -1 }, "negative Index"}, + {"Invalid LeafHash", func(sp *SimpleProof) { sp.LeafHash = make([]byte, 10) }, "expected LeafHash size to be 32, got 10"}, + {"Too many Aunts", func(sp *SimpleProof) { sp.Aunts = make([][]byte, maxAunts+1) }, "expected no more than 100 aunts, got 101"}, + {"Invalid Aunt", func(sp *SimpleProof) { sp.Aunts[0] = make([]byte, 10) }, "expected Aunts#0 size to be 32, got 10"}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.testName, func(t *testing.T) { + _, proofs := SimpleProofsFromByteSlices([][]byte{ + []byte("apple"), + []byte("watermelon"), + []byte("kiwi"), + }) + tc.malleateProof(proofs[0]) + err := proofs[0].ValidateBasic() + if tc.errStr != "" { + assert.Contains(t, err.Error(), tc.errStr) + } + }) + } +} From 668318f928cf6c45ada1526109e94570facf9810 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 1 Nov 2019 18:16:53 -0400 Subject: [PATCH 37/86] Add more comments about the hard-coded limits (#4100) * crypto: expose MaxAunts for documentation purposes * types: update godoc for new maxes * docs: make hard-coded limits more explicit * wal: add todo to clarify max size * shorten lines in test --- crypto/merkle/simple_proof.go | 13 ++++++++----- crypto/merkle/simple_proof_test.go | 9 ++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 93b6f9f..79a4729 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -9,7 +9,10 @@ import ( ) const ( - maxAunts = 100 + // MaxAunts is the maximum number of aunts that can be included in a SimpleProof. + // This corresponds to a tree of size 2^100, which should be sufficient for all conceivable purposes. + // This maximum helps prevent Denial-of-Service attacks by limitting the size of the proofs. + MaxAunts = 100 ) // SimpleProof represents a simple Merkle proof. @@ -114,8 +117,8 @@ func (sp *SimpleProof) StringIndented(indent string) string { } // ValidateBasic performs basic validation. -// NOTE: - it expects LeafHash and Aunts of tmhash.Size size -// - it expects no more than 100 aunts +// NOTE: it expects the LeafHash and the elements of Aunts to be of size tmhash.Size, +// and it expects at most MaxAunts elements in Aunts. func (sp *SimpleProof) ValidateBasic() error { if sp.Total < 0 { return errors.New("negative Total") @@ -126,8 +129,8 @@ func (sp *SimpleProof) ValidateBasic() error { if len(sp.LeafHash) != tmhash.Size { return errors.Errorf("expected LeafHash size to be %d, got %d", tmhash.Size, len(sp.LeafHash)) } - if len(sp.Aunts) > maxAunts { - return errors.Errorf("expected no more than %d aunts, got %d", maxAunts, len(sp.Aunts)) + if len(sp.Aunts) > MaxAunts { + return errors.Errorf("expected no more than %d aunts, got %d", MaxAunts, len(sp.Aunts)) } for i, auntHash := range sp.Aunts { if len(auntHash) != tmhash.Size { diff --git a/crypto/merkle/simple_proof_test.go b/crypto/merkle/simple_proof_test.go index 1a51790..68e6912 100644 --- a/crypto/merkle/simple_proof_test.go +++ b/crypto/merkle/simple_proof_test.go @@ -15,9 +15,12 @@ func TestSimpleProofValidateBasic(t *testing.T) { {"Good", func(sp *SimpleProof) {}, ""}, {"Negative Total", func(sp *SimpleProof) { sp.Total = -1 }, "negative Total"}, {"Negative Index", func(sp *SimpleProof) { sp.Index = -1 }, "negative Index"}, - {"Invalid LeafHash", func(sp *SimpleProof) { sp.LeafHash = make([]byte, 10) }, "expected LeafHash size to be 32, got 10"}, - {"Too many Aunts", func(sp *SimpleProof) { sp.Aunts = make([][]byte, maxAunts+1) }, "expected no more than 100 aunts, got 101"}, - {"Invalid Aunt", func(sp *SimpleProof) { sp.Aunts[0] = make([]byte, 10) }, "expected Aunts#0 size to be 32, got 10"}, + {"Invalid LeafHash", func(sp *SimpleProof) { sp.LeafHash = make([]byte, 10) }, + "expected LeafHash size to be 32, got 10"}, + {"Too many Aunts", func(sp *SimpleProof) { sp.Aunts = make([][]byte, MaxAunts+1) }, + "expected no more than 100 aunts, got 101"}, + {"Invalid Aunt", func(sp *SimpleProof) { sp.Aunts[0] = make([]byte, 10) }, + "expected Aunts#0 size to be 32, got 10"}, } for _, tc := range testCases { From 644cb8cab8a9797e3359adf0ac8ab3a67a2dabbd Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sat, 16 Nov 2019 19:35:39 +0400 Subject: [PATCH 38/86] linters: enable stylecheck (#4153) Refs #3262 --- crypto/merkle/proof.go | 8 ++++---- crypto/merkle/proof_test.go | 4 ++-- crypto/merkle/rfc6962_test.go | 6 +++--- crypto/merkle/simple_proof.go | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index ad101d9..50bcdd0 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -44,11 +44,11 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er key := op.GetKey() if len(key) != 0 { if len(keys) == 0 { - return errors.Errorf("Key path has insufficient # of parts: expected no more keys but got %+v", string(key)) + return errors.Errorf("key path has insufficient # of parts: expected no more keys but got %+v", string(key)) } lastKey := keys[len(keys)-1] if !bytes.Equal(lastKey, key) { - return errors.Errorf("Key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) + return errors.Errorf("key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) } keys = keys[:len(keys)-1] } @@ -58,10 +58,10 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er } } if !bytes.Equal(root, args[0]) { - return errors.Errorf("Calculated root hash is invalid: expected %+v but got %+v", root, args[0]) + return errors.Errorf("calculated root hash is invalid: expected %+v but got %+v", root, args[0]) } if len(keys) != 0 { - return errors.New("Keypath not consumed all") + return errors.New("keypath not consumed all") } return nil } diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 4dc916a..c24e791 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -50,10 +50,10 @@ func (dop DominoOp) ProofOp() ProofOp { func (dop DominoOp) Run(input [][]byte) (output [][]byte, err error) { if len(input) != 1 { - return nil, errors.New("Expected input of length 1") + return nil, errors.New("expected input of length 1") } if string(input[0]) != dop.Input { - return nil, errors.Errorf("Expected input %v, got %v", + return nil, errors.Errorf("expected input %v, got %v", dop.Input, string(input[0])) } return [][]byte{[]byte(dop.Output)}, nil diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go index e2fe7f6..6c50816 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/crypto/merkle/rfc6962_test.go @@ -77,7 +77,7 @@ func TestRFC6962HasherCollisions(t *testing.T) { _, leafHashTrail = trailsFromByteSlices([][]byte{leaf2}) hash2 := leafHashTrail.Hash if bytes.Equal(hash1, hash2) { - t.Errorf("Leaf hashes should differ, but both are %x", hash1) + t.Errorf("leaf hashes should differ, but both are %x", hash1) } // Compute an intermediate subtree hash. _, subHash1Trail := trailsFromByteSlices([][]byte{hash1, hash2}) @@ -87,12 +87,12 @@ func TestRFC6962HasherCollisions(t *testing.T) { _, forgedHashTrail := trailsFromByteSlices([][]byte{preimage}) forgedHash := forgedHashTrail.Hash if bytes.Equal(subHash1, forgedHash) { - t.Errorf("Hasher is not second-preimage resistant") + t.Errorf("hasher is not second-preimage resistant") } // Swap the order of nodes and check that the hash is different. _, subHash2Trail := trailsFromByteSlices([][]byte{hash2, hash1}) subHash2 := subHash2Trail.Hash if bytes.Equal(subHash1, subHash2) { - t.Errorf("Subtree hash does not depend on the order of leaves") + t.Errorf("subtree hash does not depend on the order of leaves") } } diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 79a4729..660bf23 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -76,10 +76,10 @@ func SimpleProofsFromMap(m map[string][]byte) (rootHash []byte, proofs map[strin func (sp *SimpleProof) Verify(rootHash []byte, leaf []byte) error { leafHash := leafHash(leaf) if sp.Total < 0 { - return errors.New("Proof total must be positive") + return errors.New("proof total must be positive") } if sp.Index < 0 { - return errors.New("Proof index cannot be negative") + return errors.New("proof index cannot be negative") } if !bytes.Equal(sp.LeafHash, leafHash) { return errors.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) From 7eb4753fed29acf7c104bed3731f83d97d95b501 Mon Sep 17 00:00:00 2001 From: Anthony Date: Thu, 21 Nov 2019 15:00:33 +0700 Subject: [PATCH 39/86] prefix proto types (#4007) Fixes #3986 This pull request is prefixing all the types in proto to avoid conflict. When a go application is using Tendermint as a library and also define similar types in gogo proto some conflicts might occur (as types is a common package in go). By prefixing the types with tendermint, this highly reduces the risk of conflicts. BREAKING CHANGE. This modification breaks the ABCI Application endpoint. What was accessible before with `/types.ABCIApplication/Flush` is now accessible with `/tendermint.abci.types.ABCIApplication/Flush`. --- crypto/merkle/merkle.pb.go | 30 ++++++++++++++++-------------- crypto/merkle/merkle.proto | 3 ++- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/crypto/merkle/merkle.pb.go b/crypto/merkle/merkle.pb.go index 2ebbee1..84d01e5 100644 --- a/crypto/merkle/merkle.pb.go +++ b/crypto/merkle/merkle.pb.go @@ -139,27 +139,29 @@ func (m *Proof) GetOps() []ProofOp { } func init() { - proto.RegisterType((*ProofOp)(nil), "merkle.ProofOp") - proto.RegisterType((*Proof)(nil), "merkle.Proof") + proto.RegisterType((*ProofOp)(nil), "tendermint.crypto.merkle.ProofOp") + proto.RegisterType((*Proof)(nil), "tendermint.crypto.merkle.Proof") } func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_9c1c2162d560d38e) } var fileDescriptor_9c1c2162d560d38e = []byte{ - // 200 bytes of a gzipped FileDescriptorProto + // 226 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c, 0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25, - 0xf9, 0x42, 0x6c, 0x10, 0x9e, 0x94, 0x6e, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, - 0xae, 0x7e, 0x7a, 0x7e, 0x7a, 0xbe, 0x3e, 0x58, 0x3a, 0xa9, 0x34, 0x0d, 0xcc, 0x03, 0x73, 0xc0, - 0x2c, 0x88, 0x36, 0x25, 0x67, 0x2e, 0xf6, 0x80, 0xa2, 0xfc, 0xfc, 0x34, 0xff, 0x02, 0x21, 0x21, - 0x2e, 0x96, 0x92, 0xca, 0x82, 0x54, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, 0x20, 0x30, 0x5b, 0x48, - 0x80, 0x8b, 0x39, 0x3b, 0xb5, 0x52, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xc4, 0x04, 0xa9, - 0x4a, 0x49, 0x2c, 0x49, 0x94, 0x60, 0x06, 0x0b, 0x81, 0xd9, 0x4a, 0x06, 0x5c, 0xac, 0x60, 0x43, - 0x84, 0xd4, 0xb9, 0x98, 0xf3, 0x0b, 0x8a, 0x25, 0x18, 0x15, 0x98, 0x35, 0xb8, 0x8d, 0xf8, 0xf5, - 0xa0, 0x0e, 0x84, 0x5a, 0xe0, 0xc4, 0x72, 0xe2, 0x9e, 0x3c, 0x43, 0x10, 0x48, 0x85, 0x93, 0xc8, - 0x8f, 0x87, 0x72, 0x8c, 0x2b, 0x1e, 0xc9, 0x31, 0x9e, 0x78, 0x24, 0xc7, 0x78, 0xe1, 0x91, 0x1c, - 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x49, 0x6c, 0x60, 0x37, 0x19, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, - 0xb9, 0x2b, 0x0f, 0xd1, 0xe8, 0x00, 0x00, 0x00, + 0xf9, 0x42, 0x12, 0x25, 0xa9, 0x79, 0x29, 0xa9, 0x45, 0xb9, 0x99, 0x79, 0x25, 0x7a, 0x10, 0x65, + 0x7a, 0x10, 0x79, 0x29, 0xdd, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0xfd, + 0xf4, 0xfc, 0xf4, 0x7c, 0x7d, 0xb0, 0x86, 0xa4, 0xd2, 0x34, 0x30, 0x0f, 0xcc, 0x01, 0xb3, 0x20, + 0x06, 0x29, 0x39, 0x73, 0xb1, 0x07, 0x14, 0xe5, 0xe7, 0xa7, 0xf9, 0x17, 0x08, 0x09, 0x71, 0xb1, + 0x94, 0x54, 0x16, 0xa4, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x81, 0xd9, 0x42, 0x02, 0x5c, + 0xcc, 0xd9, 0xa9, 0x95, 0x12, 0x4c, 0x0a, 0x8c, 0x1a, 0x3c, 0x41, 0x20, 0x26, 0x48, 0x55, 0x4a, + 0x62, 0x49, 0xa2, 0x04, 0x33, 0x58, 0x08, 0xcc, 0x56, 0x72, 0xe2, 0x62, 0x05, 0x1b, 0x22, 0x64, + 0xc9, 0xc5, 0x9c, 0x5f, 0x50, 0x2c, 0xc1, 0xa8, 0xc0, 0xac, 0xc1, 0x6d, 0xa4, 0xa8, 0x87, 0xcb, + 0x91, 0x7a, 0x50, 0x2b, 0x9d, 0x58, 0x4e, 0xdc, 0x93, 0x67, 0x08, 0x02, 0xe9, 0x71, 0x72, 0xf9, + 0xf1, 0x50, 0x8e, 0x71, 0xc5, 0x23, 0x39, 0xc6, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, + 0x7c, 0xf0, 0x48, 0x8e, 0x31, 0x4a, 0x0f, 0xc9, 0x37, 0x08, 0xd3, 0x90, 0x99, 0x28, 0x81, 0x94, + 0xc4, 0x06, 0xf6, 0x95, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0xbb, 0x95, 0x22, 0x4e, 0x3c, 0x01, + 0x00, 0x00, } func (this *ProofOp) Equal(that interface{}) bool { diff --git a/crypto/merkle/merkle.proto b/crypto/merkle/merkle.proto index 8a6c467..6685492 100644 --- a/crypto/merkle/merkle.proto +++ b/crypto/merkle/merkle.proto @@ -1,5 +1,6 @@ syntax = "proto3"; -package merkle; +package tendermint.crypto.merkle; +option go_package = "github.com/tendermint/tendermint/crypto/merkle"; // For more information on gogo.proto, see: // https://github.com/gogo/protobuf/blob/master/extensions.md From f8d638807d27da09f34f4d7a65cb61194181ab72 Mon Sep 17 00:00:00 2001 From: Marko Date: Thu, 5 Dec 2019 10:12:08 +0100 Subject: [PATCH 40/86] lint: Enable Golint (#4212) * Fix many golint errors * Fix golint errors in the 'lite' package * Don't export Pool.store * Fix typo * Revert unwanted changes * Fix errors in counter package * Fix linter errors in kvstore package * Fix linter error in example package * Fix error in tests package * Fix linter errors in v2 package * Fix linter errors in consensus package * Fix linter errors in evidence package * Fix linter error in fail package * Fix linter errors in query package * Fix linter errors in core package * Fix linter errors in node package * Fix linter errors in mempool package * Fix linter error in conn package * Fix linter errors in pex package * Rename PEXReactor export to Reactor * Fix linter errors in trust package * Fix linter errors in upnp package * Fix linter errors in p2p package * Fix linter errors in proxy package * Fix linter errors in mock_test package * Fix linter error in client_test package * Fix linter errors in coretypes package * Fix linter errors in coregrpc package * Fix linter errors in rpcserver package * Fix linter errors in rpctypes package * Fix linter errors in rpctest package * Fix linter error in json2wal script * Fix linter error in wal2json script * Fix linter errors in kv package * Fix linter error in state package * Fix linter error in grpc_client * Fix linter errors in types package * Fix linter error in version package * Fix remaining errors * Address review comments * Fix broken tests * Reconcile package coregrpc * Fix golangci bot error * Fix new golint errors * Fix broken reference * Enable golint linter * minor changes to bring golint into line * fix failing test * fix pex reactor naming * address PR comments --- crypto/merkle/simple_tree.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 03dc9d9..2a57bbe 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -82,9 +82,9 @@ func SimpleHashFromByteSlicesIterative(input [][]byte) []byte { rp += 2 } else { items[wp] = items[rp] - rp += 1 + rp++ } - wp += 1 + wp++ } size = wp } From 3d0e4bb5ff09d0292d04e8410c5dbf77196d350e Mon Sep 17 00:00:00 2001 From: Marko Date: Tue, 10 Dec 2019 12:40:01 +0100 Subject: [PATCH 41/86] libs/common: refactor libs/common 01 (#4230) * libs/common: Refactor libs/common 01 Signed-off-by: Marko Baricevic * regenerate proto files, move intslice to where its used * update kv.KVPair(s) to kv.Pair(s) * add changelog entry * make intInSlice private --- crypto/merkle/simple_map.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go index bde4422..36434f6 100644 --- a/crypto/merkle/simple_map.go +++ b/crypto/merkle/simple_map.go @@ -5,14 +5,14 @@ import ( amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/crypto/tmhash" - cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/kv" ) // Merkle tree from a map. // Leaves are `hash(key) | hash(value)`. // Leaves are sorted before Merkle hashing. type simpleMap struct { - kvs cmn.KVPairs + kvs kv.Pairs sorted bool } @@ -33,7 +33,7 @@ func (sm *simpleMap) Set(key string, value []byte) { // and make a determination to fetch or not. vhash := tmhash.Sum(value) - sm.kvs = append(sm.kvs, cmn.KVPair{ + sm.kvs = append(sm.kvs, kv.Pair{ Key: []byte(key), Value: vhash, }) @@ -56,9 +56,9 @@ func (sm *simpleMap) Sort() { // Returns a copy of sorted KVPairs. // NOTE these contain the hashed key and value. -func (sm *simpleMap) KVPairs() cmn.KVPairs { +func (sm *simpleMap) KVPairs() kv.Pairs { sm.Sort() - kvs := make(cmn.KVPairs, len(sm.kvs)) + kvs := make(kv.Pairs, len(sm.kvs)) copy(kvs, sm.kvs) return kvs } @@ -68,7 +68,7 @@ func (sm *simpleMap) KVPairs() cmn.KVPairs { // A local extension to KVPair that can be hashed. // Key and value are length prefixed and concatenated, // then hashed. -type KVPair cmn.KVPair +type KVPair kv.Pair // Bytes returns key || value, with both the // key and value length prefixed. @@ -85,7 +85,7 @@ func (kv KVPair) Bytes() []byte { return b.Bytes() } -func hashKVPairs(kvs cmn.KVPairs) []byte { +func hashKVPairs(kvs kv.Pairs) []byte { kvsH := make([][]byte, len(kvs)) for i, kvp := range kvs { kvsH[i] = KVPair(kvp).Bytes() From ded6685fe2c761b01c61ba1e2e2bbd991933a6d5 Mon Sep 17 00:00:00 2001 From: Marko Date: Tue, 10 Dec 2019 18:38:34 +0100 Subject: [PATCH 42/86] libs/common: refactor libs/common 2 (#4231) * libs/common: refactor libs/common 2 - move random function to there own pkg Signed-off-by: Marko Baricevic * change imports and usage throughout repo * fix goimports * add changelog entry --- crypto/merkle/simple_tree_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index 5bbe294..c59e504 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/rand" . "github.com/tendermint/tendermint/libs/test" "github.com/tendermint/tendermint/crypto/tmhash" @@ -23,7 +23,7 @@ func TestSimpleProof(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(cmn.RandBytes(tmhash.Size)) + items[i] = testItem(rand.RandBytes(tmhash.Size)) } rootHash := SimpleHashFromByteSlices(items) @@ -47,7 +47,7 @@ func TestSimpleProof(t *testing.T) { // Trail too long should make it fail origAunts := proof.Aunts - proof.Aunts = append(proof.Aunts, cmn.RandBytes(32)) + proof.Aunts = append(proof.Aunts, rand.RandBytes(32)) err = proof.Verify(rootHash, item) require.Error(t, err, "Expected verification to fail for wrong trail length") @@ -76,7 +76,7 @@ func TestSimpleHashAlternatives(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(cmn.RandBytes(tmhash.Size)) + items[i] = testItem(rand.RandBytes(tmhash.Size)) } rootHash1 := SimpleHashFromByteSlicesIterative(items) @@ -89,7 +89,7 @@ func BenchmarkSimpleHashAlternatives(b *testing.B) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(cmn.RandBytes(tmhash.Size)) + items[i] = testItem(rand.RandBytes(tmhash.Size)) } b.ResetTimer() From 81a88381fc118681c92057d4f8524a5dee28a073 Mon Sep 17 00:00:00 2001 From: Marko Date: Tue, 17 Dec 2019 13:02:45 +0100 Subject: [PATCH 43/86] lint: golint issue fixes (#4258) * lint: golint issue fixes - on my local machine golint is a lot stricter than the bot so slowly going through and fixing things. Signed-off-by: Marko Baricevic * more fixes from golint * remove isPeerPersistentFn * add changelog entry --- crypto/merkle/simple_tree_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index c59e504..665017a 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/libs/rand" + tmrand "github.com/tendermint/tendermint/libs/rand" . "github.com/tendermint/tendermint/libs/test" "github.com/tendermint/tendermint/crypto/tmhash" @@ -23,7 +23,7 @@ func TestSimpleProof(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(rand.RandBytes(tmhash.Size)) + items[i] = testItem(tmrand.Bytes(tmhash.Size)) } rootHash := SimpleHashFromByteSlices(items) @@ -47,7 +47,7 @@ func TestSimpleProof(t *testing.T) { // Trail too long should make it fail origAunts := proof.Aunts - proof.Aunts = append(proof.Aunts, rand.RandBytes(32)) + proof.Aunts = append(proof.Aunts, tmrand.Bytes(32)) err = proof.Verify(rootHash, item) require.Error(t, err, "Expected verification to fail for wrong trail length") @@ -76,7 +76,7 @@ func TestSimpleHashAlternatives(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(rand.RandBytes(tmhash.Size)) + items[i] = testItem(tmrand.Bytes(tmhash.Size)) } rootHash1 := SimpleHashFromByteSlicesIterative(items) @@ -89,7 +89,7 @@ func BenchmarkSimpleHashAlternatives(b *testing.B) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(rand.RandBytes(tmhash.Size)) + items[i] = testItem(tmrand.Bytes(tmhash.Size)) } b.ResetTimer() From b2ec315a054d5b9ff6fd5fd319f311527904fd46 Mon Sep 17 00:00:00 2001 From: Marko Date: Tue, 11 Feb 2020 10:31:15 +0100 Subject: [PATCH 44/86] proto: add buf and protogen script (#4369) * proto: add buf and protogen script - add buf with minimal changes - add protogen script to easier generate proto files Signed-off-by: Marko Baricevic * add protoc needs * add some needed shell cmds * remove buf from tools as it is not needed everytime * add proto lint and breakage to ci * add section in changelog and upgrading files * address pr comments * remove space in circle config * remove spaces in makefile comment * add section on contributing on how to work with proto * bump buf to 0.7 * test bufbuild image * test install make in bufbuild image * revert to tendermintdev image * Update Makefile Co-Authored-By: Anton Kaliaev Co-authored-by: Anton Kaliaev --- crypto/merkle/merkle.pb.go | 84 +++++++++++++------------------------- crypto/merkle/merkle.proto | 2 +- 2 files changed, 30 insertions(+), 56 deletions(-) diff --git a/crypto/merkle/merkle.pb.go b/crypto/merkle/merkle.pb.go index 84d01e5..80823dd 100644 --- a/crypto/merkle/merkle.pb.go +++ b/crypto/merkle/merkle.pb.go @@ -146,22 +146,22 @@ func init() { func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_9c1c2162d560d38e) } var fileDescriptor_9c1c2162d560d38e = []byte{ - // 226 bytes of a gzipped FileDescriptorProto + // 230 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c, 0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0x12, 0x25, 0xa9, 0x79, 0x29, 0xa9, 0x45, 0xb9, 0x99, 0x79, 0x25, 0x7a, 0x10, 0x65, - 0x7a, 0x10, 0x79, 0x29, 0xdd, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0xfd, - 0xf4, 0xfc, 0xf4, 0x7c, 0x7d, 0xb0, 0x86, 0xa4, 0xd2, 0x34, 0x30, 0x0f, 0xcc, 0x01, 0xb3, 0x20, - 0x06, 0x29, 0x39, 0x73, 0xb1, 0x07, 0x14, 0xe5, 0xe7, 0xa7, 0xf9, 0x17, 0x08, 0x09, 0x71, 0xb1, - 0x94, 0x54, 0x16, 0xa4, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x81, 0xd9, 0x42, 0x02, 0x5c, - 0xcc, 0xd9, 0xa9, 0x95, 0x12, 0x4c, 0x0a, 0x8c, 0x1a, 0x3c, 0x41, 0x20, 0x26, 0x48, 0x55, 0x4a, - 0x62, 0x49, 0xa2, 0x04, 0x33, 0x58, 0x08, 0xcc, 0x56, 0x72, 0xe2, 0x62, 0x05, 0x1b, 0x22, 0x64, - 0xc9, 0xc5, 0x9c, 0x5f, 0x50, 0x2c, 0xc1, 0xa8, 0xc0, 0xac, 0xc1, 0x6d, 0xa4, 0xa8, 0x87, 0xcb, - 0x91, 0x7a, 0x50, 0x2b, 0x9d, 0x58, 0x4e, 0xdc, 0x93, 0x67, 0x08, 0x02, 0xe9, 0x71, 0x72, 0xf9, - 0xf1, 0x50, 0x8e, 0x71, 0xc5, 0x23, 0x39, 0xc6, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, - 0x7c, 0xf0, 0x48, 0x8e, 0x31, 0x4a, 0x0f, 0xc9, 0x37, 0x08, 0xd3, 0x90, 0x99, 0x28, 0x81, 0x94, - 0xc4, 0x06, 0xf6, 0x95, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0xbb, 0x95, 0x22, 0x4e, 0x3c, 0x01, - 0x00, 0x00, + 0x7a, 0x10, 0x79, 0x29, 0xb5, 0x92, 0x8c, 0xcc, 0xa2, 0x94, 0xf8, 0x82, 0xc4, 0xa2, 0x92, 0x4a, + 0x7d, 0xb0, 0x62, 0xfd, 0xf4, 0xfc, 0xf4, 0x7c, 0x04, 0x0b, 0x62, 0x82, 0x92, 0x33, 0x17, 0x7b, + 0x40, 0x51, 0x7e, 0x7e, 0x9a, 0x7f, 0x81, 0x90, 0x10, 0x17, 0x4b, 0x49, 0x65, 0x41, 0xaa, 0x04, + 0xa3, 0x02, 0xa3, 0x06, 0x67, 0x10, 0x98, 0x2d, 0x24, 0xc0, 0xc5, 0x9c, 0x9d, 0x5a, 0x29, 0xc1, + 0xa4, 0xc0, 0xa8, 0xc1, 0x13, 0x04, 0x62, 0x82, 0x54, 0xa5, 0x24, 0x96, 0x24, 0x4a, 0x30, 0x83, + 0x85, 0xc0, 0x6c, 0x25, 0x27, 0x2e, 0x56, 0xb0, 0x21, 0x42, 0x96, 0x5c, 0xcc, 0xf9, 0x05, 0xc5, + 0x12, 0x8c, 0x0a, 0xcc, 0x1a, 0xdc, 0x46, 0x8a, 0x7a, 0xb8, 0x5c, 0xa7, 0x07, 0xb5, 0xd2, 0x89, + 0xe5, 0xc4, 0x3d, 0x79, 0x86, 0x20, 0x90, 0x1e, 0x27, 0x97, 0x1f, 0x0f, 0xe5, 0x18, 0x57, 0x3c, + 0x92, 0x63, 0x3c, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0xa3, + 0xf4, 0xd2, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, 0xf5, 0x11, 0xa6, 0x21, 0x33, + 0x51, 0x42, 0x27, 0x89, 0x0d, 0xec, 0x2b, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc8, 0xcc, + 0x2c, 0x91, 0x35, 0x01, 0x00, 0x00, } func (this *ProofOp) Equal(that interface{}) bool { @@ -729,6 +729,7 @@ func (m *Proof) Unmarshal(dAtA []byte) error { func skipMerkle(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 + depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -760,10 +761,8 @@ func skipMerkle(dAtA []byte) (n int, err error) { break } } - return iNdEx, nil case 1: iNdEx += 8 - return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -784,55 +783,30 @@ func skipMerkle(dAtA []byte) (n int, err error) { return 0, ErrInvalidLengthMerkle } iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthMerkle - } - return iNdEx, nil case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowMerkle - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipMerkle(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthMerkle - } - } - return iNdEx, nil + depth++ case 4: - return iNdEx, nil + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupMerkle + } + depth-- case 5: iNdEx += 4 - return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } + if iNdEx < 0 { + return 0, ErrInvalidLengthMerkle + } + if depth == 0 { + return iNdEx, nil + } } - panic("unreachable") + return 0, io.ErrUnexpectedEOF } var ( - ErrInvalidLengthMerkle = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowMerkle = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthMerkle = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowMerkle = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupMerkle = fmt.Errorf("proto: unexpected end of group") ) diff --git a/crypto/merkle/merkle.proto b/crypto/merkle/merkle.proto index 6685492..9dbb2be 100644 --- a/crypto/merkle/merkle.proto +++ b/crypto/merkle/merkle.proto @@ -4,7 +4,7 @@ option go_package = "github.com/tendermint/tendermint/crypto/merkle"; // For more information on gogo.proto, see: // https://github.com/gogo/protobuf/blob/master/extensions.md -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "third_party/proto/gogoproto/gogo.proto"; option (gogoproto.marshaler_all) = true; option (gogoproto.unmarshaler_all) = true; From 087a589b5a6f35915d865b540116bdf185b89037 Mon Sep 17 00:00:00 2001 From: Marko Date: Tue, 11 Feb 2020 15:07:05 +0100 Subject: [PATCH 45/86] proto: minor linting to proto files (#4386) * proto: minor linting minor linting after working with the proto files in the sdk. there is no logic change just spacing fixes Signed-off-by: Marko Baricevic * hardcore linting --- crypto/merkle/merkle.proto | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crypto/merkle/merkle.proto b/crypto/merkle/merkle.proto index 9dbb2be..159fc58 100644 --- a/crypto/merkle/merkle.proto +++ b/crypto/merkle/merkle.proto @@ -1,17 +1,17 @@ syntax = "proto3"; package tendermint.crypto.merkle; -option go_package = "github.com/tendermint/tendermint/crypto/merkle"; +option go_package = "github.com/tendermint/tendermint/crypto/merkle"; // For more information on gogo.proto, see: // https://github.com/gogo/protobuf/blob/master/extensions.md import "third_party/proto/gogoproto/gogo.proto"; -option (gogoproto.marshaler_all) = true; +option (gogoproto.marshaler_all) = true; option (gogoproto.unmarshaler_all) = true; -option (gogoproto.sizer_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.populate_all) = true; -option (gogoproto.equal_all) = true; +option (gogoproto.equal_all) = true; //---------------------------------------- // Message types @@ -21,11 +21,11 @@ option (gogoproto.equal_all) = true; // for example neighbouring node hash message ProofOp { string type = 1; - bytes key = 2; - bytes data = 3; + bytes key = 2; + bytes data = 3; } // Proof is Merkle proof defined by the list of ProofOps message Proof { - repeated ProofOp ops = 1 [(gogoproto.nullable)=false]; + repeated ProofOp ops = 1 [(gogoproto.nullable) = false]; } From 0638cf5784bb8470fb632f4e53058d0f0e8cae6c Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 23 Mar 2020 09:19:26 +0100 Subject: [PATCH 46/86] format: add format cmd & goimport repo (#4586) * format: add format cmd & goimport repo - replaced format command - added goimports to format command - ran goimports Signed-off-by: Marko Baricevic * fix outliers & undo proto file changes --- crypto/merkle/simple_map.go | 1 + crypto/merkle/simple_proof.go | 1 + 2 files changed, 2 insertions(+) diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go index 36434f6..840bebd 100644 --- a/crypto/merkle/simple_map.go +++ b/crypto/merkle/simple_map.go @@ -4,6 +4,7 @@ import ( "bytes" amino "github.com/tendermint/go-amino" + "github.com/tendermint/tendermint/crypto/tmhash" "github.com/tendermint/tendermint/libs/kv" ) diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 660bf23..44b97f6 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/pkg/errors" + "github.com/tendermint/tendermint/crypto/tmhash" ) From 1053e48bdb8cfedd43bf6319e8ac6721eeb86e57 Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Thu, 9 Apr 2020 00:03:51 +0200 Subject: [PATCH 47/86] replace all imports of to use new org (#5) - effectively replace all "github.com/tendermint/tendermint/* with "github.com/lazyledger/lazyledger-core/* - update go.mod --- crypto/merkle/hash.go | 2 +- crypto/merkle/proof_simple_value.go | 2 +- crypto/merkle/rfc6962_test.go | 2 +- crypto/merkle/simple_map.go | 4 ++-- crypto/merkle/simple_proof.go | 2 +- crypto/merkle/simple_tree_test.go | 6 +++--- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index 4e24046..23f087b 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -1,7 +1,7 @@ package merkle import ( - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/lazyledger/lazyledger-core/crypto/tmhash" ) // TODO: make these have a large predefined capacity diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_simple_value.go index 55337b7..9e8cfdc 100644 --- a/crypto/merkle/proof_simple_value.go +++ b/crypto/merkle/proof_simple_value.go @@ -6,7 +6,7 @@ import ( "github.com/pkg/errors" - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/lazyledger/lazyledger-core/crypto/tmhash" ) const ProofOpSimpleValue = "simple:v" diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go index 6c50816..2b483c6 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/crypto/merkle/rfc6962_test.go @@ -20,7 +20,7 @@ import ( "encoding/hex" "testing" - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/lazyledger/lazyledger-core/crypto/tmhash" ) func TestRFC6962Hasher(t *testing.T) { diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go index 840bebd..f62244e 100644 --- a/crypto/merkle/simple_map.go +++ b/crypto/merkle/simple_map.go @@ -5,8 +5,8 @@ import ( amino "github.com/tendermint/go-amino" - "github.com/tendermint/tendermint/crypto/tmhash" - "github.com/tendermint/tendermint/libs/kv" + "github.com/lazyledger/lazyledger-core/crypto/tmhash" + "github.com/lazyledger/lazyledger-core/libs/kv" ) // Merkle tree from a map. diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 44b97f6..0a8b7d8 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -6,7 +6,7 @@ import ( "github.com/pkg/errors" - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/lazyledger/lazyledger-core/crypto/tmhash" ) const ( diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index 665017a..ee53951 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -5,10 +5,10 @@ import ( "github.com/stretchr/testify/require" - tmrand "github.com/tendermint/tendermint/libs/rand" - . "github.com/tendermint/tendermint/libs/test" + tmrand "github.com/lazyledger/lazyledger-core/libs/rand" + . "github.com/lazyledger/lazyledger-core/libs/test" - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/lazyledger/lazyledger-core/crypto/tmhash" ) type testItem []byte From f39c00e8e4ca093a490b2461b37bb344506ee93c Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 27 Apr 2020 17:33:20 +0200 Subject: [PATCH 48/86] crypto: remove SimpleHashFromMap() and SimpleProofsFromMap() Fixes #2593 @alexanderbez This is used in a single place in the SDK, how upset are you about removing it? ______ For contributor use: - [x] ~Wrote tests~ - [x] Updated CHANGELOG_PENDING.md - [x] Linked to Github issue with discussion and accepted design OR link to spec that describes this work. - [x] Updated relevant documentation (`docs/`) and code comments - [x] Re-reviewed `Files changed` in the Github PR explorer - [x] Applied Appropriate Labels --- crypto/merkle/simple_map.go | 95 -------------------------------- crypto/merkle/simple_map_test.go | 49 ---------------- crypto/merkle/simple_proof.go | 25 --------- crypto/merkle/simple_tree.go | 12 ---- 4 files changed, 181 deletions(-) delete mode 100644 crypto/merkle/simple_map.go delete mode 100644 crypto/merkle/simple_map_test.go diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go deleted file mode 100644 index 840bebd..0000000 --- a/crypto/merkle/simple_map.go +++ /dev/null @@ -1,95 +0,0 @@ -package merkle - -import ( - "bytes" - - amino "github.com/tendermint/go-amino" - - "github.com/tendermint/tendermint/crypto/tmhash" - "github.com/tendermint/tendermint/libs/kv" -) - -// Merkle tree from a map. -// Leaves are `hash(key) | hash(value)`. -// Leaves are sorted before Merkle hashing. -type simpleMap struct { - kvs kv.Pairs - sorted bool -} - -func newSimpleMap() *simpleMap { - return &simpleMap{ - kvs: nil, - sorted: false, - } -} - -// Set creates a kv pair of the key and the hash of the value, -// and then appends it to simpleMap's kv pairs. -func (sm *simpleMap) Set(key string, value []byte) { - sm.sorted = false - - // The value is hashed, so you can - // check for equality with a cached value (say) - // and make a determination to fetch or not. - vhash := tmhash.Sum(value) - - sm.kvs = append(sm.kvs, kv.Pair{ - Key: []byte(key), - Value: vhash, - }) -} - -// Hash Merkle root hash of items sorted by key -// (UNSTABLE: and by value too if duplicate key). -func (sm *simpleMap) Hash() []byte { - sm.Sort() - return hashKVPairs(sm.kvs) -} - -func (sm *simpleMap) Sort() { - if sm.sorted { - return - } - sm.kvs.Sort() - sm.sorted = true -} - -// Returns a copy of sorted KVPairs. -// NOTE these contain the hashed key and value. -func (sm *simpleMap) KVPairs() kv.Pairs { - sm.Sort() - kvs := make(kv.Pairs, len(sm.kvs)) - copy(kvs, sm.kvs) - return kvs -} - -//---------------------------------------- - -// A local extension to KVPair that can be hashed. -// Key and value are length prefixed and concatenated, -// then hashed. -type KVPair kv.Pair - -// Bytes returns key || value, with both the -// key and value length prefixed. -func (kv KVPair) Bytes() []byte { - var b bytes.Buffer - err := amino.EncodeByteSlice(&b, kv.Key) - if err != nil { - panic(err) - } - err = amino.EncodeByteSlice(&b, kv.Value) - if err != nil { - panic(err) - } - return b.Bytes() -} - -func hashKVPairs(kvs kv.Pairs) []byte { - kvsH := make([][]byte, len(kvs)) - for i, kvp := range kvs { - kvsH[i] = KVPair(kvp).Bytes() - } - return SimpleHashFromByteSlices(kvsH) -} diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go deleted file mode 100644 index 20868a7..0000000 --- a/crypto/merkle/simple_map_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package merkle - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSimpleMap(t *testing.T) { - tests := []struct { - keys []string - values []string // each string gets converted to []byte in test - want string - }{ - {[]string{"key1"}, []string{"value1"}, "a44d3cc7daba1a4600b00a2434b30f8b970652169810d6dfa9fb1793a2189324"}, - {[]string{"key1"}, []string{"value2"}, "0638e99b3445caec9d95c05e1a3fc1487b4ddec6a952ff337080360b0dcc078c"}, - // swap order with 2 keys - { - []string{"key1", "key2"}, - []string{"value1", "value2"}, - "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3", - }, - { - []string{"key2", "key1"}, - []string{"value2", "value1"}, - "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3", - }, - // swap order with 3 keys - { - []string{"key1", "key2", "key3"}, - []string{"value1", "value2", "value3"}, - "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc", - }, - { - []string{"key1", "key3", "key2"}, - []string{"value1", "value3", "value2"}, - "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc", - }, - } - for i, tc := range tests { - db := newSimpleMap() - for i := 0; i < len(tc.keys); i++ { - db.Set(tc.keys[i], []byte(tc.values[i])) - } - got := db.Hash() - assert.Equal(t, tc.want, fmt.Sprintf("%x", got), "Hash didn't match on tc %d", i) - } -} diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 44b97f6..26c8bae 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -47,31 +47,6 @@ func SimpleProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*Simp return } -// SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values -// in the underlying key-value pairs. -// The keys are sorted before the proofs are computed. -func SimpleProofsFromMap(m map[string][]byte) (rootHash []byte, proofs map[string]*SimpleProof, keys []string) { - sm := newSimpleMap() - for k, v := range m { - sm.Set(k, v) - } - sm.Sort() - kvs := sm.kvs - kvsBytes := make([][]byte, len(kvs)) - for i, kvp := range kvs { - kvsBytes[i] = KVPair(kvp).Bytes() - } - - rootHash, proofList := SimpleProofsFromByteSlices(kvsBytes) - proofs = make(map[string]*SimpleProof) - keys = make([]string, len(proofList)) - for i, kvp := range kvs { - proofs[string(kvp.Key)] = proofList[i] - keys[i] = string(kvp.Key) - } - return -} - // Verify that the SimpleProof proves the root hash. // Check sp.Index/sp.Total manually if needed func (sp *SimpleProof) Verify(rootHash []byte, leaf []byte) error { diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 2a57bbe..d2b931e 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -91,18 +91,6 @@ func SimpleHashFromByteSlicesIterative(input [][]byte) []byte { } } -// SimpleHashFromMap computes a Merkle tree from sorted map. -// Like calling SimpleHashFromHashers with -// `item = []byte(Hash(key) | Hash(value))`, -// sorted by `item`. -func SimpleHashFromMap(m map[string][]byte) []byte { - sm := newSimpleMap() - for k, v := range m { - sm.Set(k, v) - } - return sm.Hash() -} - // getSplitPoint returns the largest power of 2 less than length func getSplitPoint(length int) int { if length < 1 { From c65d0276bbb78f66aa71c692464446795dc683bb Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 4 May 2020 09:49:53 +0200 Subject: [PATCH 49/86] lint: enable nolintlinter, disable on tests ## Description - enable nolintlint - disable linting on tests Closes: #XXX --- crypto/merkle/result.go | 1 - 1 file changed, 1 deletion(-) diff --git a/crypto/merkle/result.go b/crypto/merkle/result.go index c7bbb57..a17e38e 100644 --- a/crypto/merkle/result.go +++ b/crypto/merkle/result.go @@ -1,4 +1,3 @@ -// nolint: dupl package merkle import ( From 50d7a3e8825ce1a3da084c4d65c7360bbc6f887e Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 12 May 2020 07:35:47 +0400 Subject: [PATCH 50/86] change use of errors.Wrap to fmt.Errorf with %w verb Closes #4603 Commands used (VIM): ``` :args `rg -l errors.Wrap` :argdo normal @q | update ``` where q is a macros rewriting the `errors.Wrap` to `fmt.Errorf`. --- crypto/merkle/proof.go | 16 ++++++++-------- crypto/merkle/proof_key_path.go | 7 +++---- crypto/merkle/proof_simple_value.go | 10 ++++------ crypto/merkle/proof_test.go | 7 ++++--- crypto/merkle/simple_proof.go | 13 ++++++------- 5 files changed, 25 insertions(+), 28 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 50bcdd0..84c9bbc 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -2,8 +2,8 @@ package merkle import ( "bytes" - - "github.com/pkg/errors" + "errors" + "fmt" ) //---------------------------------------- @@ -44,11 +44,11 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er key := op.GetKey() if len(key) != 0 { if len(keys) == 0 { - return errors.Errorf("key path has insufficient # of parts: expected no more keys but got %+v", string(key)) + return fmt.Errorf("key path has insufficient # of parts: expected no more keys but got %+v", string(key)) } lastKey := keys[len(keys)-1] if !bytes.Equal(lastKey, key) { - return errors.Errorf("key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) + return fmt.Errorf("key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) } keys = keys[:len(keys)-1] } @@ -58,7 +58,7 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er } } if !bytes.Equal(root, args[0]) { - return errors.Errorf("calculated root hash is invalid: expected %+v but got %+v", root, args[0]) + return fmt.Errorf("calculated root hash is invalid: expected %+v but got %+v", root, args[0]) } if len(keys) != 0 { return errors.New("keypath not consumed all") @@ -92,7 +92,7 @@ func (prt *ProofRuntime) RegisterOpDecoder(typ string, dec OpDecoder) { func (prt *ProofRuntime) Decode(pop ProofOp) (ProofOperator, error) { decoder := prt.decoders[pop.Type] if decoder == nil { - return nil, errors.Errorf("unrecognized proof type %v", pop.Type) + return nil, fmt.Errorf("unrecognized proof type %v", pop.Type) } return decoder(pop) } @@ -102,7 +102,7 @@ func (prt *ProofRuntime) DecodeProof(proof *Proof) (ProofOperators, error) { for _, pop := range proof.Ops { operator, err := prt.Decode(pop) if err != nil { - return nil, errors.Wrap(err, "decoding a proof operator") + return nil, fmt.Errorf("decoding a proof operator: %w", err) } poz = append(poz, operator) } @@ -122,7 +122,7 @@ func (prt *ProofRuntime) VerifyAbsence(proof *Proof, root []byte, keypath string func (prt *ProofRuntime) Verify(proof *Proof, root []byte, keypath string, args [][]byte) (err error) { poz, err := prt.DecodeProof(proof) if err != nil { - return errors.Wrap(err, "decoding proof") + return fmt.Errorf("decoding proof: %w", err) } return poz.Verify(root, keypath, args) } diff --git a/crypto/merkle/proof_key_path.go b/crypto/merkle/proof_key_path.go index 7ea6785..beafc4f 100644 --- a/crypto/merkle/proof_key_path.go +++ b/crypto/merkle/proof_key_path.go @@ -2,11 +2,10 @@ package merkle import ( "encoding/hex" + "errors" "fmt" "net/url" "strings" - - "github.com/pkg/errors" ) /* @@ -96,13 +95,13 @@ func KeyPathToKeys(path string) (keys [][]byte, err error) { hexPart := part[2:] key, err := hex.DecodeString(hexPart) if err != nil { - return nil, errors.Wrapf(err, "decoding hex-encoded part #%d: /%s", i, part) + return nil, fmt.Errorf("decoding hex-encoded part #%d: /%s: %w", i, part, err) } keys[i] = key } else { key, err := url.PathUnescape(part) if err != nil { - return nil, errors.Wrapf(err, "decoding url-encoded part #%d: /%s", i, part) + return nil, fmt.Errorf("decoding url-encoded part #%d: /%s: %w", i, part, err) } keys[i] = []byte(key) // TODO Test this with random bytes, I'm not sure that it works for arbitrary bytes... } diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_simple_value.go index 55337b7..d1a0426 100644 --- a/crypto/merkle/proof_simple_value.go +++ b/crypto/merkle/proof_simple_value.go @@ -4,8 +4,6 @@ import ( "bytes" "fmt" - "github.com/pkg/errors" - "github.com/tendermint/tendermint/crypto/tmhash" ) @@ -40,12 +38,12 @@ func NewSimpleValueOp(key []byte, proof *SimpleProof) SimpleValueOp { func SimpleValueOpDecoder(pop ProofOp) (ProofOperator, error) { if pop.Type != ProofOpSimpleValue { - return nil, errors.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpSimpleValue) + return nil, fmt.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpSimpleValue) } var op SimpleValueOp // a bit strange as we'll discard this, but it works. err := cdc.UnmarshalBinaryLengthPrefixed(pop.Data, &op) if err != nil { - return nil, errors.Wrap(err, "decoding ProofOp.Data into SimpleValueOp") + return nil, fmt.Errorf("decoding ProofOp.Data into SimpleValueOp: %w", err) } return NewSimpleValueOp(pop.Key, op.Proof), nil } @@ -65,7 +63,7 @@ func (op SimpleValueOp) String() string { func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { if len(args) != 1 { - return nil, errors.Errorf("expected 1 arg, got %v", len(args)) + return nil, fmt.Errorf("expected 1 arg, got %v", len(args)) } value := args[0] hasher := tmhash.New() @@ -79,7 +77,7 @@ func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { kvhash := leafHash(bz.Bytes()) if !bytes.Equal(kvhash, op.Proof.LeafHash) { - return nil, errors.Errorf("leaf hash mismatch: want %X got %X", op.Proof.LeafHash, kvhash) + return nil, fmt.Errorf("leaf hash mismatch: want %X got %X", op.Proof.LeafHash, kvhash) } return [][]byte{ diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index c24e791..ca9a982 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -1,9 +1,10 @@ package merkle import ( + "errors" + "fmt" "testing" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" amino "github.com/tendermint/go-amino" ) @@ -34,7 +35,7 @@ func DominoOpDecoder(pop ProofOp) (ProofOperator, error) { var op DominoOp // a bit strange as we'll discard this, but it works. err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) if err != nil { - return nil, errors.Wrap(err, "decoding ProofOp.Data into SimpleValueOp") + return nil, fmt.Errorf("decoding ProofOp.Data into SimpleValueOp: %w", err) } return NewDominoOp(string(pop.Key), op.Input, op.Output), nil } @@ -53,7 +54,7 @@ func (dop DominoOp) Run(input [][]byte) (output [][]byte, err error) { return nil, errors.New("expected input of length 1") } if string(input[0]) != dop.Input { - return nil, errors.Errorf("expected input %v, got %v", + return nil, fmt.Errorf("expected input %v, got %v", dop.Input, string(input[0])) } return [][]byte{[]byte(dop.Output)}, nil diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 26c8bae..2f78431 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -2,10 +2,9 @@ package merkle import ( "bytes" + "errors" "fmt" - "github.com/pkg/errors" - "github.com/tendermint/tendermint/crypto/tmhash" ) @@ -58,11 +57,11 @@ func (sp *SimpleProof) Verify(rootHash []byte, leaf []byte) error { return errors.New("proof index cannot be negative") } if !bytes.Equal(sp.LeafHash, leafHash) { - return errors.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) + return fmt.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) } computedHash := sp.ComputeRootHash() if !bytes.Equal(computedHash, rootHash) { - return errors.Errorf("invalid root hash: wanted %X got %X", rootHash, computedHash) + return fmt.Errorf("invalid root hash: wanted %X got %X", rootHash, computedHash) } return nil } @@ -103,14 +102,14 @@ func (sp *SimpleProof) ValidateBasic() error { return errors.New("negative Index") } if len(sp.LeafHash) != tmhash.Size { - return errors.Errorf("expected LeafHash size to be %d, got %d", tmhash.Size, len(sp.LeafHash)) + return fmt.Errorf("expected LeafHash size to be %d, got %d", tmhash.Size, len(sp.LeafHash)) } if len(sp.Aunts) > MaxAunts { - return errors.Errorf("expected no more than %d aunts, got %d", MaxAunts, len(sp.Aunts)) + return fmt.Errorf("expected no more than %d aunts, got %d", MaxAunts, len(sp.Aunts)) } for i, auntHash := range sp.Aunts { if len(auntHash) != tmhash.Size { - return errors.Errorf("expected Aunts#%d size to be %d, got %d", i, tmhash.Size, len(auntHash)) + return fmt.Errorf("expected Aunts#%d size to be %d, got %d", i, tmhash.Size, len(auntHash)) } } return nil From 1874ba94b5fe99934393515a73e7103382773ba2 Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 13 May 2020 16:30:33 +0200 Subject: [PATCH 51/86] proto: remove test files ## Description remove test files for proto stubs Closes: #XXX --- crypto/merkle/merkle.pb.go | 184 +------------------------------------ crypto/merkle/merkle.proto | 3 - 2 files changed, 5 insertions(+), 182 deletions(-) diff --git a/crypto/merkle/merkle.pb.go b/crypto/merkle/merkle.pb.go index 80823dd..79379c8 100644 --- a/crypto/merkle/merkle.pb.go +++ b/crypto/merkle/merkle.pb.go @@ -4,7 +4,6 @@ package merkle import ( - bytes "bytes" fmt "fmt" _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" @@ -146,7 +145,7 @@ func init() { func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_9c1c2162d560d38e) } var fileDescriptor_9c1c2162d560d38e = []byte{ - // 230 bytes of a gzipped FileDescriptorProto + // 221 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c, 0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0x12, 0x25, 0xa9, 0x79, 0x29, 0xa9, 0x45, 0xb9, 0x99, 0x79, 0x25, 0x7a, 0x10, 0x65, @@ -157,78 +156,12 @@ var fileDescriptor_9c1c2162d560d38e = []byte{ 0xa4, 0xc0, 0xa8, 0xc1, 0x13, 0x04, 0x62, 0x82, 0x54, 0xa5, 0x24, 0x96, 0x24, 0x4a, 0x30, 0x83, 0x85, 0xc0, 0x6c, 0x25, 0x27, 0x2e, 0x56, 0xb0, 0x21, 0x42, 0x96, 0x5c, 0xcc, 0xf9, 0x05, 0xc5, 0x12, 0x8c, 0x0a, 0xcc, 0x1a, 0xdc, 0x46, 0x8a, 0x7a, 0xb8, 0x5c, 0xa7, 0x07, 0xb5, 0xd2, 0x89, - 0xe5, 0xc4, 0x3d, 0x79, 0x86, 0x20, 0x90, 0x1e, 0x27, 0x97, 0x1f, 0x0f, 0xe5, 0x18, 0x57, 0x3c, - 0x92, 0x63, 0x3c, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0xa3, - 0xf4, 0xd2, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, 0xf5, 0x11, 0xa6, 0x21, 0x33, - 0x51, 0x42, 0x27, 0x89, 0x0d, 0xec, 0x2b, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc8, 0xcc, - 0x2c, 0x91, 0x35, 0x01, 0x00, 0x00, + 0xe5, 0xc4, 0x3d, 0x79, 0x86, 0x20, 0x90, 0x1e, 0x27, 0x9b, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, + 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x31, 0x4a, 0x2f, 0x3d, 0xb3, 0x24, 0xa3, 0x34, 0x49, 0x2f, + 0x39, 0x3f, 0x57, 0x1f, 0x61, 0x0a, 0x32, 0x13, 0x25, 0x54, 0x92, 0xd8, 0xc0, 0xbe, 0x31, 0x06, + 0x04, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x3d, 0x69, 0x56, 0x2d, 0x01, 0x00, 0x00, } -func (this *ProofOp) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*ProofOp) - if !ok { - that2, ok := that.(ProofOp) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.Type != that1.Type { - return false - } - if !bytes.Equal(this.Key, that1.Key) { - return false - } - if !bytes.Equal(this.Data, that1.Data) { - return false - } - if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { - return false - } - return true -} -func (this *Proof) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*Proof) - if !ok { - that2, ok := that.(Proof) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if len(this.Ops) != len(that1.Ops) { - return false - } - for i := range this.Ops { - if !this.Ops[i].Equal(&that1.Ops[i]) { - return false - } - } - if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { - return false - } - return true -} func (m *ProofOp) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -329,113 +262,6 @@ func encodeVarintMerkle(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } -func NewPopulatedProofOp(r randyMerkle, easy bool) *ProofOp { - this := &ProofOp{} - this.Type = string(randStringMerkle(r)) - v1 := r.Intn(100) - this.Key = make([]byte, v1) - for i := 0; i < v1; i++ { - this.Key[i] = byte(r.Intn(256)) - } - v2 := r.Intn(100) - this.Data = make([]byte, v2) - for i := 0; i < v2; i++ { - this.Data[i] = byte(r.Intn(256)) - } - if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedMerkle(r, 4) - } - return this -} - -func NewPopulatedProof(r randyMerkle, easy bool) *Proof { - this := &Proof{} - if r.Intn(5) != 0 { - v3 := r.Intn(5) - this.Ops = make([]ProofOp, v3) - for i := 0; i < v3; i++ { - v4 := NewPopulatedProofOp(r, easy) - this.Ops[i] = *v4 - } - } - if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedMerkle(r, 2) - } - return this -} - -type randyMerkle interface { - Float32() float32 - Float64() float64 - Int63() int64 - Int31() int32 - Uint32() uint32 - Intn(n int) int -} - -func randUTF8RuneMerkle(r randyMerkle) rune { - ru := r.Intn(62) - if ru < 10 { - return rune(ru + 48) - } else if ru < 36 { - return rune(ru + 55) - } - return rune(ru + 61) -} -func randStringMerkle(r randyMerkle) string { - v5 := r.Intn(100) - tmps := make([]rune, v5) - for i := 0; i < v5; i++ { - tmps[i] = randUTF8RuneMerkle(r) - } - return string(tmps) -} -func randUnrecognizedMerkle(r randyMerkle, maxFieldNumber int) (dAtA []byte) { - l := r.Intn(5) - for i := 0; i < l; i++ { - wire := r.Intn(4) - if wire == 3 { - wire = 5 - } - fieldNumber := maxFieldNumber + r.Intn(100) - dAtA = randFieldMerkle(dAtA, r, fieldNumber, wire) - } - return dAtA -} -func randFieldMerkle(dAtA []byte, r randyMerkle, fieldNumber int, wire int) []byte { - key := uint32(fieldNumber)<<3 | uint32(wire) - switch wire { - case 0: - dAtA = encodeVarintPopulateMerkle(dAtA, uint64(key)) - v6 := r.Int63() - if r.Intn(2) == 0 { - v6 *= -1 - } - dAtA = encodeVarintPopulateMerkle(dAtA, uint64(v6)) - case 1: - dAtA = encodeVarintPopulateMerkle(dAtA, uint64(key)) - dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) - case 2: - dAtA = encodeVarintPopulateMerkle(dAtA, uint64(key)) - ll := r.Intn(100) - dAtA = encodeVarintPopulateMerkle(dAtA, uint64(ll)) - for j := 0; j < ll; j++ { - dAtA = append(dAtA, byte(r.Intn(256))) - } - default: - dAtA = encodeVarintPopulateMerkle(dAtA, uint64(key)) - dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) - } - return dAtA -} -func encodeVarintPopulateMerkle(dAtA []byte, v uint64) []byte { - for v >= 1<<7 { - dAtA = append(dAtA, uint8(uint64(v)&0x7f|0x80)) - v >>= 7 - } - dAtA = append(dAtA, uint8(v)) - return dAtA -} func (m *ProofOp) Size() (n int) { if m == nil { return 0 diff --git a/crypto/merkle/merkle.proto b/crypto/merkle/merkle.proto index 159fc58..ccf032d 100644 --- a/crypto/merkle/merkle.proto +++ b/crypto/merkle/merkle.proto @@ -10,9 +10,6 @@ option (gogoproto.marshaler_all) = true; option (gogoproto.unmarshaler_all) = true; option (gogoproto.sizer_all) = true; -option (gogoproto.populate_all) = true; -option (gogoproto.equal_all) = true; - //---------------------------------------- // Message types From 58be168833fc28ba818d9ccac47048f85403b1e4 Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Tue, 2 Jun 2020 17:47:01 +0200 Subject: [PATCH 52/86] Merge conflicts aftermath --- crypto/merkle/simple_map.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 crypto/merkle/simple_map.go diff --git a/crypto/merkle/simple_map.go b/crypto/merkle/simple_map.go deleted file mode 100644 index e69de29..0000000 From c0b74934289d0ee6bc18c10f0556ec29b0c520ea Mon Sep 17 00:00:00 2001 From: Marko Date: Fri, 5 Jun 2020 06:42:04 +0200 Subject: [PATCH 53/86] proto: add more to/from (#4956) ## Description adding in some more to/from methods/functions Closes: #XXX --- crypto/merkle/simple_proof.go | 41 ++++++++++++++++++++++++++----- crypto/merkle/simple_tree.go | 6 ++--- crypto/merkle/simple_tree_test.go | 8 +++--- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index 2f78431..7c262e3 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" + tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" ) const ( @@ -23,8 +24,8 @@ const ( // everything. This also affects the generalized proof system as // well. type SimpleProof struct { - Total int `json:"total"` // Total number of items. - Index int `json:"index"` // Index of item to prove. + Total int64 `json:"total"` // Total number of items. + Index int64 `json:"index"` // Index of item to prove. LeafHash []byte `json:"leaf_hash"` // Hash of item value. Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. } @@ -37,8 +38,8 @@ func SimpleProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*Simp proofs = make([]*SimpleProof, len(items)) for i, trail := range trails { proofs[i] = &SimpleProof{ - Total: len(items), - Index: i, + Total: int64(len(items)), + Index: int64(i), LeafHash: trail.Hash, Aunts: trail.FlattenAunts(), } @@ -115,10 +116,38 @@ func (sp *SimpleProof) ValidateBasic() error { return nil } +func (sp *SimpleProof) ToProto() *tmmerkle.SimpleProof { + if sp == nil { + return nil + } + pb := new(tmmerkle.SimpleProof) + + pb.Total = sp.Total + pb.Index = sp.Index + pb.LeafHash = sp.LeafHash + pb.Aunts = sp.Aunts + + return pb +} + +func SimpleProofFromProto(pb *tmmerkle.SimpleProof) (*SimpleProof, error) { + if pb == nil { + return nil, errors.New("nil proof") + } + sp := new(SimpleProof) + + sp.Total = pb.Total + sp.Index = pb.Index + sp.LeafHash = pb.LeafHash + sp.Aunts = pb.Aunts + + return sp, sp.ValidateBasic() +} + // Use the leafHash and innerHashes to get the root merkle hash. // If the length of the innerHashes slice isn't exactly correct, the result is nil. // Recursive impl. -func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][]byte) []byte { +func computeHashFromAunts(index, total int64, leafHash []byte, innerHashes [][]byte) []byte { if index >= total || index < 0 || total <= 0 { return nil } @@ -192,7 +221,7 @@ func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *Simp trail := &SimpleProofNode{leafHash(items[0]), nil, nil, nil} return []*SimpleProofNode{trail}, trail default: - k := getSplitPoint(len(items)) + k := getSplitPoint(int64(len(items))) lefts, leftRoot := trailsFromByteSlices(items[:k]) rights, rightRoot := trailsFromByteSlices(items[k:]) rootHash := innerHash(leftRoot.Hash, rightRoot.Hash) diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index d2b931e..c9c4784 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -13,7 +13,7 @@ func SimpleHashFromByteSlices(items [][]byte) []byte { case 1: return leafHash(items[0]) default: - k := getSplitPoint(len(items)) + k := getSplitPoint(int64(len(items))) left := SimpleHashFromByteSlices(items[:k]) right := SimpleHashFromByteSlices(items[k:]) return innerHash(left, right) @@ -92,13 +92,13 @@ func SimpleHashFromByteSlicesIterative(input [][]byte) []byte { } // getSplitPoint returns the largest power of 2 less than length -func getSplitPoint(length int) int { +func getSplitPoint(length int64) int64 { if length < 1 { panic("Trying to split a tree with size < 1") } uLength := uint(length) bitlen := bits.Len(uLength) - k := 1 << uint(bitlen-1) + k := int64(1 << uint(bitlen-1)) if k == length { k >>= 1 } diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index 665017a..d6d4345 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -37,9 +37,9 @@ func TestSimpleProof(t *testing.T) { proof := proofs[i] // Check total/index - require.Equal(t, proof.Index, i, "Unmatched indicies: %d vs %d", proof.Index, i) + require.EqualValues(t, proof.Index, i, "Unmatched indicies: %d vs %d", proof.Index, i) - require.Equal(t, proof.Total, total, "Unmatched totals: %d vs %d", proof.Total, total) + require.EqualValues(t, proof.Total, total, "Unmatched totals: %d vs %d", proof.Total, total) // Verify success err := proof.Verify(rootHash, item) @@ -108,7 +108,7 @@ func BenchmarkSimpleHashAlternatives(b *testing.B) { func Test_getSplitPoint(t *testing.T) { tests := []struct { - length int + length int64 want int }{ {1, 0}, @@ -125,6 +125,6 @@ func Test_getSplitPoint(t *testing.T) { } for _, tt := range tests { got := getSplitPoint(tt.length) - require.Equal(t, tt.want, got, "getSplitPoint(%d) = %v, want %v", tt.length, got, tt.want) + require.EqualValues(t, tt.want, got, "getSplitPoint(%d) = %v, want %v", tt.length, got, tt.want) } } From 40efb52f161cded9954b9005b78d462761ffaee6 Mon Sep 17 00:00:00 2001 From: Marko Date: Fri, 5 Jun 2020 14:18:44 +0200 Subject: [PATCH 54/86] proto: change to use gogofaster (#4957) --- crypto/merkle/merkle.pb.go | 41 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/crypto/merkle/merkle.pb.go b/crypto/merkle/merkle.pb.go index 79379c8..071e47b 100644 --- a/crypto/merkle/merkle.pb.go +++ b/crypto/merkle/merkle.pb.go @@ -27,12 +27,9 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package // The data could be arbitrary format, providing nessecary data // for example neighbouring node hash type ProofOp struct { - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` - Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` } func (m *ProofOp) Reset() { *m = ProofOp{} } @@ -91,10 +88,7 @@ func (m *ProofOp) GetData() []byte { // Proof is Merkle proof defined by the list of ProofOps type Proof struct { - Ops []ProofOp `protobuf:"bytes,1,rep,name=ops,proto3" json:"ops"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Ops []ProofOp `protobuf:"bytes,1,rep,name=ops,proto3" json:"ops"` } func (m *Proof) Reset() { *m = Proof{} } @@ -145,7 +139,7 @@ func init() { func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_9c1c2162d560d38e) } var fileDescriptor_9c1c2162d560d38e = []byte{ - // 221 bytes of a gzipped FileDescriptorProto + // 234 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c, 0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0x12, 0x25, 0xa9, 0x79, 0x29, 0xa9, 0x45, 0xb9, 0x99, 0x79, 0x25, 0x7a, 0x10, 0x65, @@ -156,10 +150,11 @@ var fileDescriptor_9c1c2162d560d38e = []byte{ 0xa4, 0xc0, 0xa8, 0xc1, 0x13, 0x04, 0x62, 0x82, 0x54, 0xa5, 0x24, 0x96, 0x24, 0x4a, 0x30, 0x83, 0x85, 0xc0, 0x6c, 0x25, 0x27, 0x2e, 0x56, 0xb0, 0x21, 0x42, 0x96, 0x5c, 0xcc, 0xf9, 0x05, 0xc5, 0x12, 0x8c, 0x0a, 0xcc, 0x1a, 0xdc, 0x46, 0x8a, 0x7a, 0xb8, 0x5c, 0xa7, 0x07, 0xb5, 0xd2, 0x89, - 0xe5, 0xc4, 0x3d, 0x79, 0x86, 0x20, 0x90, 0x1e, 0x27, 0x9b, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, - 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x31, 0x4a, 0x2f, 0x3d, 0xb3, 0x24, 0xa3, 0x34, 0x49, 0x2f, - 0x39, 0x3f, 0x57, 0x1f, 0x61, 0x0a, 0x32, 0x13, 0x25, 0x54, 0x92, 0xd8, 0xc0, 0xbe, 0x31, 0x06, - 0x04, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x3d, 0x69, 0x56, 0x2d, 0x01, 0x00, 0x00, + 0xe5, 0xc4, 0x3d, 0x79, 0x86, 0x20, 0x90, 0x1e, 0x27, 0x8f, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, + 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, + 0x3c, 0x96, 0x63, 0x88, 0xd2, 0x4b, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, + 0x47, 0x98, 0x88, 0xcc, 0x44, 0x09, 0xa1, 0x24, 0x36, 0xb0, 0xcf, 0x8c, 0x01, 0x01, 0x00, 0x00, + 0xff, 0xff, 0x39, 0xc0, 0xa3, 0x13, 0x39, 0x01, 0x00, 0x00, } func (m *ProofOp) Marshal() (dAtA []byte, err error) { @@ -182,10 +177,6 @@ func (m *ProofOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Data) > 0 { i -= len(m.Data) copy(dAtA[i:], m.Data) @@ -230,10 +221,6 @@ func (m *Proof) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } if len(m.Ops) > 0 { for iNdEx := len(m.Ops) - 1; iNdEx >= 0; iNdEx-- { { @@ -280,9 +267,6 @@ func (m *ProofOp) Size() (n int) { if l > 0 { n += 1 + l + sovMerkle(uint64(l)) } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -298,9 +282,6 @@ func (m *Proof) Size() (n int) { n += 1 + l + sovMerkle(uint64(l)) } } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } return n } @@ -454,7 +435,6 @@ func (m *ProofOp) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } @@ -542,7 +522,6 @@ func (m *Proof) Unmarshal(dAtA []byte) error { if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } From 32fbc45654a0c9f093b43ef872d2352d1cdaaab0 Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 10 Jun 2020 14:08:47 +0200 Subject: [PATCH 55/86] consensus: proto migration (#4984) ## Description migrate consensus to protobuf Closes: #XXX --- crypto/merkle/simple_proof_test.go | 35 ++++++++++++++++++++++++++++++ crypto/merkle/simple_tree_test.go | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/crypto/merkle/simple_proof_test.go b/crypto/merkle/simple_proof_test.go index 68e6912..2dd8b7e 100644 --- a/crypto/merkle/simple_proof_test.go +++ b/crypto/merkle/simple_proof_test.go @@ -4,6 +4,9 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + tmrand "github.com/tendermint/tendermint/libs/rand" ) func TestSimpleProofValidateBasic(t *testing.T) { @@ -39,3 +42,35 @@ func TestSimpleProofValidateBasic(t *testing.T) { }) } } + +func TestSimpleProofProtoBuf(t *testing.T) { + testCases := []struct { + testName string + ps1 *SimpleProof + expPass bool + }{ + {"failure empty", &SimpleProof{}, false}, + {"failure nil", nil, false}, + {"success", + &SimpleProof{ + Total: 1, + Index: 1, + LeafHash: tmrand.Bytes(32), + Aunts: [][]byte{tmrand.Bytes(32), tmrand.Bytes(32), tmrand.Bytes(32)}, + }, true}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.testName, func(t *testing.T) { + proto := tc.ps1.ToProto() + p, err := SimpleProofFromProto(proto) + if tc.expPass { + require.NoError(t, err) + require.Equal(t, tc.ps1, p, tc.testName) + } else { + require.Error(t, err, tc.testName) + } + }) + } +} diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index d6d4345..01d6058 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -109,7 +109,7 @@ func BenchmarkSimpleHashAlternatives(b *testing.B) { func Test_getSplitPoint(t *testing.T) { tests := []struct { length int64 - want int + want int64 }{ {1, 0}, {2, 1}, From 73119ba16ee7924603a639734d86e4e25ea5565e Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 10 Jun 2020 16:57:38 +0200 Subject: [PATCH 56/86] crypto/merkle: remove simple prefix (#4989) ## Description This PR removes simple prefix from all types in the crypto/merkle directory. The two proto types `Proof` & `ProofOp` have been moved to the `proto/crypto/merkle` directory. proto messge `Proof` was renamed to `ProofOps` and `SimpleProof` message to `Proof`. Closes: #2755 --- crypto/merkle/merkle.pb.go | 617 ------------------ crypto/merkle/merkle.proto | 28 - crypto/merkle/proof.go | 289 +++++--- crypto/merkle/proof_key_path.go | 2 +- crypto/merkle/proof_op.go | 139 ++++ crypto/merkle/proof_test.go | 43 +- .../{proof_simple_value.go => proof_value.go} | 43 +- crypto/merkle/result.go | 52 -- crypto/merkle/simple_proof.go | 235 ------- crypto/merkle/simple_proof_test.go | 76 --- crypto/merkle/{simple_tree.go => tree.go} | 26 +- .../{simple_tree_test.go => tree_test.go} | 18 +- 12 files changed, 416 insertions(+), 1152 deletions(-) delete mode 100644 crypto/merkle/merkle.pb.go delete mode 100644 crypto/merkle/merkle.proto create mode 100644 crypto/merkle/proof_op.go rename crypto/merkle/{proof_simple_value.go => proof_value.go} (58%) delete mode 100644 crypto/merkle/result.go delete mode 100644 crypto/merkle/simple_proof.go delete mode 100644 crypto/merkle/simple_proof_test.go rename crypto/merkle/{simple_tree.go => tree.go} (73%) rename crypto/merkle/{simple_tree_test.go => tree_test.go} (86%) diff --git a/crypto/merkle/merkle.pb.go b/crypto/merkle/merkle.pb.go deleted file mode 100644 index 071e47b..0000000 --- a/crypto/merkle/merkle.pb.go +++ /dev/null @@ -1,617 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: crypto/merkle/merkle.proto - -package merkle - -import ( - fmt "fmt" - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -// ProofOp defines an operation used for calculating Merkle root -// The data could be arbitrary format, providing nessecary data -// for example neighbouring node hash -type ProofOp struct { - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` - Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` -} - -func (m *ProofOp) Reset() { *m = ProofOp{} } -func (m *ProofOp) String() string { return proto.CompactTextString(m) } -func (*ProofOp) ProtoMessage() {} -func (*ProofOp) Descriptor() ([]byte, []int) { - return fileDescriptor_9c1c2162d560d38e, []int{0} -} -func (m *ProofOp) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ProofOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ProofOp.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *ProofOp) XXX_Merge(src proto.Message) { - xxx_messageInfo_ProofOp.Merge(m, src) -} -func (m *ProofOp) XXX_Size() int { - return m.Size() -} -func (m *ProofOp) XXX_DiscardUnknown() { - xxx_messageInfo_ProofOp.DiscardUnknown(m) -} - -var xxx_messageInfo_ProofOp proto.InternalMessageInfo - -func (m *ProofOp) GetType() string { - if m != nil { - return m.Type - } - return "" -} - -func (m *ProofOp) GetKey() []byte { - if m != nil { - return m.Key - } - return nil -} - -func (m *ProofOp) GetData() []byte { - if m != nil { - return m.Data - } - return nil -} - -// Proof is Merkle proof defined by the list of ProofOps -type Proof struct { - Ops []ProofOp `protobuf:"bytes,1,rep,name=ops,proto3" json:"ops"` -} - -func (m *Proof) Reset() { *m = Proof{} } -func (m *Proof) String() string { return proto.CompactTextString(m) } -func (*Proof) ProtoMessage() {} -func (*Proof) Descriptor() ([]byte, []int) { - return fileDescriptor_9c1c2162d560d38e, []int{1} -} -func (m *Proof) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Proof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Proof.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Proof) XXX_Merge(src proto.Message) { - xxx_messageInfo_Proof.Merge(m, src) -} -func (m *Proof) XXX_Size() int { - return m.Size() -} -func (m *Proof) XXX_DiscardUnknown() { - xxx_messageInfo_Proof.DiscardUnknown(m) -} - -var xxx_messageInfo_Proof proto.InternalMessageInfo - -func (m *Proof) GetOps() []ProofOp { - if m != nil { - return m.Ops - } - return nil -} - -func init() { - proto.RegisterType((*ProofOp)(nil), "tendermint.crypto.merkle.ProofOp") - proto.RegisterType((*Proof)(nil), "tendermint.crypto.merkle.Proof") -} - -func init() { proto.RegisterFile("crypto/merkle/merkle.proto", fileDescriptor_9c1c2162d560d38e) } - -var fileDescriptor_9c1c2162d560d38e = []byte{ - // 234 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0x2e, 0xaa, 0x2c, - 0x28, 0xc9, 0xd7, 0xcf, 0x4d, 0x2d, 0xca, 0xce, 0x49, 0x85, 0x52, 0x7a, 0x05, 0x45, 0xf9, 0x25, - 0xf9, 0x42, 0x12, 0x25, 0xa9, 0x79, 0x29, 0xa9, 0x45, 0xb9, 0x99, 0x79, 0x25, 0x7a, 0x10, 0x65, - 0x7a, 0x10, 0x79, 0x29, 0xb5, 0x92, 0x8c, 0xcc, 0xa2, 0x94, 0xf8, 0x82, 0xc4, 0xa2, 0x92, 0x4a, - 0x7d, 0xb0, 0x62, 0xfd, 0xf4, 0xfc, 0xf4, 0x7c, 0x04, 0x0b, 0x62, 0x82, 0x92, 0x33, 0x17, 0x7b, - 0x40, 0x51, 0x7e, 0x7e, 0x9a, 0x7f, 0x81, 0x90, 0x10, 0x17, 0x4b, 0x49, 0x65, 0x41, 0xaa, 0x04, - 0xa3, 0x02, 0xa3, 0x06, 0x67, 0x10, 0x98, 0x2d, 0x24, 0xc0, 0xc5, 0x9c, 0x9d, 0x5a, 0x29, 0xc1, - 0xa4, 0xc0, 0xa8, 0xc1, 0x13, 0x04, 0x62, 0x82, 0x54, 0xa5, 0x24, 0x96, 0x24, 0x4a, 0x30, 0x83, - 0x85, 0xc0, 0x6c, 0x25, 0x27, 0x2e, 0x56, 0xb0, 0x21, 0x42, 0x96, 0x5c, 0xcc, 0xf9, 0x05, 0xc5, - 0x12, 0x8c, 0x0a, 0xcc, 0x1a, 0xdc, 0x46, 0x8a, 0x7a, 0xb8, 0x5c, 0xa7, 0x07, 0xb5, 0xd2, 0x89, - 0xe5, 0xc4, 0x3d, 0x79, 0x86, 0x20, 0x90, 0x1e, 0x27, 0x8f, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, - 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, - 0x3c, 0x96, 0x63, 0x88, 0xd2, 0x4b, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, - 0x47, 0x98, 0x88, 0xcc, 0x44, 0x09, 0xa1, 0x24, 0x36, 0xb0, 0xcf, 0x8c, 0x01, 0x01, 0x00, 0x00, - 0xff, 0xff, 0x39, 0xc0, 0xa3, 0x13, 0x39, 0x01, 0x00, 0x00, -} - -func (m *ProofOp) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ProofOp) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ProofOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Data) > 0 { - i -= len(m.Data) - copy(dAtA[i:], m.Data) - i = encodeVarintMerkle(dAtA, i, uint64(len(m.Data))) - i-- - dAtA[i] = 0x1a - } - if len(m.Key) > 0 { - i -= len(m.Key) - copy(dAtA[i:], m.Key) - i = encodeVarintMerkle(dAtA, i, uint64(len(m.Key))) - i-- - dAtA[i] = 0x12 - } - if len(m.Type) > 0 { - i -= len(m.Type) - copy(dAtA[i:], m.Type) - i = encodeVarintMerkle(dAtA, i, uint64(len(m.Type))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *Proof) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Proof) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Proof) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Ops) > 0 { - for iNdEx := len(m.Ops) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Ops[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintMerkle(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - } - return len(dAtA) - i, nil -} - -func encodeVarintMerkle(dAtA []byte, offset int, v uint64) int { - offset -= sovMerkle(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *ProofOp) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Type) - if l > 0 { - n += 1 + l + sovMerkle(uint64(l)) - } - l = len(m.Key) - if l > 0 { - n += 1 + l + sovMerkle(uint64(l)) - } - l = len(m.Data) - if l > 0 { - n += 1 + l + sovMerkle(uint64(l)) - } - return n -} - -func (m *Proof) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if len(m.Ops) > 0 { - for _, e := range m.Ops { - l = e.Size() - n += 1 + l + sovMerkle(uint64(l)) - } - } - return n -} - -func sovMerkle(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozMerkle(x uint64) (n int) { - return sovMerkle(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *ProofOp) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMerkle - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ProofOp: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ProofOp: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMerkle - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthMerkle - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthMerkle - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Type = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMerkle - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthMerkle - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthMerkle - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) - if m.Key == nil { - m.Key = []byte{} - } - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMerkle - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthMerkle - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthMerkle - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) - if m.Data == nil { - m.Data = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipMerkle(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthMerkle - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthMerkle - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Proof) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMerkle - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Proof: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Proof: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMerkle - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthMerkle - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthMerkle - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Ops = append(m.Ops, ProofOp{}) - if err := m.Ops[len(m.Ops)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipMerkle(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthMerkle - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthMerkle - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipMerkle(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - depth := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowMerkle - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowMerkle - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - case 1: - iNdEx += 8 - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowMerkle - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthMerkle - } - iNdEx += length - case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupMerkle - } - depth-- - case 5: - iNdEx += 4 - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - if iNdEx < 0 { - return 0, ErrInvalidLengthMerkle - } - if depth == 0 { - return iNdEx, nil - } - } - return 0, io.ErrUnexpectedEOF -} - -var ( - ErrInvalidLengthMerkle = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowMerkle = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupMerkle = fmt.Errorf("proto: unexpected end of group") -) diff --git a/crypto/merkle/merkle.proto b/crypto/merkle/merkle.proto deleted file mode 100644 index ccf032d..0000000 --- a/crypto/merkle/merkle.proto +++ /dev/null @@ -1,28 +0,0 @@ -syntax = "proto3"; -package tendermint.crypto.merkle; -option go_package = "github.com/tendermint/tendermint/crypto/merkle"; - -// For more information on gogo.proto, see: -// https://github.com/gogo/protobuf/blob/master/extensions.md -import "third_party/proto/gogoproto/gogo.proto"; - -option (gogoproto.marshaler_all) = true; -option (gogoproto.unmarshaler_all) = true; -option (gogoproto.sizer_all) = true; - -//---------------------------------------- -// Message types - -// ProofOp defines an operation used for calculating Merkle root -// The data could be arbitrary format, providing nessecary data -// for example neighbouring node hash -message ProofOp { - string type = 1; - bytes key = 2; - bytes data = 3; -} - -// Proof is Merkle proof defined by the list of ProofOps -message Proof { - repeated ProofOp ops = 1 [(gogoproto.nullable) = false]; -} diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 84c9bbc..ae463e8 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -4,135 +4,232 @@ import ( "bytes" "errors" "fmt" -) - -//---------------------------------------- -// ProofOp gets converted to an instance of ProofOperator: - -// ProofOperator is a layer for calculating intermediate Merkle roots -// when a series of Merkle trees are chained together. -// Run() takes leaf values from a tree and returns the Merkle -// root for the corresponding tree. It takes and returns a list of bytes -// to allow multiple leaves to be part of a single proof, for instance in a range proof. -// ProofOp() encodes the ProofOperator in a generic way so it can later be -// decoded with OpDecoder. -type ProofOperator interface { - Run([][]byte) ([][]byte, error) - GetKey() []byte - ProofOp() ProofOp -} -//---------------------------------------- -// Operations on a list of ProofOperators + "github.com/tendermint/tendermint/crypto/tmhash" + tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" +) -// ProofOperators is a slice of ProofOperator(s). -// Each operator will be applied to the input value sequentially -// and the last Merkle root will be verified with already known data -type ProofOperators []ProofOperator +const ( + // MaxAunts is the maximum number of aunts that can be included in a Proof. + // This corresponds to a tree of size 2^100, which should be sufficient for all conceivable purposes. + // This maximum helps prevent Denial-of-Service attacks by limitting the size of the proofs. + MaxAunts = 100 +) -func (poz ProofOperators) VerifyValue(root []byte, keypath string, value []byte) (err error) { - return poz.Verify(root, keypath, [][]byte{value}) +// Proof represents a Merkle proof. +// NOTE: The convention for proofs is to include leaf hashes but to +// exclude the root hash. +// This convention is implemented across IAVL range proofs as well. +// Keep this consistent unless there's a very good reason to change +// everything. This also affects the generalized proof system as +// well. +type Proof struct { + Total int64 `json:"total"` // Total number of items. + Index int64 `json:"index"` // Index of item to prove. + LeafHash []byte `json:"leaf_hash"` // Hash of item value. + Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. } -func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (err error) { - keys, err := KeyPathToKeys(keypath) - if err != nil { - return +// ProofsFromByteSlices computes inclusion proof for given items. +// proofs[0] is the proof for items[0]. +func ProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*Proof) { + trails, rootSPN := trailsFromByteSlices(items) + rootHash = rootSPN.Hash + proofs = make([]*Proof, len(items)) + for i, trail := range trails { + proofs[i] = &Proof{ + Total: int64(len(items)), + Index: int64(i), + LeafHash: trail.Hash, + Aunts: trail.FlattenAunts(), + } } + return +} - for i, op := range poz { - key := op.GetKey() - if len(key) != 0 { - if len(keys) == 0 { - return fmt.Errorf("key path has insufficient # of parts: expected no more keys but got %+v", string(key)) - } - lastKey := keys[len(keys)-1] - if !bytes.Equal(lastKey, key) { - return fmt.Errorf("key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) - } - keys = keys[:len(keys)-1] - } - args, err = op.Run(args) - if err != nil { - return - } +// Verify that the Proof proves the root hash. +// Check sp.Index/sp.Total manually if needed +func (sp *Proof) Verify(rootHash []byte, leaf []byte) error { + leafHash := leafHash(leaf) + if sp.Total < 0 { + return errors.New("proof total must be positive") } - if !bytes.Equal(root, args[0]) { - return fmt.Errorf("calculated root hash is invalid: expected %+v but got %+v", root, args[0]) + if sp.Index < 0 { + return errors.New("proof index cannot be negative") } - if len(keys) != 0 { - return errors.New("keypath not consumed all") + if !bytes.Equal(sp.LeafHash, leafHash) { + return fmt.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) + } + computedHash := sp.ComputeRootHash() + if !bytes.Equal(computedHash, rootHash) { + return fmt.Errorf("invalid root hash: wanted %X got %X", rootHash, computedHash) } return nil } -//---------------------------------------- -// ProofRuntime - main entrypoint +// Compute the root hash given a leaf hash. Does not verify the result. +func (sp *Proof) ComputeRootHash() []byte { + return computeHashFromAunts( + sp.Index, + sp.Total, + sp.LeafHash, + sp.Aunts, + ) +} -type OpDecoder func(ProofOp) (ProofOperator, error) +// String implements the stringer interface for Proof. +// It is a wrapper around StringIndented. +func (sp *Proof) String() string { + return sp.StringIndented("") +} -type ProofRuntime struct { - decoders map[string]OpDecoder +// StringIndented generates a canonical string representation of a Proof. +func (sp *Proof) StringIndented(indent string) string { + return fmt.Sprintf(`Proof{ +%s Aunts: %X +%s}`, + indent, sp.Aunts, + indent) } -func NewProofRuntime() *ProofRuntime { - return &ProofRuntime{ - decoders: make(map[string]OpDecoder), +// ValidateBasic performs basic validation. +// NOTE: it expects the LeafHash and the elements of Aunts to be of size tmhash.Size, +// and it expects at most MaxAunts elements in Aunts. +func (sp *Proof) ValidateBasic() error { + if sp.Total < 0 { + return errors.New("negative Total") + } + if sp.Index < 0 { + return errors.New("negative Index") + } + if len(sp.LeafHash) != tmhash.Size { + return fmt.Errorf("expected LeafHash size to be %d, got %d", tmhash.Size, len(sp.LeafHash)) + } + if len(sp.Aunts) > MaxAunts { + return fmt.Errorf("expected no more than %d aunts, got %d", MaxAunts, len(sp.Aunts)) } + for i, auntHash := range sp.Aunts { + if len(auntHash) != tmhash.Size { + return fmt.Errorf("expected Aunts#%d size to be %d, got %d", i, tmhash.Size, len(auntHash)) + } + } + return nil } -func (prt *ProofRuntime) RegisterOpDecoder(typ string, dec OpDecoder) { - _, ok := prt.decoders[typ] - if ok { - panic("already registered for type " + typ) +func (sp *Proof) ToProto() *tmmerkle.Proof { + if sp == nil { + return nil } - prt.decoders[typ] = dec + pb := new(tmmerkle.Proof) + + pb.Total = sp.Total + pb.Index = sp.Index + pb.LeafHash = sp.LeafHash + pb.Aunts = sp.Aunts + + return pb } -func (prt *ProofRuntime) Decode(pop ProofOp) (ProofOperator, error) { - decoder := prt.decoders[pop.Type] - if decoder == nil { - return nil, fmt.Errorf("unrecognized proof type %v", pop.Type) +func ProofFromProto(pb *tmmerkle.Proof) (*Proof, error) { + if pb == nil { + return nil, errors.New("nil proof") } - return decoder(pop) + sp := new(Proof) + + sp.Total = pb.Total + sp.Index = pb.Index + sp.LeafHash = pb.LeafHash + sp.Aunts = pb.Aunts + + return sp, sp.ValidateBasic() } -func (prt *ProofRuntime) DecodeProof(proof *Proof) (ProofOperators, error) { - poz := make(ProofOperators, 0, len(proof.Ops)) - for _, pop := range proof.Ops { - operator, err := prt.Decode(pop) - if err != nil { - return nil, fmt.Errorf("decoding a proof operator: %w", err) +// Use the leafHash and innerHashes to get the root merkle hash. +// If the length of the innerHashes slice isn't exactly correct, the result is nil. +// Recursive impl. +func computeHashFromAunts(index, total int64, leafHash []byte, innerHashes [][]byte) []byte { + if index >= total || index < 0 || total <= 0 { + return nil + } + switch total { + case 0: + panic("Cannot call computeHashFromAunts() with 0 total") + case 1: + if len(innerHashes) != 0 { + return nil + } + return leafHash + default: + if len(innerHashes) == 0 { + return nil } - poz = append(poz, operator) + numLeft := getSplitPoint(total) + if index < numLeft { + leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if leftHash == nil { + return nil + } + return innerHash(leftHash, innerHashes[len(innerHashes)-1]) + } + rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if rightHash == nil { + return nil + } + return innerHash(innerHashes[len(innerHashes)-1], rightHash) } - return poz, nil -} - -func (prt *ProofRuntime) VerifyValue(proof *Proof, root []byte, keypath string, value []byte) (err error) { - return prt.Verify(proof, root, keypath, [][]byte{value}) } -// TODO In the long run we'll need a method of classifcation of ops, -// whether existence or absence or perhaps a third? -func (prt *ProofRuntime) VerifyAbsence(proof *Proof, root []byte, keypath string) (err error) { - return prt.Verify(proof, root, keypath, nil) +// ProofNode is a helper structure to construct merkle proof. +// The node and the tree is thrown away afterwards. +// Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. +// node.Parent.Hash = hash(node.Hash, node.Right.Hash) or +// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. +type ProofNode struct { + Hash []byte + Parent *ProofNode + Left *ProofNode // Left sibling (only one of Left,Right is set) + Right *ProofNode // Right sibling (only one of Left,Right is set) } -func (prt *ProofRuntime) Verify(proof *Proof, root []byte, keypath string, args [][]byte) (err error) { - poz, err := prt.DecodeProof(proof) - if err != nil { - return fmt.Errorf("decoding proof: %w", err) +// FlattenAunts will return the inner hashes for the item corresponding to the leaf, +// starting from a leaf ProofNode. +func (spn *ProofNode) FlattenAunts() [][]byte { + // Nonrecursive impl. + innerHashes := [][]byte{} + for spn != nil { + switch { + case spn.Left != nil: + innerHashes = append(innerHashes, spn.Left.Hash) + case spn.Right != nil: + innerHashes = append(innerHashes, spn.Right.Hash) + default: + break + } + spn = spn.Parent } - return poz.Verify(root, keypath, args) + return innerHashes } -// DefaultProofRuntime only knows about Simple value -// proofs. -// To use e.g. IAVL proofs, register op-decoders as -// defined in the IAVL package. -func DefaultProofRuntime() (prt *ProofRuntime) { - prt = NewProofRuntime() - prt.RegisterOpDecoder(ProofOpSimpleValue, SimpleValueOpDecoder) - return +// trails[0].Hash is the leaf hash for items[0]. +// trails[i].Parent.Parent....Parent == root for all i. +func trailsFromByteSlices(items [][]byte) (trails []*ProofNode, root *ProofNode) { + // Recursive impl. + switch len(items) { + case 0: + return nil, nil + case 1: + trail := &ProofNode{leafHash(items[0]), nil, nil, nil} + return []*ProofNode{trail}, trail + default: + k := getSplitPoint(int64(len(items))) + lefts, leftRoot := trailsFromByteSlices(items[:k]) + rights, rightRoot := trailsFromByteSlices(items[k:]) + rootHash := innerHash(leftRoot.Hash, rightRoot.Hash) + root := &ProofNode{rootHash, nil, nil, nil} + leftRoot.Parent = root + leftRoot.Right = rightRoot + rightRoot.Parent = root + rightRoot.Left = leftRoot + return append(lefts, rights...), root + } } diff --git a/crypto/merkle/proof_key_path.go b/crypto/merkle/proof_key_path.go index beafc4f..ca8b5f0 100644 --- a/crypto/merkle/proof_key_path.go +++ b/crypto/merkle/proof_key_path.go @@ -17,7 +17,7 @@ import ( /32:) For example, for a Cosmos-SDK application where the first two proof layers - are SimpleValueOps, and the third proof layer is an IAVLValueOp, the keys + are ValueOps, and the third proof layer is an IAVLValueOp, the keys might look like: 0: []byte("App") diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go new file mode 100644 index 0000000..8994a29 --- /dev/null +++ b/crypto/merkle/proof_op.go @@ -0,0 +1,139 @@ +package merkle + +import ( + "bytes" + "errors" + "fmt" + + tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" +) + +//---------------------------------------- +// ProofOp gets converted to an instance of ProofOperator: + +// ProofOperator is a layer for calculating intermediate Merkle roots +// when a series of Merkle trees are chained together. +// Run() takes leaf values from a tree and returns the Merkle +// root for the corresponding tree. It takes and returns a list of bytes +// to allow multiple leaves to be part of a single proof, for instance in a range proof. +// ProofOp() encodes the ProofOperator in a generic way so it can later be +// decoded with OpDecoder. +type ProofOperator interface { + Run([][]byte) ([][]byte, error) + GetKey() []byte + ProofOp() tmmerkle.ProofOp +} + +//---------------------------------------- +// Operations on a list of ProofOperators + +// ProofOperators is a slice of ProofOperator(s). +// Each operator will be applied to the input value sequentially +// and the last Merkle root will be verified with already known data +type ProofOperators []ProofOperator + +func (poz ProofOperators) VerifyValue(root []byte, keypath string, value []byte) (err error) { + return poz.Verify(root, keypath, [][]byte{value}) +} + +func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (err error) { + keys, err := KeyPathToKeys(keypath) + if err != nil { + return + } + + for i, op := range poz { + key := op.GetKey() + if len(key) != 0 { + if len(keys) == 0 { + return fmt.Errorf("key path has insufficient # of parts: expected no more keys but got %+v", string(key)) + } + lastKey := keys[len(keys)-1] + if !bytes.Equal(lastKey, key) { + return fmt.Errorf("key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) + } + keys = keys[:len(keys)-1] + } + args, err = op.Run(args) + if err != nil { + return + } + } + if !bytes.Equal(root, args[0]) { + return fmt.Errorf("calculated root hash is invalid: expected %+v but got %+v", root, args[0]) + } + if len(keys) != 0 { + return errors.New("keypath not consumed all") + } + return nil +} + +//---------------------------------------- +// ProofRuntime - main entrypoint + +type OpDecoder func(tmmerkle.ProofOp) (ProofOperator, error) + +type ProofRuntime struct { + decoders map[string]OpDecoder +} + +func NewProofRuntime() *ProofRuntime { + return &ProofRuntime{ + decoders: make(map[string]OpDecoder), + } +} + +func (prt *ProofRuntime) RegisterOpDecoder(typ string, dec OpDecoder) { + _, ok := prt.decoders[typ] + if ok { + panic("already registered for type " + typ) + } + prt.decoders[typ] = dec +} + +func (prt *ProofRuntime) Decode(pop tmmerkle.ProofOp) (ProofOperator, error) { + decoder := prt.decoders[pop.Type] + if decoder == nil { + return nil, fmt.Errorf("unrecognized proof type %v", pop.Type) + } + return decoder(pop) +} + +func (prt *ProofRuntime) DecodeProof(proof *tmmerkle.ProofOps) (ProofOperators, error) { + poz := make(ProofOperators, 0, len(proof.Ops)) + for _, pop := range proof.Ops { + operator, err := prt.Decode(pop) + if err != nil { + return nil, fmt.Errorf("decoding a proof operator: %w", err) + } + poz = append(poz, operator) + } + return poz, nil +} + +func (prt *ProofRuntime) VerifyValue(proof *tmmerkle.ProofOps, root []byte, keypath string, value []byte) (err error) { + return prt.Verify(proof, root, keypath, [][]byte{value}) +} + +// TODO In the long run we'll need a method of classifcation of ops, +// whether existence or absence or perhaps a third? +func (prt *ProofRuntime) VerifyAbsence(proof *tmmerkle.ProofOps, root []byte, keypath string) (err error) { + return prt.Verify(proof, root, keypath, nil) +} + +func (prt *ProofRuntime) Verify(proof *tmmerkle.ProofOps, root []byte, keypath string, args [][]byte) (err error) { + poz, err := prt.DecodeProof(proof) + if err != nil { + return fmt.Errorf("decoding proof: %w", err) + } + return poz.Verify(root, keypath, args) +} + +// DefaultProofRuntime only knows about value proofs. +// To use e.g. IAVL proofs, register op-decoders as +// defined in the IAVL package. +func DefaultProofRuntime() (prt *ProofRuntime) { + prt = NewProofRuntime() + prt.RegisterOpDecoder(ProofOpValue, ValueOpDecoder) + return +} diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index ca9a982..398cb3e 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" amino "github.com/tendermint/go-amino" + tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" ) const ProofOpDomino = "test:domino" @@ -28,21 +29,21 @@ func NewDominoOp(key, input, output string) DominoOp { } //nolint:unused -func DominoOpDecoder(pop ProofOp) (ProofOperator, error) { +func DominoOpDecoder(pop tmmerkle.ProofOp) (ProofOperator, error) { if pop.Type != ProofOpDomino { panic("unexpected proof op type") } var op DominoOp // a bit strange as we'll discard this, but it works. err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) if err != nil { - return nil, fmt.Errorf("decoding ProofOp.Data into SimpleValueOp: %w", err) + return nil, fmt.Errorf("decoding ProofOp.Data into ValueOp: %w", err) } return NewDominoOp(string(pop.Key), op.Input, op.Output), nil } -func (dop DominoOp) ProofOp() ProofOp { +func (dop DominoOp) ProofOp() tmmerkle.ProofOp { bz := amino.MustMarshalBinaryLengthPrefixed(dop) - return ProofOp{ + return tmmerkle.ProofOp{ Type: ProofOpDomino, Key: []byte(dop.key), Data: bz, @@ -140,3 +141,37 @@ func TestProofOperators(t *testing.T) { func bz(s string) []byte { return []byte(s) } + +func TestProofValidateBasic(t *testing.T) { + testCases := []struct { + testName string + malleateProof func(*Proof) + errStr string + }{ + {"Good", func(sp *Proof) {}, ""}, + {"Negative Total", func(sp *Proof) { sp.Total = -1 }, "negative Total"}, + {"Negative Index", func(sp *Proof) { sp.Index = -1 }, "negative Index"}, + {"Invalid LeafHash", func(sp *Proof) { sp.LeafHash = make([]byte, 10) }, + "expected LeafHash size to be 32, got 10"}, + {"Too many Aunts", func(sp *Proof) { sp.Aunts = make([][]byte, MaxAunts+1) }, + "expected no more than 100 aunts, got 101"}, + {"Invalid Aunt", func(sp *Proof) { sp.Aunts[0] = make([]byte, 10) }, + "expected Aunts#0 size to be 32, got 10"}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.testName, func(t *testing.T) { + _, proofs := ProofsFromByteSlices([][]byte{ + []byte("apple"), + []byte("watermelon"), + []byte("kiwi"), + }) + tc.malleateProof(proofs[0]) + err := proofs[0].ValidateBasic() + if tc.errStr != "" { + assert.Contains(t, err.Error(), tc.errStr) + } + }) + } +} diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_value.go similarity index 58% rename from crypto/merkle/proof_simple_value.go rename to crypto/merkle/proof_value.go index d1a0426..b9be68c 100644 --- a/crypto/merkle/proof_simple_value.go +++ b/crypto/merkle/proof_value.go @@ -5,63 +5,64 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" + tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" ) -const ProofOpSimpleValue = "simple:v" +const ProofOpValue = "simple:v" -// SimpleValueOp takes a key and a single value as argument and +// ValueOp takes a key and a single value as argument and // produces the root hash. The corresponding tree structure is // the SimpleMap tree. SimpleMap takes a Hasher, and currently -// Tendermint uses aminoHasher. SimpleValueOp should support +// Tendermint uses aminoHasher. ValueOp should support // the hash function as used in aminoHasher. TODO support // additional hash functions here as options/args to this // operator. // // If the produced root hash matches the expected hash, the // proof is good. -type SimpleValueOp struct { +type ValueOp struct { // Encoded in ProofOp.Key. key []byte // To encode in ProofOp.Data - Proof *SimpleProof `json:"simple_proof"` + Proof *Proof `json:"proof"` } -var _ ProofOperator = SimpleValueOp{} +var _ ProofOperator = ValueOp{} -func NewSimpleValueOp(key []byte, proof *SimpleProof) SimpleValueOp { - return SimpleValueOp{ +func NewValueOp(key []byte, proof *Proof) ValueOp { + return ValueOp{ key: key, Proof: proof, } } -func SimpleValueOpDecoder(pop ProofOp) (ProofOperator, error) { - if pop.Type != ProofOpSimpleValue { - return nil, fmt.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpSimpleValue) +func ValueOpDecoder(pop tmmerkle.ProofOp) (ProofOperator, error) { + if pop.Type != ProofOpValue { + return nil, fmt.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpValue) } - var op SimpleValueOp // a bit strange as we'll discard this, but it works. + var op ValueOp // a bit strange as we'll discard this, but it works. err := cdc.UnmarshalBinaryLengthPrefixed(pop.Data, &op) if err != nil { - return nil, fmt.Errorf("decoding ProofOp.Data into SimpleValueOp: %w", err) + return nil, fmt.Errorf("decoding ProofOp.Data into ValueOp: %w", err) } - return NewSimpleValueOp(pop.Key, op.Proof), nil + return NewValueOp(pop.Key, op.Proof), nil } -func (op SimpleValueOp) ProofOp() ProofOp { +func (op ValueOp) ProofOp() tmmerkle.ProofOp { bz := cdc.MustMarshalBinaryLengthPrefixed(op) - return ProofOp{ - Type: ProofOpSimpleValue, + return tmmerkle.ProofOp{ + Type: ProofOpValue, Key: op.key, Data: bz, } } -func (op SimpleValueOp) String() string { - return fmt.Sprintf("SimpleValueOp{%v}", op.GetKey()) +func (op ValueOp) String() string { + return fmt.Sprintf("ValueOp{%v}", op.GetKey()) } -func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { +func (op ValueOp) Run(args [][]byte) ([][]byte, error) { if len(args) != 1 { return nil, fmt.Errorf("expected 1 arg, got %v", len(args)) } @@ -85,6 +86,6 @@ func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { }, nil } -func (op SimpleValueOp) GetKey() []byte { +func (op ValueOp) GetKey() []byte { return op.key } diff --git a/crypto/merkle/result.go b/crypto/merkle/result.go deleted file mode 100644 index a17e38e..0000000 --- a/crypto/merkle/result.go +++ /dev/null @@ -1,52 +0,0 @@ -package merkle - -import ( - "bytes" - "encoding/json" - - "github.com/gogo/protobuf/jsonpb" -) - -//--------------------------------------------------------------------------- -// override JSON marshalling so we emit defaults (ie. disable omitempty) - -var ( - jsonpbMarshaller = jsonpb.Marshaler{ - EnumsAsInts: true, - EmitDefaults: true, - } - jsonpbUnmarshaller = jsonpb.Unmarshaler{} -) - -func (r *ProofOp) MarshalJSON() ([]byte, error) { - s, err := jsonpbMarshaller.MarshalToString(r) - return []byte(s), err -} - -func (r *ProofOp) UnmarshalJSON(b []byte) error { - reader := bytes.NewBuffer(b) - return jsonpbUnmarshaller.Unmarshal(reader, r) -} - -func (r *Proof) MarshalJSON() ([]byte, error) { - s, err := jsonpbMarshaller.MarshalToString(r) - return []byte(s), err -} - -func (r *Proof) UnmarshalJSON(b []byte) error { - reader := bytes.NewBuffer(b) - return jsonpbUnmarshaller.Unmarshal(reader, r) -} - -// Some compile time assertions to ensure we don't -// have accidental runtime surprises later on. -// jsonEncodingRoundTripper ensures that asserted -// interfaces implement both MarshalJSON and UnmarshalJSON - -type jsonRoundTripper interface { - json.Marshaler - json.Unmarshaler -} - -var _ jsonRoundTripper = (*ProofOp)(nil) -var _ jsonRoundTripper = (*Proof)(nil) diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go deleted file mode 100644 index 7c262e3..0000000 --- a/crypto/merkle/simple_proof.go +++ /dev/null @@ -1,235 +0,0 @@ -package merkle - -import ( - "bytes" - "errors" - "fmt" - - "github.com/tendermint/tendermint/crypto/tmhash" - tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" -) - -const ( - // MaxAunts is the maximum number of aunts that can be included in a SimpleProof. - // This corresponds to a tree of size 2^100, which should be sufficient for all conceivable purposes. - // This maximum helps prevent Denial-of-Service attacks by limitting the size of the proofs. - MaxAunts = 100 -) - -// SimpleProof represents a simple Merkle proof. -// NOTE: The convention for proofs is to include leaf hashes but to -// exclude the root hash. -// This convention is implemented across IAVL range proofs as well. -// Keep this consistent unless there's a very good reason to change -// everything. This also affects the generalized proof system as -// well. -type SimpleProof struct { - Total int64 `json:"total"` // Total number of items. - Index int64 `json:"index"` // Index of item to prove. - LeafHash []byte `json:"leaf_hash"` // Hash of item value. - Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. -} - -// SimpleProofsFromByteSlices computes inclusion proof for given items. -// proofs[0] is the proof for items[0]. -func SimpleProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*SimpleProof) { - trails, rootSPN := trailsFromByteSlices(items) - rootHash = rootSPN.Hash - proofs = make([]*SimpleProof, len(items)) - for i, trail := range trails { - proofs[i] = &SimpleProof{ - Total: int64(len(items)), - Index: int64(i), - LeafHash: trail.Hash, - Aunts: trail.FlattenAunts(), - } - } - return -} - -// Verify that the SimpleProof proves the root hash. -// Check sp.Index/sp.Total manually if needed -func (sp *SimpleProof) Verify(rootHash []byte, leaf []byte) error { - leafHash := leafHash(leaf) - if sp.Total < 0 { - return errors.New("proof total must be positive") - } - if sp.Index < 0 { - return errors.New("proof index cannot be negative") - } - if !bytes.Equal(sp.LeafHash, leafHash) { - return fmt.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) - } - computedHash := sp.ComputeRootHash() - if !bytes.Equal(computedHash, rootHash) { - return fmt.Errorf("invalid root hash: wanted %X got %X", rootHash, computedHash) - } - return nil -} - -// Compute the root hash given a leaf hash. Does not verify the result. -func (sp *SimpleProof) ComputeRootHash() []byte { - return computeHashFromAunts( - sp.Index, - sp.Total, - sp.LeafHash, - sp.Aunts, - ) -} - -// String implements the stringer interface for SimpleProof. -// It is a wrapper around StringIndented. -func (sp *SimpleProof) String() string { - return sp.StringIndented("") -} - -// StringIndented generates a canonical string representation of a SimpleProof. -func (sp *SimpleProof) StringIndented(indent string) string { - return fmt.Sprintf(`SimpleProof{ -%s Aunts: %X -%s}`, - indent, sp.Aunts, - indent) -} - -// ValidateBasic performs basic validation. -// NOTE: it expects the LeafHash and the elements of Aunts to be of size tmhash.Size, -// and it expects at most MaxAunts elements in Aunts. -func (sp *SimpleProof) ValidateBasic() error { - if sp.Total < 0 { - return errors.New("negative Total") - } - if sp.Index < 0 { - return errors.New("negative Index") - } - if len(sp.LeafHash) != tmhash.Size { - return fmt.Errorf("expected LeafHash size to be %d, got %d", tmhash.Size, len(sp.LeafHash)) - } - if len(sp.Aunts) > MaxAunts { - return fmt.Errorf("expected no more than %d aunts, got %d", MaxAunts, len(sp.Aunts)) - } - for i, auntHash := range sp.Aunts { - if len(auntHash) != tmhash.Size { - return fmt.Errorf("expected Aunts#%d size to be %d, got %d", i, tmhash.Size, len(auntHash)) - } - } - return nil -} - -func (sp *SimpleProof) ToProto() *tmmerkle.SimpleProof { - if sp == nil { - return nil - } - pb := new(tmmerkle.SimpleProof) - - pb.Total = sp.Total - pb.Index = sp.Index - pb.LeafHash = sp.LeafHash - pb.Aunts = sp.Aunts - - return pb -} - -func SimpleProofFromProto(pb *tmmerkle.SimpleProof) (*SimpleProof, error) { - if pb == nil { - return nil, errors.New("nil proof") - } - sp := new(SimpleProof) - - sp.Total = pb.Total - sp.Index = pb.Index - sp.LeafHash = pb.LeafHash - sp.Aunts = pb.Aunts - - return sp, sp.ValidateBasic() -} - -// Use the leafHash and innerHashes to get the root merkle hash. -// If the length of the innerHashes slice isn't exactly correct, the result is nil. -// Recursive impl. -func computeHashFromAunts(index, total int64, leafHash []byte, innerHashes [][]byte) []byte { - if index >= total || index < 0 || total <= 0 { - return nil - } - switch total { - case 0: - panic("Cannot call computeHashFromAunts() with 0 total") - case 1: - if len(innerHashes) != 0 { - return nil - } - return leafHash - default: - if len(innerHashes) == 0 { - return nil - } - numLeft := getSplitPoint(total) - if index < numLeft { - leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) - if leftHash == nil { - return nil - } - return innerHash(leftHash, innerHashes[len(innerHashes)-1]) - } - rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) - if rightHash == nil { - return nil - } - return innerHash(innerHashes[len(innerHashes)-1], rightHash) - } -} - -// SimpleProofNode is a helper structure to construct merkle proof. -// The node and the tree is thrown away afterwards. -// Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. -// node.Parent.Hash = hash(node.Hash, node.Right.Hash) or -// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. -type SimpleProofNode struct { - Hash []byte - Parent *SimpleProofNode - Left *SimpleProofNode // Left sibling (only one of Left,Right is set) - Right *SimpleProofNode // Right sibling (only one of Left,Right is set) -} - -// FlattenAunts will return the inner hashes for the item corresponding to the leaf, -// starting from a leaf SimpleProofNode. -func (spn *SimpleProofNode) FlattenAunts() [][]byte { - // Nonrecursive impl. - innerHashes := [][]byte{} - for spn != nil { - switch { - case spn.Left != nil: - innerHashes = append(innerHashes, spn.Left.Hash) - case spn.Right != nil: - innerHashes = append(innerHashes, spn.Right.Hash) - default: - break - } - spn = spn.Parent - } - return innerHashes -} - -// trails[0].Hash is the leaf hash for items[0]. -// trails[i].Parent.Parent....Parent == root for all i. -func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *SimpleProofNode) { - // Recursive impl. - switch len(items) { - case 0: - return nil, nil - case 1: - trail := &SimpleProofNode{leafHash(items[0]), nil, nil, nil} - return []*SimpleProofNode{trail}, trail - default: - k := getSplitPoint(int64(len(items))) - lefts, leftRoot := trailsFromByteSlices(items[:k]) - rights, rightRoot := trailsFromByteSlices(items[k:]) - rootHash := innerHash(leftRoot.Hash, rightRoot.Hash) - root := &SimpleProofNode{rootHash, nil, nil, nil} - leftRoot.Parent = root - leftRoot.Right = rightRoot - rightRoot.Parent = root - rightRoot.Left = leftRoot - return append(lefts, rights...), root - } -} diff --git a/crypto/merkle/simple_proof_test.go b/crypto/merkle/simple_proof_test.go deleted file mode 100644 index 2dd8b7e..0000000 --- a/crypto/merkle/simple_proof_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package merkle - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - tmrand "github.com/tendermint/tendermint/libs/rand" -) - -func TestSimpleProofValidateBasic(t *testing.T) { - testCases := []struct { - testName string - malleateProof func(*SimpleProof) - errStr string - }{ - {"Good", func(sp *SimpleProof) {}, ""}, - {"Negative Total", func(sp *SimpleProof) { sp.Total = -1 }, "negative Total"}, - {"Negative Index", func(sp *SimpleProof) { sp.Index = -1 }, "negative Index"}, - {"Invalid LeafHash", func(sp *SimpleProof) { sp.LeafHash = make([]byte, 10) }, - "expected LeafHash size to be 32, got 10"}, - {"Too many Aunts", func(sp *SimpleProof) { sp.Aunts = make([][]byte, MaxAunts+1) }, - "expected no more than 100 aunts, got 101"}, - {"Invalid Aunt", func(sp *SimpleProof) { sp.Aunts[0] = make([]byte, 10) }, - "expected Aunts#0 size to be 32, got 10"}, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.testName, func(t *testing.T) { - _, proofs := SimpleProofsFromByteSlices([][]byte{ - []byte("apple"), - []byte("watermelon"), - []byte("kiwi"), - }) - tc.malleateProof(proofs[0]) - err := proofs[0].ValidateBasic() - if tc.errStr != "" { - assert.Contains(t, err.Error(), tc.errStr) - } - }) - } -} - -func TestSimpleProofProtoBuf(t *testing.T) { - testCases := []struct { - testName string - ps1 *SimpleProof - expPass bool - }{ - {"failure empty", &SimpleProof{}, false}, - {"failure nil", nil, false}, - {"success", - &SimpleProof{ - Total: 1, - Index: 1, - LeafHash: tmrand.Bytes(32), - Aunts: [][]byte{tmrand.Bytes(32), tmrand.Bytes(32), tmrand.Bytes(32)}, - }, true}, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.testName, func(t *testing.T) { - proto := tc.ps1.ToProto() - p, err := SimpleProofFromProto(proto) - if tc.expPass { - require.NoError(t, err) - require.Equal(t, tc.ps1, p, tc.testName) - } else { - require.Error(t, err, tc.testName) - } - }) - } -} diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/tree.go similarity index 73% rename from crypto/merkle/simple_tree.go rename to crypto/merkle/tree.go index c9c4784..cbd51ca 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/tree.go @@ -4,9 +4,9 @@ import ( "math/bits" ) -// SimpleHashFromByteSlices computes a Merkle tree where the leaves are the byte slice, +// HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, // in the provided order. -func SimpleHashFromByteSlices(items [][]byte) []byte { +func HashFromByteSlices(items [][]byte) []byte { switch len(items) { case 0: return nil @@ -14,27 +14,27 @@ func SimpleHashFromByteSlices(items [][]byte) []byte { return leafHash(items[0]) default: k := getSplitPoint(int64(len(items))) - left := SimpleHashFromByteSlices(items[:k]) - right := SimpleHashFromByteSlices(items[k:]) + left := HashFromByteSlices(items[:k]) + right := HashFromByteSlices(items[k:]) return innerHash(left, right) } } -// SimpleHashFromByteSliceIterative is an iterative alternative to -// SimpleHashFromByteSlice motivated by potential performance improvements. +// HashFromByteSliceIterative is an iterative alternative to +// HashFromByteSlice motivated by potential performance improvements. // (#2611) had suggested that an iterative version of -// SimpleHashFromByteSlice would be faster, presumably because +// HashFromByteSlice would be faster, presumably because // we can envision some overhead accumulating from stack // frames and function calls. Additionally, a recursive algorithm risks // hitting the stack limit and causing a stack overflow should the tree // be too large. // -// Provided here is an iterative alternative, a simple test to assert +// Provided here is an iterative alternative, a test to assert // correctness and a benchmark. On the performance side, there appears to // be no overall difference: // -// BenchmarkSimpleHashAlternatives/recursive-4 20000 77677 ns/op -// BenchmarkSimpleHashAlternatives/iterative-4 20000 76802 ns/op +// BenchmarkHashAlternatives/recursive-4 20000 77677 ns/op +// BenchmarkHashAlternatives/iterative-4 20000 76802 ns/op // // On the surface it might seem that the additional overhead is due to // the different allocation patterns of the implementations. The recursive @@ -47,9 +47,9 @@ func SimpleHashFromByteSlices(items [][]byte) []byte { // // These preliminary results suggest: // -// 1. The performance of the SimpleHashFromByteSlice is pretty good +// 1. The performance of the HashFromByteSlice is pretty good // 2. Go has low overhead for recursive functions -// 3. The performance of the SimpleHashFromByteSlice routine is dominated +// 3. The performance of the HashFromByteSlice routine is dominated // by the actual hashing of data // // Although this work is in no way exhaustive, point #3 suggests that @@ -59,7 +59,7 @@ func SimpleHashFromByteSlices(items [][]byte) []byte { // Finally, considering that the recursive implementation is easier to // read, it might not be worthwhile to switch to a less intuitive // implementation for so little benefit. -func SimpleHashFromByteSlicesIterative(input [][]byte) []byte { +func HashFromByteSlicesIterative(input [][]byte) []byte { items := make([][]byte, len(input)) for i, leaf := range input { diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/tree_test.go similarity index 86% rename from crypto/merkle/simple_tree_test.go rename to crypto/merkle/tree_test.go index 01d6058..b867925 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/tree_test.go @@ -17,7 +17,7 @@ func (tI testItem) Hash() []byte { return []byte(tI) } -func TestSimpleProof(t *testing.T) { +func TestProof(t *testing.T) { total := 100 @@ -26,9 +26,9 @@ func TestSimpleProof(t *testing.T) { items[i] = testItem(tmrand.Bytes(tmhash.Size)) } - rootHash := SimpleHashFromByteSlices(items) + rootHash := HashFromByteSlices(items) - rootHash2, proofs := SimpleProofsFromByteSlices(items) + rootHash2, proofs := ProofsFromByteSlices(items) require.Equal(t, rootHash, rootHash2, "Unmatched root hashes: %X vs %X", rootHash, rootHash2) @@ -70,7 +70,7 @@ func TestSimpleProof(t *testing.T) { } } -func TestSimpleHashAlternatives(t *testing.T) { +func TestHashAlternatives(t *testing.T) { total := 100 @@ -79,12 +79,12 @@ func TestSimpleHashAlternatives(t *testing.T) { items[i] = testItem(tmrand.Bytes(tmhash.Size)) } - rootHash1 := SimpleHashFromByteSlicesIterative(items) - rootHash2 := SimpleHashFromByteSlices(items) + rootHash1 := HashFromByteSlicesIterative(items) + rootHash2 := HashFromByteSlices(items) require.Equal(t, rootHash1, rootHash2, "Unmatched root hashes: %X vs %X", rootHash1, rootHash2) } -func BenchmarkSimpleHashAlternatives(b *testing.B) { +func BenchmarkHashAlternatives(b *testing.B) { total := 100 items := make([][]byte, total) @@ -95,13 +95,13 @@ func BenchmarkSimpleHashAlternatives(b *testing.B) { b.ResetTimer() b.Run("recursive", func(b *testing.B) { for i := 0; i < b.N; i++ { - _ = SimpleHashFromByteSlices(items) + _ = HashFromByteSlices(items) } }) b.Run("iterative", func(b *testing.B) { for i := 0; i < b.N; i++ { - _ = SimpleHashFromByteSlicesIterative(items) + _ = HashFromByteSlicesIterative(items) } }) } From f0bb09726962a526027b8932c716f3f101211bbb Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 15 Jun 2020 11:14:36 +0200 Subject: [PATCH 57/86] proto: leftover amino (#4986) --- crypto/merkle/codec.go | 12 --------- crypto/merkle/compile.sh | 6 ----- crypto/merkle/proof.go | 1 + crypto/merkle/proof_test.go | 50 ++++++++++++++++++++++++++---------- crypto/merkle/proof_value.go | 24 ++++++++++++----- crypto/merkle/types.go | 12 ++++++--- 6 files changed, 64 insertions(+), 41 deletions(-) delete mode 100644 crypto/merkle/codec.go delete mode 100644 crypto/merkle/compile.sh diff --git a/crypto/merkle/codec.go b/crypto/merkle/codec.go deleted file mode 100644 index 2b6ee35..0000000 --- a/crypto/merkle/codec.go +++ /dev/null @@ -1,12 +0,0 @@ -package merkle - -import ( - amino "github.com/tendermint/go-amino" -) - -var cdc *amino.Codec - -func init() { - cdc = amino.NewCodec() - cdc.Seal() -} diff --git a/crypto/merkle/compile.sh b/crypto/merkle/compile.sh deleted file mode 100644 index 8e4c739..0000000 --- a/crypto/merkle/compile.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/bash - -protoc --gogo_out=. -I $GOPATH/src/ -I . -I $GOPATH/src/github.com/gogo/protobuf/protobuf merkle.proto -echo "--> adding nolint declarations to protobuf generated files" -awk '/package merkle/ { print "//nolint: gas"; print; next }1' merkle.pb.go > merkle.pb.go.new -mv merkle.pb.go.new merkle.pb.go diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index ae463e8..5f869a2 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -134,6 +134,7 @@ func ProofFromProto(pb *tmmerkle.Proof) (*Proof, error) { if pb == nil { return nil, errors.New("nil proof") } + sp := new(Proof) sp.Total = pb.Total diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 398cb3e..b076730 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - amino "github.com/tendermint/go-amino" + "github.com/stretchr/testify/require" tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" ) @@ -28,21 +28,17 @@ func NewDominoOp(key, input, output string) DominoOp { } } -//nolint:unused -func DominoOpDecoder(pop tmmerkle.ProofOp) (ProofOperator, error) { - if pop.Type != ProofOpDomino { - panic("unexpected proof op type") +func (dop DominoOp) ProofOp() tmmerkle.ProofOp { + dopb := tmmerkle.DominoOp{ + Key: dop.key, + Input: dop.Input, + Output: dop.Output, } - var op DominoOp // a bit strange as we'll discard this, but it works. - err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) + bz, err := dopb.Marshal() if err != nil { - return nil, fmt.Errorf("decoding ProofOp.Data into ValueOp: %w", err) + panic(err) } - return NewDominoOp(string(pop.Key), op.Input, op.Output), nil -} -func (dop DominoOp) ProofOp() tmmerkle.ProofOp { - bz := amino.MustMarshalBinaryLengthPrefixed(dop) return tmmerkle.ProofOp{ Type: ProofOpDomino, Key: []byte(dop.key), @@ -72,8 +68,6 @@ func TestProofOperators(t *testing.T) { // ProofRuntime setup // TODO test this somehow. - // prt := NewProofRuntime() - // prt.RegisterOpDecoder(ProofOpDomino, DominoOpDecoder) // ProofOperators setup op1 := NewDominoOp("KEY1", "INPUT1", "INPUT2") @@ -175,3 +169,31 @@ func TestProofValidateBasic(t *testing.T) { }) } } +func TestVoteProtobuf(t *testing.T) { + + _, proofs := ProofsFromByteSlices([][]byte{ + []byte("apple"), + []byte("watermelon"), + []byte("kiwi"), + }) + testCases := []struct { + testName string + v1 *Proof + expPass bool + }{ + {"empty proof", &Proof{}, false}, + {"failure nil", nil, false}, + {"success", proofs[0], true}, + } + for _, tc := range testCases { + pb := tc.v1.ToProto() + + v, err := ProofFromProto(pb) + if tc.expPass { + require.NoError(t, err) + require.Equal(t, tc.v1, v, tc.testName) + } else { + require.Error(t, err) + } + } +} diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index b9be68c..9d92d69 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -13,8 +13,8 @@ const ProofOpValue = "simple:v" // ValueOp takes a key and a single value as argument and // produces the root hash. The corresponding tree structure is // the SimpleMap tree. SimpleMap takes a Hasher, and currently -// Tendermint uses aminoHasher. ValueOp should support -// the hash function as used in aminoHasher. TODO support +// Tendermint uses tmhash. SimpleValueOp should support +// the hash function as used in tmhash. TODO support // additional hash functions here as options/args to this // operator. // @@ -41,16 +41,28 @@ func ValueOpDecoder(pop tmmerkle.ProofOp) (ProofOperator, error) { if pop.Type != ProofOpValue { return nil, fmt.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpValue) } - var op ValueOp // a bit strange as we'll discard this, but it works. - err := cdc.UnmarshalBinaryLengthPrefixed(pop.Data, &op) + var pbop tmmerkle.ValueOp // a bit strange as we'll discard this, but it works. + err := pbop.Unmarshal(pop.Data) if err != nil { return nil, fmt.Errorf("decoding ProofOp.Data into ValueOp: %w", err) } - return NewValueOp(pop.Key, op.Proof), nil + + sp, err := ProofFromProto(pbop.Proof) + if err != nil { + return nil, err + } + return NewValueOp(pop.Key, sp), nil } func (op ValueOp) ProofOp() tmmerkle.ProofOp { - bz := cdc.MustMarshalBinaryLengthPrefixed(op) + pbval := tmmerkle.ValueOp{ + Key: op.key, + Proof: op.Proof.ToProto(), + } + bz, err := pbval.Marshal() + if err != nil { + panic(err) + } return tmmerkle.ProofOp{ Type: ProofOpValue, Key: op.key, diff --git a/crypto/merkle/types.go b/crypto/merkle/types.go index 97a4787..6a5c7e6 100644 --- a/crypto/merkle/types.go +++ b/crypto/merkle/types.go @@ -1,9 +1,8 @@ package merkle import ( + "encoding/binary" "io" - - amino "github.com/tendermint/go-amino" ) // Tree is a Merkle tree interface. @@ -29,5 +28,12 @@ type Tree interface { // Uvarint length prefixed byteslice func encodeByteSlice(w io.Writer, bz []byte) (err error) { - return amino.EncodeByteSlice(w, bz) + var buf [binary.MaxVarintLen64]byte + n := binary.PutUvarint(buf[:], uint64(len(bz))) + _, err = w.Write(buf[0:n]) + if err != nil { + return + } + _, err = w.Write(bz) + return } From 23e6e1686e2f91717161ba8e60992e38820dbff2 Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 15 Jun 2020 13:17:12 +0200 Subject: [PATCH 58/86] encoding: remove codecs (#4996) ## Description This pr removes amino from tendermint. Closes: #4278 --- crypto/merkle/proof_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index b076730..fd89be2 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" ) From 1e45e294496e23d7d39419ecadca2f4f34774f33 Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 22 Jun 2020 10:00:51 +0200 Subject: [PATCH 59/86] proto: folder structure adhere to buf (#5025) --- crypto/merkle/proof.go | 2 +- crypto/merkle/proof_op.go | 2 +- crypto/merkle/proof_test.go | 2 +- crypto/merkle/proof_value.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 5f869a2..9a64410 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" - tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" + tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto/merkle" ) const ( diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 8994a29..ea73a13 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" + tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto/merkle" ) //---------------------------------------- diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index fd89be2..fc502d2 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" + tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto/merkle" ) const ProofOpDomino = "test:domino" diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 9d92d69..ef899f9 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" - tmmerkle "github.com/tendermint/tendermint/proto/crypto/merkle" + tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto/merkle" ) const ProofOpValue = "simple:v" From 589855c5cefdc9a9804e3dcb927f0f7ce6d9123d Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Wed, 8 Jul 2020 17:47:01 +0200 Subject: [PATCH 60/86] proto: reorganize Protobuf schemas (#5102) Reorganizes the Protobuf schemas. It is mostly bikeshedding, so if something is contentious or causes a lot of extra work then I'm fine with reverting. Some Protobuf and Go import paths will change. * Move `abci/types/types.proto` to `abci/types.proto`. * Move `crypto/keys/types.proto` and `crypto/merkle/types.proto` to `crypto/keys.proto` and `crypto/proof.proto`. * Drop the use of `msgs` in filenames, as "message" is a very overloaded term (all Protobuf types are messages, and we also have `message Message`). Use `types.proto` as a catch-all, and otherwise name files by conceptual grouping instead of message kind. --- crypto/merkle/proof.go | 8 ++++---- crypto/merkle/proof_op.go | 16 ++++++++-------- crypto/merkle/proof_test.go | 8 ++++---- crypto/merkle/proof_value.go | 12 ++++++------ 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 9a64410..f9a42fd 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" - tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto/merkle" + tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ( @@ -116,11 +116,11 @@ func (sp *Proof) ValidateBasic() error { return nil } -func (sp *Proof) ToProto() *tmmerkle.Proof { +func (sp *Proof) ToProto() *tmcrypto.Proof { if sp == nil { return nil } - pb := new(tmmerkle.Proof) + pb := new(tmcrypto.Proof) pb.Total = sp.Total pb.Index = sp.Index @@ -130,7 +130,7 @@ func (sp *Proof) ToProto() *tmmerkle.Proof { return pb } -func ProofFromProto(pb *tmmerkle.Proof) (*Proof, error) { +func ProofFromProto(pb *tmcrypto.Proof) (*Proof, error) { if pb == nil { return nil, errors.New("nil proof") } diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index ea73a13..8c6937a 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto/merkle" + tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) //---------------------------------------- @@ -21,7 +21,7 @@ import ( type ProofOperator interface { Run([][]byte) ([][]byte, error) GetKey() []byte - ProofOp() tmmerkle.ProofOp + ProofOp() tmcrypto.ProofOp } //---------------------------------------- @@ -71,7 +71,7 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er //---------------------------------------- // ProofRuntime - main entrypoint -type OpDecoder func(tmmerkle.ProofOp) (ProofOperator, error) +type OpDecoder func(tmcrypto.ProofOp) (ProofOperator, error) type ProofRuntime struct { decoders map[string]OpDecoder @@ -91,7 +91,7 @@ func (prt *ProofRuntime) RegisterOpDecoder(typ string, dec OpDecoder) { prt.decoders[typ] = dec } -func (prt *ProofRuntime) Decode(pop tmmerkle.ProofOp) (ProofOperator, error) { +func (prt *ProofRuntime) Decode(pop tmcrypto.ProofOp) (ProofOperator, error) { decoder := prt.decoders[pop.Type] if decoder == nil { return nil, fmt.Errorf("unrecognized proof type %v", pop.Type) @@ -99,7 +99,7 @@ func (prt *ProofRuntime) Decode(pop tmmerkle.ProofOp) (ProofOperator, error) { return decoder(pop) } -func (prt *ProofRuntime) DecodeProof(proof *tmmerkle.ProofOps) (ProofOperators, error) { +func (prt *ProofRuntime) DecodeProof(proof *tmcrypto.ProofOps) (ProofOperators, error) { poz := make(ProofOperators, 0, len(proof.Ops)) for _, pop := range proof.Ops { operator, err := prt.Decode(pop) @@ -111,17 +111,17 @@ func (prt *ProofRuntime) DecodeProof(proof *tmmerkle.ProofOps) (ProofOperators, return poz, nil } -func (prt *ProofRuntime) VerifyValue(proof *tmmerkle.ProofOps, root []byte, keypath string, value []byte) (err error) { +func (prt *ProofRuntime) VerifyValue(proof *tmcrypto.ProofOps, root []byte, keypath string, value []byte) (err error) { return prt.Verify(proof, root, keypath, [][]byte{value}) } // TODO In the long run we'll need a method of classifcation of ops, // whether existence or absence or perhaps a third? -func (prt *ProofRuntime) VerifyAbsence(proof *tmmerkle.ProofOps, root []byte, keypath string) (err error) { +func (prt *ProofRuntime) VerifyAbsence(proof *tmcrypto.ProofOps, root []byte, keypath string) (err error) { return prt.Verify(proof, root, keypath, nil) } -func (prt *ProofRuntime) Verify(proof *tmmerkle.ProofOps, root []byte, keypath string, args [][]byte) (err error) { +func (prt *ProofRuntime) Verify(proof *tmcrypto.ProofOps, root []byte, keypath string, args [][]byte) (err error) { poz, err := prt.DecodeProof(proof) if err != nil { return fmt.Errorf("decoding proof: %w", err) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index fc502d2..22ab900 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto/merkle" + tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ProofOpDomino = "test:domino" @@ -29,8 +29,8 @@ func NewDominoOp(key, input, output string) DominoOp { } } -func (dop DominoOp) ProofOp() tmmerkle.ProofOp { - dopb := tmmerkle.DominoOp{ +func (dop DominoOp) ProofOp() tmcrypto.ProofOp { + dopb := tmcrypto.DominoOp{ Key: dop.key, Input: dop.Input, Output: dop.Output, @@ -40,7 +40,7 @@ func (dop DominoOp) ProofOp() tmmerkle.ProofOp { panic(err) } - return tmmerkle.ProofOp{ + return tmcrypto.ProofOp{ Type: ProofOpDomino, Key: []byte(dop.key), Data: bz, diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index ef899f9..2bc9ffc 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" - tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto/merkle" + tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ProofOpValue = "simple:v" @@ -37,11 +37,11 @@ func NewValueOp(key []byte, proof *Proof) ValueOp { } } -func ValueOpDecoder(pop tmmerkle.ProofOp) (ProofOperator, error) { +func ValueOpDecoder(pop tmcrypto.ProofOp) (ProofOperator, error) { if pop.Type != ProofOpValue { return nil, fmt.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpValue) } - var pbop tmmerkle.ValueOp // a bit strange as we'll discard this, but it works. + var pbop tmcrypto.ValueOp // a bit strange as we'll discard this, but it works. err := pbop.Unmarshal(pop.Data) if err != nil { return nil, fmt.Errorf("decoding ProofOp.Data into ValueOp: %w", err) @@ -54,8 +54,8 @@ func ValueOpDecoder(pop tmmerkle.ProofOp) (ProofOperator, error) { return NewValueOp(pop.Key, sp), nil } -func (op ValueOp) ProofOp() tmmerkle.ProofOp { - pbval := tmmerkle.ValueOp{ +func (op ValueOp) ProofOp() tmcrypto.ProofOp { + pbval := tmcrypto.ValueOp{ Key: op.key, Proof: op.Proof.ToProto(), } @@ -63,7 +63,7 @@ func (op ValueOp) ProofOp() tmmerkle.ProofOp { if err != nil { panic(err) } - return tmmerkle.ProofOp{ + return tmcrypto.ProofOp{ Type: ProofOpValue, Key: op.key, Data: bz, From 2561e733e691d092563338d2502ce507364f03c4 Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 3 Aug 2020 09:57:06 +0200 Subject: [PATCH 61/86] ci: freeze golangci action version (#5196) ## Description This PR updates golang-ci to latest and stops looking at master for the action. Closes: #XXX --- crypto/merkle/proof_key_path_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/merkle/proof_key_path_test.go b/crypto/merkle/proof_key_path_test.go index 34c918f..22e3e21 100644 --- a/crypto/merkle/proof_key_path_test.go +++ b/crypto/merkle/proof_key_path_test.go @@ -26,7 +26,7 @@ func TestKeyPath(t *testing.T) { keys[i][j] = alphanum[rand.Intn(len(alphanum))] } case KeyEncodingHex: - rand.Read(keys[i]) //nolint: gosec + rand.Read(keys[i]) default: panic("Unexpected encoding") } From f9e7f1dbae5834eebdcce27bd91f7236cf5f7d76 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Tue, 11 Aug 2020 12:31:05 +0200 Subject: [PATCH 62/86] merkle: return hashes for empty merkle trees (#5193) Fixes #5192. @liamsi Can you verify that the test vectors match the Rust implementation? I updated `ProofsFromByteSlices()` as well, anything else that should be updated? --- crypto/merkle/hash.go | 5 +++++ crypto/merkle/proof.go | 2 +- crypto/merkle/rfc6962_test.go | 11 +++++++++-- crypto/merkle/tree.go | 6 +++--- crypto/merkle/tree_test.go | 33 ++++++++++++++++++++++++++++++++- 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index 4e24046..d45130f 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -10,6 +10,11 @@ var ( innerPrefix = []byte{1} ) +// returns tmhash() +func emptyHash() []byte { + return tmhash.Sum([]byte{}) +} + // returns tmhash(0x00 || leaf) func leafHash(leaf []byte) []byte { return tmhash.Sum(append(leafPrefix, leaf...)) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index f9a42fd..ab43f30 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -217,7 +217,7 @@ func trailsFromByteSlices(items [][]byte) (trails []*ProofNode, root *ProofNode) // Recursive impl. switch len(items) { case 0: - return nil, nil + return []*ProofNode{}, &ProofNode{emptyHash(), nil, nil, nil} case 1: trail := &ProofNode{leafHash(items[0]), nil, nil, nil} return []*ProofNode{trail}, trail diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go index 6c50816..c762cda 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/crypto/merkle/rfc6962_test.go @@ -28,13 +28,20 @@ func TestRFC6962Hasher(t *testing.T) { leafHash := leafHashTrail.Hash _, leafHashTrail = trailsFromByteSlices([][]byte{{}}) emptyLeafHash := leafHashTrail.Hash + _, emptyHashTrail := trailsFromByteSlices([][]byte{}) + emptyTreeHash := emptyHashTrail.Hash for _, tc := range []struct { desc string got []byte want string }{ - // Since creating a merkle tree of no leaves is unsupported here, we skip - // the corresponding trillian test vector. + // Check that empty trees return the hash of an empty string. + // echo -n '' | sha256sum + { + desc: "RFC6962 Empty Tree", + want: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"[:tmhash.Size*2], + got: emptyTreeHash, + }, // Check that the empty hash is not the same as the hash of an empty leaf. // echo -n 00 | xxd -r -p | sha256sum diff --git a/crypto/merkle/tree.go b/crypto/merkle/tree.go index cbd51ca..466c434 100644 --- a/crypto/merkle/tree.go +++ b/crypto/merkle/tree.go @@ -5,11 +5,11 @@ import ( ) // HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, -// in the provided order. +// in the provided order. It follows RFC-6962. func HashFromByteSlices(items [][]byte) []byte { switch len(items) { case 0: - return nil + return emptyHash() case 1: return leafHash(items[0]) default: @@ -70,7 +70,7 @@ func HashFromByteSlicesIterative(input [][]byte) []byte { for { switch size { case 0: - return nil + return emptyHash() case 1: return items[0] default: diff --git a/crypto/merkle/tree_test.go b/crypto/merkle/tree_test.go index b867925..bdf4698 100644 --- a/crypto/merkle/tree_test.go +++ b/crypto/merkle/tree_test.go @@ -1,8 +1,10 @@ package merkle import ( + "encoding/hex" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" @@ -17,8 +19,37 @@ func (tI testItem) Hash() []byte { return []byte(tI) } +func TestHashFromByteSlices(t *testing.T) { + testcases := map[string]struct { + slices [][]byte + expectHash string // in hex format + }{ + "nil": {nil, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + "empty": {[][]byte{}, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + "single": {[][]byte{{1, 2, 3}}, "054edec1d0211f624fed0cbca9d4f9400b0e491c43742af2c5b0abebf0c990d8"}, + "single blank": {[][]byte{{}}, "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d"}, + "two": {[][]byte{{1, 2, 3}, {4, 5, 6}}, "82e6cfce00453804379b53962939eaa7906b39904be0813fcadd31b100773c4b"}, + "many": { + [][]byte{{1, 2}, {3, 4}, {5, 6}, {7, 8}, {9, 10}}, + "f326493eceab4f2d9ffbc78c59432a0a005d6ea98392045c74df5d14a113be18", + }, + } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + hash := HashFromByteSlices(tc.slices) + assert.Equal(t, tc.expectHash, hex.EncodeToString(hash)) + }) + } +} + func TestProof(t *testing.T) { + // Try an empty proof first + rootHash, proofs := ProofsFromByteSlices([][]byte{}) + require.Equal(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", hex.EncodeToString(rootHash)) + require.Empty(t, proofs) + total := 100 items := make([][]byte, total) @@ -26,7 +57,7 @@ func TestProof(t *testing.T) { items[i] = testItem(tmrand.Bytes(tmhash.Size)) } - rootHash := HashFromByteSlices(items) + rootHash = HashFromByteSlices(items) rootHash2, proofs := ProofsFromByteSlices(items) From cfbd27884ad5b9312da3f1498a6bcb436db38184 Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 17 Aug 2020 16:40:50 +0200 Subject: [PATCH 63/86] lint: add markdown linter (#5254) --- crypto/merkle/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/merkle/README.md b/crypto/merkle/README.md index c449783..16b1abb 100644 --- a/crypto/merkle/README.md +++ b/crypto/merkle/README.md @@ -1,4 +1,4 @@ -## Simple Merkle Tree +# Merkle Tree For smaller static data structures that don't require immutable snapshots or mutability; for instance the transactions and validation signatures of a block can be hashed using this simple merkle tree logic. From 1b6ad2814dca9cedb92768af0f61ec6e9a971403 Mon Sep 17 00:00:00 2001 From: Marko Date: Fri, 4 Sep 2020 13:58:03 +0200 Subject: [PATCH 64/86] lint: add errchecks (#5316) ## Description Work towards enabling errcheck ref #5059 --- crypto/merkle/proof_value.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 2bc9ffc..b613ebe 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -80,13 +80,13 @@ func (op ValueOp) Run(args [][]byte) ([][]byte, error) { } value := args[0] hasher := tmhash.New() - hasher.Write(value) // does not error + hasher.Write(value) //nolint: errcheck // does not error vhash := hasher.Sum(nil) bz := new(bytes.Buffer) // Wrap to hash the KVPair. - encodeByteSlice(bz, op.key) // does not error - encodeByteSlice(bz, vhash) // does not error + encodeByteSlice(bz, op.key) //nolint: errcheck // does not error + encodeByteSlice(bz, vhash) //nolint: errcheck // does not error kvhash := leafHash(bz.Bytes()) if !bytes.Equal(kvhash, op.Proof.LeafHash) { From f5530721321f343b0462b3c27aad18227eb5371c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 12 Oct 2020 16:36:37 +0400 Subject: [PATCH 65/86] light/rpc: fix ABCIQuery (#5375) Closes #5106 --- crypto/merkle/proof_op.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 8c6937a..038037c 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -60,7 +60,7 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er } } if !bytes.Equal(root, args[0]) { - return fmt.Errorf("calculated root hash is invalid: expected %+v but got %+v", root, args[0]) + return fmt.Errorf("calculated root hash is invalid: expected %X but got %X", root, args[0]) } if len(keys) != 0 { return errors.New("keypath not consumed all") From f5e7b5905d3bf3b879f0bc76c034ee00bafcbdbc Mon Sep 17 00:00:00 2001 From: Marko Baricevic Date: Mon, 19 Oct 2020 10:52:38 +0200 Subject: [PATCH 66/86] merge latest master --- crypto/merkle/proof_op.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index c2ad1a0..902784d 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -60,7 +60,7 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er } } if !bytes.Equal(root, args[0]) { - return fmt.Errorf("calculated root hash is invalid: expected %+v but got %+v", root, args[0]) + return fmt.Errorf("calculated root hash is invalid: expected %X but got %X", root, args[0]) } if len(keys) != 0 { return errors.New("keypath not consumed all") From ab60d444f9bb6be0c311bdf3aba39a8d0bd25e45 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 12 Oct 2020 16:36:37 +0400 Subject: [PATCH 67/86] light/rpc: fix ABCIQuery (#5375) Closes #5106 --- crypto/merkle/proof_op.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 8c6937a..038037c 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -60,7 +60,7 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er } } if !bytes.Equal(root, args[0]) { - return fmt.Errorf("calculated root hash is invalid: expected %+v but got %+v", root, args[0]) + return fmt.Errorf("calculated root hash is invalid: expected %X but got %X", root, args[0]) } if len(keys) != 0 { return errors.New("keypath not consumed all") From 19b8acd4f40494b60bfe44a8c25e440700a7422b Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 17 Mar 2021 14:55:05 +0000 Subject: [PATCH 68/86] rpc/jsonrpc/server: return an error in WriteRPCResponseHTTP(Error) (bp #6204) (#6230) * rpc/jsonrpc/server: return an error in WriteRPCResponseHTTP(Error) (#6204) instead of panicking Closes #5529 (cherry picked from commit 00b952416836cb568ee904903d30b35a6178bad1) # Conflicts: # CHANGELOG_PENDING.md # rpc/jsonrpc/server/http_json_handler.go # rpc/jsonrpc/server/http_server.go # rpc/jsonrpc/server/http_server_test.go # rpc/jsonrpc/server/http_uri_handler.go * resolve conflicts * fix linting * fix conflict Co-authored-by: Anton Kaliaev Co-authored-by: Marko Baricevic --- crypto/merkle/proof_value.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index b613ebe..ab77621 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -80,13 +80,13 @@ func (op ValueOp) Run(args [][]byte) ([][]byte, error) { } value := args[0] hasher := tmhash.New() - hasher.Write(value) //nolint: errcheck // does not error + hasher.Write(value) vhash := hasher.Sum(nil) bz := new(bytes.Buffer) // Wrap to hash the KVPair. - encodeByteSlice(bz, op.key) //nolint: errcheck // does not error - encodeByteSlice(bz, vhash) //nolint: errcheck // does not error + encodeByteSlice(bz, op.key) // nolint: errcheck // does not error + encodeByteSlice(bz, vhash) // nolint: errcheck // does not error kvhash := leafHash(bz.Bytes()) if !bytes.Equal(kvhash, op.Proof.LeafHash) { From edfeffbc3f8dfd041bd4f2cc084e9acad9d9426b Mon Sep 17 00:00:00 2001 From: Marko Date: Fri, 16 Apr 2021 16:11:59 +0000 Subject: [PATCH 69/86] db: remove tm-db (#241) * bring over badger and memdb * cleanup e2e tests * add tests * cleanup unused code * remove extra flush Co-authored-by: Ismail Khoffi Co-authored-by: Marko Baricevic --- crypto/merkle/proof_value.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 411d1f3..b683320 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -80,7 +80,7 @@ func (op ValueOp) Run(args [][]byte) ([][]byte, error) { } value := args[0] hasher := tmhash.New() - hasher.Write(value) //nolint: errcheck // does not error + hasher.Write(value) vhash := hasher.Sum(nil) bz := new(bytes.Buffer) From 38c048c3cbfbb197f543147d2fcd59910cd34aed Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Sat, 22 May 2021 19:42:53 -0300 Subject: [PATCH 70/86] crypto/merkle: optimize cpu and allocations Signed-off-by: Ignacio Hagopian --- crypto/merkle/hash.go | 18 ++++++++++++++++++ crypto/merkle/proof.go | 2 +- crypto/merkle/proof_key_path_test.go | 1 + crypto/merkle/proof_test.go | 1 - crypto/merkle/tree.go | 18 ++++++++++++------ 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index 80d3c63..9fcf39e 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -1,6 +1,8 @@ package merkle import ( + "hash" + "github.com/lazyledger/lazyledger-core/crypto/tmhash" ) @@ -20,7 +22,23 @@ func leafHash(leaf []byte) []byte { return tmhash.Sum(append(leafPrefix, leaf...)) } +// returns tmhash(0x00 || leaf) +func leafHashOpt(s hash.Hash, leaf []byte) []byte { + s.Reset() + s.Write(leafPrefix) + s.Write(leaf) + return s.Sum(nil) +} + // returns tmhash(0x01 || left || right) func innerHash(left []byte, right []byte) []byte { return tmhash.Sum(append(innerPrefix, append(left, right...)...)) } + +func innerHashOpt(s hash.Hash, left []byte, right []byte) []byte { + s.Reset() + s.Write(innerPrefix) + s.Write(left) + s.Write(right) + return s.Sum(nil) +} diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 6fb3f65..a9d5a3d 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -50,13 +50,13 @@ func ProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*Proof) { // Verify that the Proof proves the root hash. // Check sp.Index/sp.Total manually if needed func (sp *Proof) Verify(rootHash []byte, leaf []byte) error { - leafHash := leafHash(leaf) if sp.Total < 0 { return errors.New("proof total must be positive") } if sp.Index < 0 { return errors.New("proof index cannot be negative") } + leafHash := leafHash(leaf) if !bytes.Equal(sp.LeafHash, leafHash) { return fmt.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) } diff --git a/crypto/merkle/proof_key_path_test.go b/crypto/merkle/proof_key_path_test.go index 22e3e21..0cc9476 100644 --- a/crypto/merkle/proof_key_path_test.go +++ b/crypto/merkle/proof_key_path_test.go @@ -35,6 +35,7 @@ func TestKeyPath(t *testing.T) { res, err := KeyPathToKeys(path.String()) require.Nil(t, err) + require.Equal(t, len(keys), len(res)) for i, key := range keys { require.Equal(t, key, res[i]) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 1b5153b..ea31015 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -171,7 +171,6 @@ func TestProofValidateBasic(t *testing.T) { } } func TestVoteProtobuf(t *testing.T) { - _, proofs := ProofsFromByteSlices([][]byte{ []byte("apple"), []byte("watermelon"), diff --git a/crypto/merkle/tree.go b/crypto/merkle/tree.go index 466c434..7ce052b 100644 --- a/crypto/merkle/tree.go +++ b/crypto/merkle/tree.go @@ -1,22 +1,27 @@ package merkle import ( + "crypto/sha256" + "hash" "math/bits" ) // HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, // in the provided order. It follows RFC-6962. func HashFromByteSlices(items [][]byte) []byte { + return hashFromByteSlices(sha256.New(), items) +} +func hashFromByteSlices(sha hash.Hash, items [][]byte) []byte { switch len(items) { case 0: return emptyHash() case 1: - return leafHash(items[0]) + return leafHashOpt(sha, items[0]) default: k := getSplitPoint(int64(len(items))) - left := HashFromByteSlices(items[:k]) - right := HashFromByteSlices(items[k:]) - return innerHash(left, right) + left := hashFromByteSlices(sha, items[:k]) + right := hashFromByteSlices(sha, items[k:]) + return innerHashOpt(sha, left, right) } } @@ -60,10 +65,11 @@ func HashFromByteSlices(items [][]byte) []byte { // read, it might not be worthwhile to switch to a less intuitive // implementation for so little benefit. func HashFromByteSlicesIterative(input [][]byte) []byte { + sha := sha256.New() items := make([][]byte, len(input)) for i, leaf := range input { - items[i] = leafHash(leaf) + items[i] = leafHashOpt(sha, leaf) } size := len(items) @@ -78,7 +84,7 @@ func HashFromByteSlicesIterative(input [][]byte) []byte { wp := 0 // write position for rp < size { if rp+1 < size { - items[wp] = innerHash(items[rp], items[rp+1]) + items[wp] = innerHashOpt(sha, items[rp], items[rp+1]) rp += 2 } else { items[wp] = items[rp] From 214123f57e6f77426abc73902a659a118594009f Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Mon, 16 Aug 2021 17:19:04 +0300 Subject: [PATCH 71/86] Rebrand from lazyledger/lazyledger-core -> celestiaorg/celestia-core (#476) * rebrand * replace usage of lazyledger versions of nmt and rsmt2d * replace refs in adrs * update readme * update make file * clean up remaining instances of lazyledger * update to the latest rsmt2d api * only use original square when merging shares * go mod tidy + remove debugging statements * increase timeout for TestWriteDiscoveryReadData from 10s to 15s for CI * continues rebrand for ADRs and else * remove redundant declaration of originalWidth Co-authored-by: Ismail Khoffi Co-authored-by: Wondertan Co-authored-by: Ismail Khoffi --- crypto/merkle/hash.go | 2 +- crypto/merkle/proof.go | 4 ++-- crypto/merkle/proof_op.go | 2 +- crypto/merkle/proof_test.go | 2 +- crypto/merkle/proof_value.go | 4 ++-- crypto/merkle/rfc6962_test.go | 2 +- crypto/merkle/tree_test.go | 6 +++--- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index 9fcf39e..e10b2cf 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -3,7 +3,7 @@ package merkle import ( "hash" - "github.com/lazyledger/lazyledger-core/crypto/tmhash" + "github.com/celestiaorg/celestia-core/crypto/tmhash" ) // TODO: make these have a large predefined capacity diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index a9d5a3d..925c6ab 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" - "github.com/lazyledger/lazyledger-core/crypto/tmhash" - tmcrypto "github.com/lazyledger/lazyledger-core/proto/tendermint/crypto" + "github.com/celestiaorg/celestia-core/crypto/tmhash" + tmcrypto "github.com/celestiaorg/celestia-core/proto/tendermint/crypto" ) const ( diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 902784d..87ac90b 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - tmcrypto "github.com/lazyledger/lazyledger-core/proto/tendermint/crypto" + tmcrypto "github.com/celestiaorg/celestia-core/proto/tendermint/crypto" ) //---------------------------------------- diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index ea31015..0c1c82d 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmcrypto "github.com/lazyledger/lazyledger-core/proto/tendermint/crypto" + tmcrypto "github.com/celestiaorg/celestia-core/proto/tendermint/crypto" ) const ProofOpDomino = "test:domino" diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index b683320..37ecc74 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" - "github.com/lazyledger/lazyledger-core/crypto/tmhash" - tmcrypto "github.com/lazyledger/lazyledger-core/proto/tendermint/crypto" + "github.com/celestiaorg/celestia-core/crypto/tmhash" + tmcrypto "github.com/celestiaorg/celestia-core/proto/tendermint/crypto" ) const ProofOpValue = "simple:v" diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go index 80e45b8..1481523 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/crypto/merkle/rfc6962_test.go @@ -20,7 +20,7 @@ import ( "encoding/hex" "testing" - "github.com/lazyledger/lazyledger-core/crypto/tmhash" + "github.com/celestiaorg/celestia-core/crypto/tmhash" ) func TestRFC6962Hasher(t *testing.T) { diff --git a/crypto/merkle/tree_test.go b/crypto/merkle/tree_test.go index f7c00f6..486163e 100644 --- a/crypto/merkle/tree_test.go +++ b/crypto/merkle/tree_test.go @@ -7,10 +7,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmrand "github.com/lazyledger/lazyledger-core/libs/rand" - . "github.com/lazyledger/lazyledger-core/libs/test" + tmrand "github.com/celestiaorg/celestia-core/libs/rand" + . "github.com/celestiaorg/celestia-core/libs/test" - "github.com/lazyledger/lazyledger-core/crypto/tmhash" + "github.com/celestiaorg/celestia-core/crypto/tmhash" ) type testItem []byte From 8895f29f9347cfa2ef4be0a3abd1699a9b9eb487 Mon Sep 17 00:00:00 2001 From: samricotta <37125168+samricotta@users.noreply.github.com> Date: Thu, 11 Aug 2022 10:41:41 +0200 Subject: [PATCH 72/86] Backport of sam/abci-responses (#9090) (#9159) *backport of sam/abci-responses Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> --- crypto/merkle/doc.go | 25 ++++++++++++------------- crypto/merkle/tree.go | 8 ++++---- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/crypto/merkle/doc.go b/crypto/merkle/doc.go index 865c302..fe50b34 100644 --- a/crypto/merkle/doc.go +++ b/crypto/merkle/doc.go @@ -12,20 +12,19 @@ second pre-image attacks. Hence, use this library with caution. Otherwise you might run into similar issues as, e.g., in early Bitcoin: https://bitcointalk.org/?topic=102395 - * - / \ - / \ - / \ - / \ - * * - / \ / \ - / \ / \ - / \ / \ - * * * h6 - / \ / \ / \ - h0 h1 h2 h3 h4 h5 + * + / \ + / \ + / \ + / \ + * * + / \ / \ + / \ / \ + / \ / \ + * * * h6 + / \ / \ / \ + h0 h1 h2 h3 h4 h5 TODO(ismail): add 2nd pre-image protection or clarify further on how we use this and why this secure. - */ package merkle diff --git a/crypto/merkle/tree.go b/crypto/merkle/tree.go index 466c434..089c2f8 100644 --- a/crypto/merkle/tree.go +++ b/crypto/merkle/tree.go @@ -47,10 +47,10 @@ func HashFromByteSlices(items [][]byte) []byte { // // These preliminary results suggest: // -// 1. The performance of the HashFromByteSlice is pretty good -// 2. Go has low overhead for recursive functions -// 3. The performance of the HashFromByteSlice routine is dominated -// by the actual hashing of data +// 1. The performance of the HashFromByteSlice is pretty good +// 2. Go has low overhead for recursive functions +// 3. The performance of the HashFromByteSlice routine is dominated +// by the actual hashing of data // // Although this work is in no way exhaustive, point #3 suggests that // optimization of this routine would need to take an alternative From 0ba3dc61e2b13b57962943526028233eb830c995 Mon Sep 17 00:00:00 2001 From: samricotta <37125168+samricotta@users.noreply.github.com> Date: Thu, 11 Aug 2022 15:53:17 +0200 Subject: [PATCH 73/86] Bump linter to 1.47 (#9218) *bump linter to 1.47 Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> --- crypto/merkle/proof_value.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index ab77621..842dc82 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -85,8 +85,8 @@ func (op ValueOp) Run(args [][]byte) ([][]byte, error) { bz := new(bytes.Buffer) // Wrap to hash the KVPair. - encodeByteSlice(bz, op.key) // nolint: errcheck // does not error - encodeByteSlice(bz, vhash) // nolint: errcheck // does not error + encodeByteSlice(bz, op.key) //nolint: errcheck // does not error + encodeByteSlice(bz, vhash) //nolint: errcheck // does not error kvhash := leafHash(bz.Bytes()) if !bytes.Equal(kvhash, op.Proof.LeafHash) { From 616afe2ad189f08e0400c1b3942e76d6baa85b84 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 13 Dec 2022 10:27:58 +0100 Subject: [PATCH 74/86] chore: update to tm v0.34.23 (#906) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * mempool: reduce lock contention during CheckTx (backport #8983) (#8985) A manual cherry-pick of 9e64c95. The way this was originally structured, we reacquired the lock after issuing the initial ABCI CheckTx call, only to immediately release it. Restructure the code so that this redundant acquire is no longer necessary. * mempool: release lock during app connection flush (#8986) A manual backport of #8984. This case is symmetric to what we did for CheckTx calls, where we release the mempool mutex to ensure callbacks can fire during call setup. We also need this behaviour for application flush, for the same reason: The caller holds the lock by contract from the Mempool interface. * Prepare changelog for v0.34.20-rc1. (#8966) * config: remove obsolete mempool v1 warning (#8987) * build(deps): Bump google.golang.org/grpc from 1.47.0 to 1.48.0 (#8991) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.47.0 to 1.48.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.47.0...v1.48.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sam Kleinman * mempool: ensure evicted transactions are removed from the cache (backport #9000) (#9004) This is a manual cherry-pick of commit b94470a6a42e8ffe7e7467521de5f51eb937c454. In the original implementation transactions evicted for priority were also removed from the cache. In addition, remove expired transactions from the cache. Related: - Add Has method to cache implementations. - Update tests to exercise this condition. * mempool: ensure async requests are flushed to the server (#9010) In the v0.34 line, the socket and gRPC clients require explicit flushes to ensure that the client and server have received an async request. Add these calls explicitly where required in the backport of the priority mempool. In addition, the gRPC client's flush plumbing was not fully hooked up in the v0.34 line, so this change includes that update as well. * Prepare changelog for Release v0.34.20 (#9032) * build(deps): Bump github.com/golangci/golangci-lint (#9036) Bumps [github.com/golangci/golangci-lint](https://github.com/golangci/golangci-lint) from 1.46.2 to 1.47.0. - [Release notes](https://github.com/golangci/golangci-lint/releases) - [Changelog](https://github.com/golangci/golangci-lint/blob/master/CHANGELOG.md) - [Commits](https://github.com/golangci/golangci-lint/compare/v1.46.2...v1.47.0) --- updated-dependencies: - dependency-name: github.com/golangci/golangci-lint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * mempool: rework lock discipline to mitigate callback deadlocks (backport #9030) (#9033) (manual cherry-pick of commit 22ed610083cb8275a954406296832149c4cc1dcd) * build(deps): Bump github.com/golangci/golangci-lint (#9043) * build(deps): Bump github.com/BurntSushi/toml from 1.1.0 to 1.2.0 (#9062) Bumps [github.com/BurntSushi/toml](https://github.com/BurntSushi/toml) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/BurntSushi/toml/releases) - [Commits](https://github.com/BurntSushi/toml/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: github.com/BurntSushi/toml dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): Bump github.com/golangci/golangci-lint (#9071) Bumps [github.com/golangci/golangci-lint](https://github.com/golangci/golangci-lint) from 1.47.1 to 1.47.2. - [Release notes](https://github.com/golangci/golangci-lint/releases) - [Changelog](https://github.com/golangci/golangci-lint/blob/master/CHANGELOG.md) - [Commits](https://github.com/golangci/golangci-lint/compare/v1.47.1...v1.47.2) --- updated-dependencies: - dependency-name: github.com/golangci/golangci-lint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * cli: add command to manually reindex tx and block events (backport: #6676) (#9034) * fix mockery generation script (#9094) (#9114) * build(deps): Bump github.com/bufbuild/buf from 1.4.0 to 1.7.0 (#9137) * backport: Fix unsafe-reset-all for working with default home (#9103) (#9113) * config: p2p.external-address (backport #9107) (#9153) * remove old proto workflow (#9167) * build(deps): Bump github.com/golangci/golangci-lint (#9188) Bumps [github.com/golangci/golangci-lint](https://github.com/golangci/golangci-lint) from 1.47.2 to 1.48.0. - [Release notes](https://github.com/golangci/golangci-lint/releases) - [Changelog](https://github.com/golangci/golangci-lint/blob/master/CHANGELOG.md) - [Commits](https://github.com/golangci/golangci-lint/compare/v1.47.2...v1.48.0) --- updated-dependencies: - dependency-name: github.com/golangci/golangci-lint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Backport of sam/abci-responses (#9090) (#9159) *backport of sam/abci-responses Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * Bump linter to 1.47 (#9218) *bump linter to 1.47 Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * chore: Bump go to 1.18 (#9212) * update to 1.18 Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * Small update to toml.go for abci-responses (#9232) * update to toml * update default (#9235) * Sync codeowners with main (#9245) * Update to ABCILastResponseskey (#9253) * update last responses key * cli: Enable reindex-event cmd (#9268) I noticed today that this wasn't enabled. Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson * spec: migrate v0.7.1. into v0.34 (#9262) * Initial commit * Add three timeouts and align pseudocode better with existing algorithm * Align protocol with Tendermint code and add find valid value mechanism * Prepare to Nuke Develop (#47) * state -> step * vote -> v * New version of the algorithm and the proof * New version of the algorithm and the proofs * Added algorithm description * Add algorithm description * Add introduction * Add conclusion * Add conclusion file * fix warnings (caption was defined twice) - only the latter is used anyways (centers captions) - this makes it possible to autom. building the paper * Update grammar * s/state_p/step_p * Address Ismail's comments * intro: language fixes * definitions: language fixes * consensus: various fixes * proof: some fixes * try to improve reviewability * \eq -> = * textwrap to 79 * various minor fixes * proof: fix itemization * proof: more minor fixes * proof: timeouts are functions * proof: fixes to lemma6 * Intro changes and improve title page * Add Marko and Ming to acks * add readme * Format algorithm correctly Clarify condition semantic and timeouts Improve descriptions * patform -> platform * Ensure that rules are mutually exclusive - various clarifications and small improvements * Release v0.6 * small nits for smoother readability * This PR is to create signed commits to be able to merge (#50) Signed-off-by: Marko Baricevic * Add consesnus and blockchain specs, (#52) - Open questions - Do we want to split lite client work from consesnsus - From the blockchain spec, is encoding nessecary in the spec Signed-off-by: Marko Baricevic * Add ABCI SPEC (#51) - move the abci spec from tendermint to spec repo Signed-off-by: Marko Baricevic * spec/consensus/signing: add more details about nil and amnesia (#54) - Add more details about nil votes and about amnesia attacks Signed-off-by: Marko Baricevic * Add Section for P2P (#53) * Add Section for P2P - moved over the section on p2p Signed-off-by: Marko Baricevic * add some more files Signed-off-by: Marko Baricevic * Fix model section * Add non-recursive specification of Bisection algorithm - Fix timing issues by introducing Delta parameter * spec: update spec with tendermint updates (#62) * spec: update spec with tendermint updates - this in preperation of deleting the spec folder in docs in tendermint/tendermint Signed-off-by: Marko Baricevic * spec: added in reactors & p2p Signed-off-by: Marko Baricevic * spec: update readme in spec to comply with docs site Signed-off-by: Marko Baricevic * docs: addded more changes from tednermint/tendermint Signed-off-by: Marko Baricevic * reflect breaking changes made to Commit (#63) * reflect breaking changes made to Commit PR: https://github.com/tendermint/tendermint/pull/4146 Issue: https://github.com/tendermint/tendermint/issues/1648 * types: rename Commit#Precommits to Signatures * update BlockIDFlagAbsent comment * remove iota * Clean up error conditions and simplify pseudocode * Apply suggestions from code review Co-Authored-By: Anca Zamfir * Add spec doc about unconditional_peer, persistent_peers_max_dial of ADR-050 (#68) * Add spec doc about unconditional_peer_ids, persistent_peers_max_dial_period of ADR-050 * Add indefinitely dialing condition * Add sr25519 amino documentation (#67) * sr25519 amino * Update spec/blockchain/encoding.md Co-Authored-By: Marko * some suggestions for pseuodocode changes * Improved error handling * Add explanation on difference between trusted models * Address reviewer's comments * Addressing reviewer's comments * Separating algorithm from proofs * Intermediate commit (aligning spec with the code) * Removing Store from API and providing end-to-end timing guarantees * Address reviewer comment's. Intermediate commit * light client dir and readmes * titles * add redirects * add diagram * detection TODO * fix image * update readme * Aligh the correctness arguments with the pseudocode changes * lite->light * Fix link in readme ./light -> ./light-client * p2p: Merlin based malleability fixes (#72) * Update the secret connection spec with the use of merlin to eliminte handshake malleability * Update spec/p2p/peer.md Co-Authored-By: Anton Kaliaev * Update spec/p2p/peer.md Co-Authored-By: Anton Kaliaev * Update spec/p2p/peer.md Co-Authored-By: Anton Kaliaev Co-authored-by: Anton Kaliaev * docs: update specs to remove cmn (#77) - cmn was remvoed in favor of sub pkgs. cmn.kvpair is now kv.pair Signed-off-by: Marko Baricevic * evidence: Add time to evidence params (#69) * evidence: Add time to evidence params - this pr is grouped together with https://github.com/tendermint/tendermint/pull/4254, once that PR is merged then this one can be as well. Signed-off-by: Marko Baricevic * remove note Signed-off-by: Marko Baricevic * Apply suggestions from code review Co-Authored-By: Anton Kaliaev Co-authored-by: Anton Kaliaev * update link to the pex reactor * add markdown link checker * changed tab spacing * removed folder-path flag * first attempt at fixing all links * second attempt at fixing all links * codeowners: add code owners (#82) * codeowners: add code owners - added some codeowners please comment if youd like to be added as well. Signed-off-by: Marko Baricevic * remove comment of repo maintainers * remove .idea dir (#83) Signed-off-by: Marko Baricevic * RFC-001: configurable block retention (#84) * Added RFC for truncated block history coordination * Clarified minimum block retention * Added hard checks on block retention and snapshot interval, and made some minor tweaks * Genesis parameters are immutable * Use local config for snapshot interval * Reordered parameter descriptions * Clarified local config option for snapshot-interval * rewrite for ABCI commit response * Renamed RFC * add block retention diagram * Removed retain_blocks table * fix image numbers * resolved open questions * image quality * accept RFC-001 (#86) * abci: add basic description of ABCI Commit.ResponseHeight (#85) Documentation for block pruning, once it's merged: tendermint/tendermint#4588. Minimum documentation, for now - we probably shouldn't encourage using this feature too much until we release state sync. * abci: add MaxAgeNumBlocks/MaxAgeDuration to EvidenceParams (#87) * abci: update MaxAgeNumBlocks & MaxAgeDuration docs (#88) * document state sync ABCI interface and P2P protocol (#90) The corresponding Tendermint PRs are tendermint/tendermint#4704 and tendermint/tendermint#4705. * Revert "document state sync ABCI interface and P2P protocol (#90)" (#92) This reverts commit 9842b4b0fb703e609ad233af9683adc773bc95ef. * blockchain: change validator set sorting method (#91) * abci: specify sorting of RequestInitChain.Validators * blockchain: change validator sorting method Refs https://github.com/tendermint/tendermint/issues/2478 * reactors/pex: specify hash function (#94) https://github.com/tendermint/tendermint/pull/4810/files * document state sync ABCI interface and P2P protocol (#93) * Revert "Revert "document state sync ABCI interface and P2P protocol (#90)" (#92)" This reverts commit 90797cef90da762db7f7a14ce834c745140f7bcd. * update with new enum case * fix links Co-authored-by: Erik Grinaker * Update evidence params with MaxNum (#95) evidence params now includes maxNum which is the maximum number of evidence that can be committed on a single block * reactors/pex: masked IP is used as group key (#96) * spec: add ProofTrialPeriod to EvidenceParam (#99) * spec: modify Header.LastResultsHash (#97) Refs: https://github.com/tendermint/tendermint/issues/1007 PR: https://github.com/tendermint/tendermint/pull/4845 * spec: link to abci server implementations (#100) * spec: update evidence in blockchain.md (#108) now evidence reflects the actual evidence present in the tendermint repo * abci: add AppVersion to ConsensusParams (#106) * abci: tweak node sync estimate (#115) * spec/abci: expand on Validator#Address (#118) Refs https://github.com/tendermint/tendermint/issues/3732 * blockchain: rename to core (#123) * blockchain: remove duplicate evidence sections (#124) * spec/consensus: canonical vs subjective commit Refs https://github.com/tendermint/tendermint/issues/2769 * Apply suggestions from code review Co-authored-by: Igor Konnov * update spec with the removal of phantom validator evidence (#126) * bring blockchain back * add correct links * spec: revert event hashing (#132) * Evidence time is sourced from block time (#138) * RFC-002: non-zero genesis (#119) * abci: add ResponseInitChain.app_hash (#140) * update hashing of empty inputs, and initial block LastResultsHash (#141) * update evidence verification (#139) * accept RFC-002 (#142) * add description of arbitrary initial height (#135) * update ResponseInitChain.app_hash description (#143) * remove unused directories and update README (#145) This change removes unused directories (`papers` and `research`) and updates the README to reflect our strategy for merging the informalsystems/tendermint-rs specs into this repository. Partially addresses #121. * ci: add markdown linter (#146) * ci: add dependabot config (#148) * build(deps): bump gaurav-nelson/github-action-markdown-link-check from 0.6.0 to 1.0.7 (#149) Bumps [gaurav-nelson/github-action-markdown-link-check](https://github.com/gaurav-nelson/github-action-markdown-link-check) from 0.6.0 to 1.0.7. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * docs: add sections to abci (#150) * spec: update abci events (#151) * spec: extract light-client to its own directory (#152) Co-authored-by: Callum Waters * spec: remove evidences (#153) * add a stale bot (#134) * Current versions of light client specs from tendermint-rs (#158) * current versions of light client specs from tendermint-rs * markdown lint * linting * links * links * links Co-authored-by: Marko Baricevic * Fastsync spec from tendermint-rs (#157) * fastsync spec from tendermint-rs * fixed broken link * fixed linting * more fixes * markdown lint * move fast_sync to rust-spec Co-authored-by: Marko Baricevic * Update README.md (#160) * spec/reactors/mempool: batch txs per peer (#155) * spec/reactors/mempool: batch txs per peer Refs https://github.com/tendermint/tendermint/issues/625 * update * spec: Light client attack detector (#164) * start with new detection and evidence spec * more definitions at top * sketch of functions * pre post draft * evidence proof * typo * evidence theory polished * some TODOs resolved * more TODOs * links * second to last revision before PR * links * I will read once more and then make a PR * removed peer handling definitions * secondary * ready to review * detector ready for review * Update rust-spec/lightclient/detection/detection.md Co-authored-by: Zarko Milosevic * Update rust-spec/lightclient/detection/detection.md Co-authored-by: Zarko Milosevic * Update rust-spec/lightclient/detection/detection.md Co-authored-by: Zarko Milosevic * Update rust-spec/lightclient/detection/detection.md Co-authored-by: Zarko Milosevic * Update rust-spec/lightclient/detection/detection.md Co-authored-by: Zarko Milosevic * Update rust-spec/lightclient/detection/detection.md Co-authored-by: Zarko Milosevic * Update rust-spec/lightclient/detection/detection.md * skip-trace * PossibleCommit explained * Update rust-spec/lightclient/detection/detection.md Co-authored-by: Zarko Milosevic * comments by Zarko * renamed and changed link in README Co-authored-by: Zarko Milosevic * fixed an overlooked conflict (#167) * describe valset sorting according to v0.34 requirements (#169) * evidence: update data structures (#165) * fix markdown linter (#172) * TLA+ specs from MBT revision (#173) * remove setOption (#181) * spec: protobuf changes (#156) Co-authored-by: Anton Kaliaev * first check latest with secondary (#184) * Extending the blockchain specification (in the light client) to produce different ratios of faults (#183) * cleaning unused definitions * introduced the ratio of faulty processes * Update README.md (#185) * build(deps): bump gaurav-nelson/github-action-markdown-link-check from 1.0.7 to 1.0.8 (#188) Bumps [gaurav-nelson/github-action-markdown-link-check](https://github.com/gaurav-nelson/github-action-markdown-link-check) from 1.0.7 to 1.0.8. - [Release notes](https://github.com/gaurav-nelson/github-action-markdown-link-check/releases) - [Commits](https://github.com/gaurav-nelson/github-action-markdown-link-check/compare/1.0.7...e3c371c731b2f494f856dc5de7f61cea4d519907) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * spec: update light client verification to match supervisor (#171) * VDD renaming of verification spec + links fixed * latest() * backwards * added TODOs * link in old file to new name * better text * revision done. needs one more round of reading * renamed constants in 001 according to TLA+ and impl * ready for PR * forgot linting * Update rust-spec/lightclient/verification/verification_002_draft.md * Update rust-spec/lightclient/verification/verification_002_draft.md * added lightstore function needed for supervisor * added lightstore functions for supervisor * ident * Update rust-spec/lightclient/verification/verification_002_draft.md * github: issue template for proposals (#190) * Sequential Supervisor (#186) * move from tendermint-rs but needs discussion * markdown lint * TODO links replaced * links * links * links lint * Update rust-spec/lightclient/supervisor/supervisor.md * Update rust-spec/lightclient/supervisor/supervisor.md * Update rust-spec/lightclient/supervisor/supervisor.md * Update rust-spec/lightclient/supervisor/supervisor.md * moved peer handling definitions to supervisor * polishing * rename * Update rust-spec/lightclient/supervisor/supervisor_001_draft.md * Update rust-spec/lightclient/supervisor/supervisor_001_draft.md * changes to maintain StateVerified again * ready for changes in verification * start of supervisor * module name * fixed * more details * supevisor completed. Now I have to add function to verification * ready for review * tla comment * removed issues * Update rust-spec/lightclient/supervisor/supervisor_001_draft.md * intro text fixed * indentation * Update rust-spec/lightclient/supervisor/supervisor_001_draft.md * comment to entry points Co-authored-by: Marko Baricevic * RFC: adopt zip 215 (#144) Co-authored-by: Robert Zaremba * Core: move validation & data structures together (#176) Co-authored-by: Callum Waters * docs: make blockchain not viewable (#211) * evidence: update data structures to reflect added support of abci evidence (#213) * encoding: add secp, ref zip215, tables (#212) * Detector English Spec ready (#215) Add detector English spec * add Ivy proofs (#210) * add Ivy proofs * fix docker-compose command * Light client detector spec in TLA+ and refactoring of light client verification TLA+ spec (#216) Add light client detector spec in TLA+ * abci: lastcommitinfo.round extra sentence (#221) * abci: add abci_version to requestInfo (#223) * BFT requires _less than_ 1/3 faulty validators (#228) Thanks fo spotting the imprecision in the text, @shahankhatch ! * Draft of evidence handling for discussion (#225) * start with accountability deliverable * problem statement * draft function * quite complete draft. ready to discuss with Igor * Update isolate-attackers_001_draft.md * Update isolate-attackers_001_draft.md * Update isolate-attackers_001_draft.md * Update isolate-attackers_001_draft.md * Update isolate-attackers_001_draft.md * ready for TLA+ to take over * isolate * isolateamnesiatodos * Update isolate-attackers_001_draft.md * Update rust-spec/lightclient/attacks/isolate-attackers_001_draft.md Co-authored-by: Igor Konnov * Update rust-spec/lightclient/attacks/isolate-attackers_001_draft.md Co-authored-by: Igor Konnov * Update rust-spec/lightclient/attacks/isolate-attackers_001_draft.md Co-authored-by: Igor Konnov * Update rust-spec/lightclient/attacks/isolate-attackers_001_draft.md Co-authored-by: Igor Konnov * Update rust-spec/lightclient/attacks/isolate-attackers_001_draft.md Co-authored-by: Igor Konnov * Update rust-spec/lightclient/attacks/isolate-attackers_001_draft.md Co-authored-by: Igor Konnov * Update rust-spec/lightclient/attacks/isolate-attackers_001_draft.md Co-authored-by: Igor Konnov * Update rust-spec/lightclient/attacks/isolate-attackers_001_draft.md Co-authored-by: Igor Konnov * The TLA+ specification of the attackers detection (#231) * the working attackers isolation spec, needs more comments * the TLA+ spec of the attackers isolation * build(deps): bump gaurav-nelson/github-action-markdown-link-check (#233) Bumps [gaurav-nelson/github-action-markdown-link-check](https://github.com/gaurav-nelson/github-action-markdown-link-check) from 1.0.8 to 1.0.11. - [Release notes](https://github.com/gaurav-nelson/github-action-markdown-link-check/releases) - [Commits](https://github.com/gaurav-nelson/github-action-markdown-link-check/compare/1.0.8...2a60e0fe41b5361f446ccace6621a1a2a5c324cf) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Computing attack types (#232) Add light attack evidence handling * Update README.md (#234) * p2p: update frame size (#235) Reflect the change made in https://github.com/tendermint/tendermint/pull/5805 The MTU (Maximum Transmission Unit) for Ethernet is 1500 bytes. The IP header and the TCP header take up 20 bytes each at least (unless optional header fields are used) and thus the max for (non-Jumbo frame) Ethernet is 1500 - 20 -20 = 1460 Source: https://stackoverflow.com/a/3074427/820520 * build(deps): bump gaurav-nelson/github-action-markdown-link-check (#239) Bumps [gaurav-nelson/github-action-markdown-link-check](https://github.com/gaurav-nelson/github-action-markdown-link-check) from 1.0.11 to 1.0.12. - [Release notes](https://github.com/gaurav-nelson/github-action-markdown-link-check/releases) - [Commits](https://github.com/gaurav-nelson/github-action-markdown-link-check/compare/1.0.11...0fe4911067fa322422f325b002d2038ba5602170) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * layout: add section titles (#240) * reactors: remove bcv1 (#241) * abci: rewrite to proto interface (#237) * Update supervisor_001_draft.md (#243) * spec: remove reactor section (#242) Co-authored-by: Tess Rinearson * non-critical bugfix in the TLA+ spec (found by new version of apalache) (#244) * params: remove block timeiota (#248) * proto: add files (#246) Co-authored-by: Erik Grinaker * proto: modify height int64 to uint64 (#253) * abci: note on concurrency (#258) Co-authored-by: Marko * spec: merge rust-spec (#252) * Fix list of RFCs (#266) * readme: cleanup (#262) * modify readme * add rfc and proto * add rust=spec back to avoid breakage * lint readme * genesis: Explain fields in genesis file (#270) * describe the genesis * Update spec/core/genesis.md Co-authored-by: Dev Ojha * Apply suggestions from code review Co-authored-by: Callum Waters * add wording on app_state * Update spec/core/genesis.md Co-authored-by: Callum Waters Co-authored-by: Dev Ojha Co-authored-by: Callum Waters * p2p: links (#268) * fix links * fix more links * Proposer-based timestamp specification (#261) * added proposer-based timestamp spec * Update spec/consensus/proposer-based-timestamp/pbts_001_draft.md Co-authored-by: Aleksandr Bezobchuk * Update spec/consensus/proposer-based-timestamp/pbts_001_draft.md Co-authored-by: Aleksandr Bezobchuk * Update spec/consensus/proposer-based-timestamp/pbts-algorithm_001_draft.md Co-authored-by: Marko * Update spec/consensus/proposer-based-timestamp/pbts-algorithm_001_draft.md * Update spec/consensus/proposer-based-timestamp/pbts-sysmodel_001_draft.md Co-authored-by: Callum Waters * fixes from PR Co-authored-by: Josef Widder <44643235+josef-widder@users.noreply.github.com> Co-authored-by: Aleksandr Bezobchuk Co-authored-by: Marko Co-authored-by: Callum Waters * abci: reorder sidebar (#282) * ABCI++ RFC (#254) * ABCI++ RFC This commit adds an RFC for ABCI++, which is a collection of three new phases of communication between the consensus engine and the application. Co-authored-by: Sunny Aggarwal * Fix bugs pointed out by @liamsi * Update rfc/004-abci++.md Co-authored-by: Federico Kunze <31522760+fedekunze@users.noreply.github.com> * Fix markdown lints * Update rfc/004-abci++.md Co-authored-by: Ismail Khoffi * Update rfc/004-abci++.md Co-authored-by: Tess Rinearson * Update rfc/004-abci++.md Co-authored-by: Tess Rinearson * Add information about the rename in the context section * Bold RFC * Add example for self-authenticating vote data * More exposition of the term IPC * Update pros / negatives * Fix sentence fragment * Add desc for no-ops Co-authored-by: Sunny Aggarwal Co-authored-by: Federico Kunze <31522760+fedekunze@users.noreply.github.com> Co-authored-by: Ismail Khoffi Co-authored-by: Tess Rinearson * RFC: ReverseSync - fetching historical data (#224) * core: update a few sections (#284) * p2p: update state sync messages for reverse sync (#285) * Update README.md (#286) * rpc: define spec for RPC (#276) * add rpc spec and support outline * add json * add more routes remove unneeded ones * add rest of rpc endpoints * add jsonrpc calls * add more jsonrpc calls * fix blockchain * cleanup unused links and add links to repos * Update spec/rpc/README.md Co-authored-by: Callum Waters * add missing param from consensus param * Update spec/rpc/README.md Co-authored-by: Callum Waters * Update spec/rpc/README.md Co-authored-by: Callum Waters * fix cast and add doc to readme Co-authored-by: Callum Waters Co-authored-by: Marko Baricevic * A few improvements to the Ivy proof (#288) * Avoid quantifier alternation cycle The problematic quantifier alternation cycle arose because the definition of accountability_violation was unfolded. This commit also restructures the induction proof for clarity. * add count_lines.sh * fix typo and add forgotten complete=fo in comment Co-authored-by: Giuliano * Fixed a broken link (#291) * fix message type for block-sync (#298) * lint: fix lint errors (#301) * build(deps): bump actions/stale from 3 to 3.0.18 (#300) Bumps [actions/stale](https://github.com/actions/stale) from 3 to 3.0.18. - [Release notes](https://github.com/actions/stale/releases) - [Commits](https://github.com/actions/stale/compare/v3...v3.0.18) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): bump actions/stale from 3.0.18 to 3.0.19 (#302) Bumps [actions/stale](https://github.com/actions/stale) from 3.0.18 to 3.0.19. - [Release notes](https://github.com/actions/stale/releases) - [Commits](https://github.com/actions/stale/compare/v3.0.18...v3.0.19) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * rename HasVote to ReceivedVote (#289) * add a changelog to track changes (#303) * add a changelog to track changes * Update CHANGELOG.md Co-authored-by: Callum Waters Co-authored-by: Callum Waters * rpc: clarify timestamps (#304) * clarify timestamps * changelog entry * Update spec/rpc/README.md Co-authored-by: Callum Waters Co-authored-by: Callum Waters * rpc: add chunked genesis endpoint (#299) * rpc: add chunked genesis endpoint * fix lint * feedback * add info about error * fix lint Co-authored-by: marbar3778 * update ResponseCheckTx (#306) * rpc: Add totalGasUSed to block_results response (#308) * Add C++ code generation and test scenario (#310) * add parameters to byzantine send action * make net not trusted it's not necessary since for proofs Ivy will assume that the environment does not break action preconditions * use require instead of assume it seems that assume is not checked when other isolates call! * add comment * add comment * run with random seed * make domain model extractable to C++ * substitute require for assume assumes in an action are not checked when the action is called! I.e. they place no requirement on the caller; we're just assuming that the caller is going to do the right thing. This wasn't very important here but it leade to a minor inconsistency slipping through. * make the net isolate not trusted there was no need for it * add tendermint_test.ivy contains a simple test scenario that show that the specification is no vacuuous * update comment * add comments * throw if trying to parse nset value in the repl * add comment * minor refactoring * add new pex messages (#312) * build(deps): bump gaurav-nelson/github-action-markdown-link-check (#313) Bumps [gaurav-nelson/github-action-markdown-link-check](https://github.com/gaurav-nelson/github-action-markdown-link-check) from 1.0.12 to 1.0.13. - [Release notes](https://github.com/gaurav-nelson/github-action-markdown-link-check/releases) - [Commits](https://github.com/gaurav-nelson/github-action-markdown-link-check/compare/1.0.12...1.0.13) --- updated-dependencies: - dependency-name: gaurav-nelson/github-action-markdown-link-check dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * update spec to reference currently used timestamp type (#317) * build(deps): bump actions/stale from 3.0.19 to 4 (#319) Bumps [actions/stale](https://github.com/actions/stale) from 3.0.19 to 4. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v3.0.19...v4) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * address discrepancies between spec and implementation (#322) * update proto files for release (#318) * stale bot: ignore issues (#325) * evidence: add section explaining evidence (#324) * statesync: new messages for gossiping consensus params (#328) * rpc: update peer format in specification in NetInfo operation (#331) * Update supervisor_001_draft.md (#334) * core: text cleanup (#332) * abci: clarify what abci stands for (#336) * abci: clarify what abci stands for * link to abci type protos. * abci: clarify connection use in-process (#337) * abci: clarify connection use in-process * Update abci.md * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * invert abci explanations * lint++ * lint++ * lint++ * lint++ Co-authored-by: M. J. Fromberger * proto: move proto files under the correct directory related to their package name (#344) * abci.md fixup (#339) * abci: points of clarification ahead of v0.1.0 * lint++ * typo * lint++ * double word score * grammar * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * Update spec/abci/abci.md Co-authored-by: M. J. Fromberger * pr feedback * wip * update non-zero status code docs * fix event description * update CheckTx description Co-authored-by: M. J. Fromberger * Update supervisor_001_draft.md (#333) * Update supervisor_001_draft.md If the only node in the *FullNodes* set is the primary, that was just deemed faulty, we can't find honest primary. * Update supervisor_001_draft.md * light: update initialization description (#320) * apps.md fixups (#341) * wip * wip * wip * remove comments in favor of gh comments * wip * udpates to language, should must etc * Apply suggestions from code review Co-authored-by: M. J. Fromberger * remove tendermint cache description Co-authored-by: M. J. Fromberger * proto: add tendermint go changes (#349) * add missed proto files * add abci changes * rename blockchain to blocksync * Update proto/tendermint/abci/types.proto Co-authored-by: Callum Waters Co-authored-by: Callum Waters * fix mockery generation script (#9094) Signed-off-by: Marko Baricevic Co-authored-by: Milosevic, Zarko Co-authored-by: Milosevic, Zarko Co-authored-by: Zarko Milosevic Co-authored-by: Marko Co-authored-by: Zarko Milosevic Co-authored-by: Anton Kaliaev Co-authored-by: Anca Zamfir Co-authored-by: dongsamb Co-authored-by: Sunny Aggarwal Co-authored-by: Anca Zamfir Co-authored-by: Ethan Buchman Co-authored-by: Zarko Milosevic Co-authored-by: Ismail Khoffi Co-authored-by: Zaki Manian Co-authored-by: Erik Grinaker Co-authored-by: Tess Rinearson Co-authored-by: Alexander Simmerl Co-authored-by: Igor Konnov Co-authored-by: Sean Braithwaite Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Josef Widder <44643235+josef-widder@users.noreply.github.com> Co-authored-by: Andrey Kuprianov <59489470+andrey-kuprianov@users.noreply.github.com> Co-authored-by: Igor Konnov Co-authored-by: Sam Hart Co-authored-by: Robert Zaremba Co-authored-by: Giuliano Co-authored-by: Shahan Khatchadourian Co-authored-by: Dev Ojha Co-authored-by: istoilkovska Co-authored-by: Aleksandr Bezobchuk Co-authored-by: Sam Kleinman Co-authored-by: Sunny Aggarwal Co-authored-by: Federico Kunze <31522760+fedekunze@users.noreply.github.com> Co-authored-by: Marko Baricevic Co-authored-by: Giuliano Co-authored-by: Jordan Sexton Co-authored-by: MengXiangJian <805442788@qq.com> Co-authored-by: Yixin Luo <18810541851@163.com> Co-authored-by: crypto-facs <84574577+crypto-facs@users.noreply.github.com> Co-authored-by: Giuliano Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: Mateusz Górski Co-authored-by: M. J. Fromberger Co-authored-by: Thane Thomson * config: Move `discard_abci_responses` flag into its own storage section (#9275) * config: Move discard_abci_responses flag into its own storage section Signed-off-by: Thane Thomson * Update config comment to highlight space saving tradeoff Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson * docs: Minor recommendations prior to v0.34.21 release (#9267) * Make reindex-event cmd docs consistent with other commands Signed-off-by: Thane Thomson * Add warning regarding DiscardABCIResponses to BlockResults Go API Signed-off-by: Thane Thomson * Update OpenAPI spec to reflect discard_abci_responses change Signed-off-by: Thane Thomson * Add release highlights to CHANGELOG_PENDING Signed-off-by: Thane Thomson * Add pending changelog entry for #9033 Signed-off-by: Thane Thomson * Format pending changelog entries consistently Signed-off-by: Thane Thomson * Correct and simplify comment wording Signed-off-by: Thane Thomson * Remove changelog entry regarding storage section Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson * release: prepare v0.34.21 (#9285) * docs: Update v0.34.x to prepare for v0.37 (#9244) * Ignore generated/copied RPC docs Signed-off-by: Thane Thomson * Sync vuepress config with main Signed-off-by: Thane Thomson * Sync docs package-lock.json with main Signed-off-by: Thane Thomson * Sync docs redirects with main Signed-off-by: Thane Thomson * Sync docs versions with main Signed-off-by: Thane Thomson * Update OpenAPI version to v0.34 Signed-off-by: Thane Thomson * Sync DOCS_README with main Signed-off-by: Thane Thomson * Update all v0.34.x docs references from master to main Signed-off-by: Thane Thomson * Update v0.34 OpenAPI references from master to main Signed-off-by: Thane Thomson * Update repo doc links from master to main Signed-off-by: Thane Thomson * Update code comment references from master to main Signed-off-by: Thane Thomson * Update repo root doc links from master to main Signed-off-by: Thane Thomson * Update repo root doc links for docs.tendermint.com from master to main Signed-off-by: Thane Thomson * Build v0.34.x as "latest" Signed-off-by: Thane Thomson * Explicitly mark v0.34 docs as latest in version selector Signed-off-by: Thane Thomson * Add nav link to main and clearly mark as unstable Signed-off-by: Thane Thomson * Direct all docs.tendermint.com links to v0.34 on v0.34.x Signed-off-by: Thane Thomson * Update all relevant links on v0.34.x branch to be v0.34-specific Signed-off-by: Thane Thomson * Update changelog refs to docs.tendermint.com Signed-off-by: Thane Thomson * Update remaining GH master link to main Signed-off-by: Thane Thomson * Sync docs build and nav config with main Signed-off-by: Thane Thomson * Migrate spec links to GitHub repo from docs site Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson * build(deps): Bump google.golang.org/grpc from 1.48.0 to 1.49.0 (#9320) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.48.0 to 1.49.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.48.0...v1.49.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * test: add the loadtime tool (Backport #9342) (#9358) * test: add the loadtime tool (#9342) This pull request adds the loadtime tool. This tool leverages the tm-load-test framework. Using the framework means that the only real logic that needs to be written is the logic for Tx generation. The framework does the rest. The tool writes a set of metadata into the transaction, including the current transaction rate, number of connections, specified size of the transaction, and the current time. * lint * test: add the loadtime report tool (backport #9351) (#9365) * test: add the loadtime report tool (#9351) This pull request adds the report tool and modifies the loadtime libraries to better support its use. (cherry picked from commit 8655080a0ff26c001025b4c1af009f39c90cbb9e) * add nolint Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: William Banfield * build(deps): Bump google.golang.org/protobuf from 1.28.0 to 1.28.1 (#9362) Bumps [google.golang.org/protobuf](https://github.com/protocolbuffers/protobuf-go) from 1.28.0 to 1.28.1. - [Release notes](https://github.com/protocolbuffers/protobuf-go/releases) - [Changelog](https://github.com/protocolbuffers/protobuf-go/blob/master/release.bash) - [Commits](https://github.com/protocolbuffers/protobuf-go/compare/v1.28.0...v1.28.1) --- updated-dependencies: - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sam Kleinman * build(deps): Bump github.com/golangci/golangci-lint (#9363) Bumps [github.com/golangci/golangci-lint](https://github.com/golangci/golangci-lint) from 1.42.1 to 1.49.0. - [Release notes](https://github.com/golangci/golangci-lint/releases) - [Changelog](https://github.com/golangci/golangci-lint/blob/master/CHANGELOG.md) - [Commits](https://github.com/golangci/golangci-lint/compare/v1.42.1...v1.49.0) --- updated-dependencies: - dependency-name: github.com/golangci/golangci-lint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sam Kleinman Co-authored-by: Thane Thomson * ci: Remove "(WARNING: BETA SOFTWARE)" tagline from all upcoming releases (backport #9371) (#9373) * ci: Remove "(WARNING: BETA SOFTWARE)" tagline from all upcoming releases (#9371) This is by no means a signal that we offer any additional guarantees with our software. This warning seems somewhat pointless given that: 1. Our open source license clearly states that we offer no warranties with this software. 2. We are clearly still pre-1.0. It also doesn't make sense to append "(WARNING: BETA SOFTWARE)" to pre-releases such as alpha releases, which are to be considered _more_ unstable than beta releases. --- #### PR checklist - [x] Tests written/updated, or no tests needed - [x] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [x] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed (cherry picked from commit d7645628f13031746be27507d1673e132cf1f6e6) # Conflicts: # .goreleaser.yml * Resolve conflicts Signed-off-by: Thane Thomson * Sync root docs with main Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson Co-authored-by: Thane Thomson * add separated runs by UUID (backport #9367) (#9380) * add separated runs by UUID (#9367) This _should_ be the last piece needed for this tool. This allows the tool to generate reports on multiple experimental runs that may have been performed against the same chain. The `load` tool has been updated to generate a `UUID` on startup to uniquely identify each experimental run. The `report` tool separates all of the results it reads by `UUID` and performs separate calculations for each discovered experiment. Sample output is as follows ``` Experiment ID: 6bd7d1e8-d82c-4dbe-a1b3-40ab99e4fa30 Connections: 1 Rate: 1000 Size: 1024 Total Valid Tx: 9000 Total Negative Latencies: 0 Minimum Latency: 86.632837ms Maximum Latency: 1.151089602s Average Latency: 813.759361ms Standard Deviation: 225.189977ms Experiment ID: 453960af-6295-4282-aed6-367fc17c0de0 Connections: 1 Rate: 1000 Size: 1024 Total Valid Tx: 9000 Total Negative Latencies: 0 Minimum Latency: 79.312992ms Maximum Latency: 1.162446243s Average Latency: 422.755139ms Standard Deviation: 241.832475ms Total Invalid Tx: 0 ``` closes: #9352 #### PR checklist - [ ] Tests written/updated, or no tests needed - [ ] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [ ] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed (cherry picked from commit 1067ba15719b89a74c89bcbec065062d2d0159d8) # Conflicts: # go.mod * fix merge conflict * fix lint Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: William Banfield * print all versions of tendermint and its sub protocols (#9329) (#9387) (cherry picked from commit ffce25327386bdc8d311a47609b7722146b0e91d) Co-authored-by: Marko * Add redirect link for tutorial (backport #9385) (#9390) * add redirect links (#9385) (cherry picked from commit 43ebbed9c299324a54eb5d562cf6f50a6d2cf65f) # Conflicts: # docs/.vuepress/redirects * Fix conflicts Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson Co-authored-by: samricotta <37125168+samricotta@users.noreply.github.com> Co-authored-by: Thane Thomson * test: generate uuid on startup for load tool (#9383) (#9393) the `NewClient` method is called by the load test framework for each connection. This means that if multiple connections are instantiated, each connection will erroneously have its own UUID. This PR changes the UUID generation to happen at the _beginning_ of the script instead of on client creation so that each experimental run shares a UUID. Caught while preparing the script for production readiness. #### PR checklist - [ ] Tests written/updated, or no tests needed - [ ] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [ ] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed (cherry picked from commit 59a711eabe90e91f91d61bc7b8fc0fab8b88d89c) Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * build(deps): Bump github.com/spf13/viper from 1.12.0 to 1.13.0 (#9402) * feat: support HTTPS inside websocket (backport #9416) (#9423) * state: restore previous error message (#9435) (#9441) * build(deps): Bump github.com/bufbuild/buf from 1.7.0 to 1.8.0 (#9448) Bumps [github.com/bufbuild/buf](https://github.com/bufbuild/buf) from 1.7.0 to 1.8.0. - [Release notes](https://github.com/bufbuild/buf/releases) - [Changelog](https://github.com/bufbuild/buf/blob/main/CHANGELOG.md) - [Commits](https://github.com/bufbuild/buf/compare/v1.7.0...v1.8.0) --- updated-dependencies: - dependency-name: github.com/bufbuild/buf dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): Bump bufbuild/buf-setup-action from 1.6.0 to 1.8.0 (#9449) Bumps [bufbuild/buf-setup-action](https://github.com/bufbuild/buf-setup-action) from 1.6.0 to 1.8.0. - [Release notes](https://github.com/bufbuild/buf-setup-action/releases) - [Commits](https://github.com/bufbuild/buf-setup-action/compare/v1.6.0...v1.8.0) --- updated-dependencies: - dependency-name: bufbuild/buf-setup-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thane Thomson * Sync Vote.Verify() in spec with implementation (#9466) (#9477) * config: Add missing storage section when generating config (backport #9483) (#9488) * config: Add missing storage section when generating config (#9483) (cherry picked from commit b7f1e1f218ffea7a164608037bc8e0bc8f57b37c) * Add pending changelog entry Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson Co-authored-by: Thane Thomson * loadtime: add block time to the data point (backport #9484) (#9490) * loadtime: add block time to the data point (#9484) This pull request adds the block time as the unix time since the epoch to the `report` tool's csv output. ```csv ... a7a8b903-1136-4da1-97aa-d25da7b4094f,1614226790,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1614196724,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1613097336,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1609365168,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1617199169,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1615197134,1663707084905417366,4,200,1024 a7a8b903-1136-4da1-97aa-d25da7b4094f,1610399447,1663707084905417366,4,200,1024 ... ``` #### PR checklist - [ ] Tests written/updated, or no tests needed - [ ] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [ ] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed (cherry picked from commit 5fe1a72416722f8045b863fa0c7c045de583b6a1) * lint fix Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: William Banfield * Update apps.md (#9461) typo: "later" to "latter" * build(deps): Bump actions/stale from 5 to 6 (#9492) Bumps [actions/stale](https://github.com/actions/stale) from 5 to 6. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Extend the load report tool to include transactions' hashes (backport #9509) (#9514) * Extend the load report tool to include transactions' hashes (#9509) * Add transaction hash to raw data * Add hash in formatted output * Cosmetic (cherry picked from commit cdd3479f20b15c7dab0c683e2d5dddb7e7b95721) # Conflicts: # test/loadtime/cmd/report/main.go * Resolve conflict * Appease linter Co-authored-by: Sergio Mena * security/p2p: prevent peers who errored being added to the peer_set (backport #9500) (#9516) * security/p2p: prevent peers who errored being added to the peer_set (#9500) * Mark failed removal of peer to address security bug Co-authored-by: Callum Waters (cherry picked from commit c0bdb2423acef508372a3750d0db3e0dd9982178) * Changelong entry and added missing functions for implementations of Peer Co-authored-by: Jasmina Malicevic * indexer: move deduplication functionality purely to the kvindexer (backport #9473) (#9521) * blocksync: retry requests after timeout (backport #9518) (#9534) * blocksync: retry requests after timeout (#9518) * blocksync: retry requests after timeout * Minimize changes to re-send block request after timeout * TO REVERT: reduce queue capacity * Add reset * Revert "TO REVERT: reduce queue capacity" This reverts commit dd0fee56924c958bed2ab7733e1917eb88fb5957. * 30 seconds * don't reset the timer * Update blocksync/pool.go Co-authored-by: Callum Waters Co-authored-by: Sergio Mena Co-authored-by: Callum Waters (cherry picked from commit a371b1e3a8ea7603ada20e21bd6b4d5bf9f664f2) * Add changelog entry Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: Sergio Mena * build(deps): Bump google.golang.org/grpc from 1.49.0 to 1.50.0 (#9527) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.49.0 to 1.50.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.49.0...v1.50.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Fix TX payload for DO testnets (#9540) (#9543) * Added print * Fix unmarshall * Fix unmarshalling * Simplified steps to unmarshall * minor * Use 'encoding/hex' * Forget about C, this is Go! * gosec warning * Set maximum payload size * nosec annotation (cherry picked from commit b42c439776811a6fcab8e22fb97fb44cee5701b3) Co-authored-by: Sergio Mena * QA Process report for v0.37.x (and baseline for v0.34.x) (backport #9499) (#9578) * QA Process report for v0.37.x (and baseline for v0.34.x) (#9499) * 1st version. 200 nodes. Missing rotating node * Small fixes * Addressed @jmalicevic's comment * Explain in method how to set the tmint version to test. Improve result section * 1st version of how to run the 'rotating node' testnet * Apply suggestions from @williambanfield Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * Addressed @williambanfield's comments * Added reference to Unix load metric * Added total TXs * Fixed some 'png's that got swapped. Excluded '.*-node-exporter' processes from memory plots * Report for rotating node * Adressed remaining comments from @williambanfield * Cosmetic * Addressed some of @thanethomson's comments * Re-executed the 200 node tests and updated the corresponding sections of the report * Ignore Python virtualenv directories Signed-off-by: Thane Thomson * Add latency vs throughput script Signed-off-by: Thane Thomson * Add README for latency vs throughput script Signed-off-by: Thane Thomson * Fix local links to folders Signed-off-by: Thane Thomson * v034: only have one level-1 heading Signed-off-by: Thane Thomson * Adjust headings Signed-off-by: Thane Thomson * v0.37.x: add links to issues/PRs Signed-off-by: Thane Thomson * v0.37.x: add note about bug being present in v0.34 Signed-off-by: Thane Thomson * method: adjust heading depths Signed-off-by: Thane Thomson * Show data points on latency vs throughput plot Signed-off-by: Thane Thomson * Add latency vs throughput plots Signed-off-by: Thane Thomson * Correct mentioning of v0.34.21 and add heading Signed-off-by: Thane Thomson * Refactor latency vs throughput script Update the latency vs throughput script to rather generate plots from the "raw" CSV output from the loadtime reporting tool as opposed to the separated CSV files from the experimental method. Also update the relevant documentation, and regenerate the images from the raw CSV data (resulting in pretty much the same plots as the previous ones). Signed-off-by: Thane Thomson * Remove unused default duration const Signed-off-by: Thane Thomson * Adjust experiment start time to be more accurate and re-plot latency vs throughput Signed-off-by: Thane Thomson * Addressed @williambanfield's comments * Apply suggestions from code review Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> * scripts: Update latency vs throughput readme for clarity Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: Thane Thomson (cherry picked from commit b06e1cea5495dc4557d805dcc433a0f771c0fc1c) * Remove v037 dir * Removed reference to v0.37 testnets Co-authored-by: Sergio Mena * fix: header link (backport #9574) (#9585) * fix: header link (#9574) Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> (cherry picked from commit c8f203293de1ba6f5f6886ccccc277a64ccd5048) # Conflicts: # spec/core/data_structures.md * fix conflict Co-authored-by: Rootul P Co-authored-by: William Banfield * Release v0.34.22 (#9583) * Add changelog entry for v0.34.22 Signed-off-by: Thane Thomson * Prepare pending changelog for v0.34.23 Signed-off-by: Thane Thomson * Bump version to v0.34.22 Signed-off-by: Thane Thomson * Reword changelog entry regarding block sync issue Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson * ci: Fix linter complaint (backport #9645) (#9647) * ci: Fix linter complaint (#9645) Fixes a very silly linter complaint that makes absolutely no sense and is blocking the merging of several PRs. --- #### PR checklist - [x] Tests written/updated, or no tests needed - [x] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [x] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed (cherry picked from commit 83b7f4ad5bc77a56fdb51ec39ae13c1204dde6e0) # Conflicts: # .github/workflows/lint.yml # .golangci.yml # cmd/tendermint/commands/debug/util.go * Resolve conflicts Signed-off-by: Thane Thomson * ci: Sync golangci-lint config with main Minus the spelling configuration that restricts spelling to US English only. Signed-off-by: Thane Thomson * make format Signed-off-by: Thane Thomson * Remove usage of deprecated io/ioutil package Signed-off-by: Thane Thomson * Remove unused mockBlockStore Signed-off-by: Thane Thomson * blockchain/v2: Remove unused method Signed-off-by: Thane Thomson * Bulk fix lints Signed-off-by: Thane Thomson * lint: Ignore auto-generated query PEG Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson Co-authored-by: Thane Thomson * ci: Remove unused Markdown link checker from v0.34.x branch (#9643) Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson * p2p: add a per-message type send and receive metric (backport #9622) (#9641) * p2p: add a per-message type send and receive metric (#9622) * p2p: ressurrect the p2p envelope and use to calculate message metric Add new SendEnvelope, TrySendEnvelope, BroadcastEnvelope, and ReceiveEnvelope methods in the p2p package to work with the new envelope type. Care was taken to ensure this was performed in a non-breaking manner. Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: William Banfield * e2e: configurable IP addresses for e2e testnet generator (backport #9592) (#9623) * e2e: configurable IP addresses for e2e testnet generator (backport #9592) * resurrect 'misbehavior' * Removes space in hyperlink (#9653) (#9664) Simple formatting issue. --- #### PR checklist - [x] Tests written/updated, or no tests needed - [x] `CHANGELOG_PENDING.md` updated, or no changelog entry needed - [x] Updated relevant documentation (`docs/`) and code comments, or no documentation updates needed (cherry picked from commit f58ba4d2f9c71071554dfb0273252edf90aa85e5) Co-authored-by: Lasaro Camargos * add allocate in Receive calls (#9667) * rpc: Add caching support (backport #9650) (#9666) * rpc: Add caching support (#9650) * Set cache control in the HTTP-RPC response header * Add a simply cache policy to the RPC routes * add a condition to check the RPC request has default height settings * fix cherry pick error * update pending log * use options struct intead of single parameter * refacor FuncOptions to functional options * add functional options in WebSocket RPC function * revert doc * replace deprecated function call * revise functional options * remove unuse comment * fix revised error * adjust cache-control settings * Update rpc/jsonrpc/server/http_json_handler.go Co-authored-by: Thane Thomson * linter: Fix false positive Signed-off-by: Thane Thomson * rpc: Separate cacheable and non-cacheable HTTP response writers Allows us to roll this change out in a non-API-breaking way, since this is an additive change. Signed-off-by: Thane Thomson * rpc: Ensure consistent caching strategy Ensure a consistent caching strategy across both JSONRPC- and URI-based requests. This requires a bit of a refactor of the previous caching logic, which is complicated a little by the complex reflection-based approach taken in the Tendermint RPC. Signed-off-by: Thane Thomson * rpc: Add more tests for caching Signed-off-by: Thane Thomson * Update CHANGELOG_PENDING Signed-off-by: Thane Thomson * light: Sync routes config with RPC core Signed-off-by: Thane Thomson * rpc: Update OpenAPI docs Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson Co-authored-by: jayt106 Co-authored-by: jay tseng Co-authored-by: JayT106 (cherry picked from commit 816c6bac00c63a421a1bdaeccbc081c5346cb0d8) # Conflicts: # CHANGELOG_PENDING.md # light/proxy/routes.go # rpc/core/routes.go # rpc/openapi/openapi.yaml # test/fuzz/tests/rpc_jsonrpc_server_test.go * Fix conflict in CHANGELOG_PENDING Signed-off-by: Thane Thomson * Resolve remaining conflicts Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson Co-authored-by: Thane Thomson * docs: Add new per-message type P2P metrics (backport #9676) (#9679) * docs: Add new per-message type P2P metrics (#9676) * docs: Monospace metric names Signed-off-by: Thane Thomson * docs: Consistently capitalize metric types Signed-off-by: Thane Thomson * docs: Monospace metric tags Signed-off-by: Thane Thomson * docs: Fix underscores in metrics page Signed-off-by: Thane Thomson * docs: Make metric description capitalization consistent Signed-off-by: Thane Thomson * docs: Add new per-message P2P metrics Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson (cherry picked from commit 3aa6c816e5606e45b362d27020a32c2e01ed20be) # Conflicts: # docs/tendermint-core/metrics.md * Resolve conflicts Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson Co-authored-by: Thane Thomson * Release v0.34.23 (#9684) * version: Bump to v0.34.23 Signed-off-by: Thane Thomson * Prepare changelog Signed-off-by: Thane Thomson Signed-off-by: Thane Thomson * fix errors Signed-off-by: Thane Thomson Signed-off-by: Marko Baricevic Signed-off-by: dependabot[bot] Co-authored-by: M. J. Fromberger Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sam Kleinman Co-authored-by: Marko Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: samricotta <37125168+samricotta@users.noreply.github.com> Co-authored-by: Thane Thomson Co-authored-by: Milosevic, Zarko Co-authored-by: Milosevic, Zarko Co-authored-by: Zarko Milosevic Co-authored-by: Zarko Milosevic Co-authored-by: Anton Kaliaev Co-authored-by: Anca Zamfir Co-authored-by: dongsamb Co-authored-by: Sunny Aggarwal Co-authored-by: Anca Zamfir Co-authored-by: Ethan Buchman Co-authored-by: Zarko Milosevic Co-authored-by: Ismail Khoffi Co-authored-by: Zaki Manian Co-authored-by: Erik Grinaker Co-authored-by: Tess Rinearson Co-authored-by: Alexander Simmerl Co-authored-by: Igor Konnov Co-authored-by: Sean Braithwaite Co-authored-by: Josef Widder <44643235+josef-widder@users.noreply.github.com> Co-authored-by: Andrey Kuprianov <59489470+andrey-kuprianov@users.noreply.github.com> Co-authored-by: Igor Konnov Co-authored-by: Sam Hart Co-authored-by: Robert Zaremba Co-authored-by: Giuliano Co-authored-by: Shahan Khatchadourian Co-authored-by: Dev Ojha Co-authored-by: istoilkovska Co-authored-by: Aleksandr Bezobchuk Co-authored-by: Sunny Aggarwal Co-authored-by: Federico Kunze <31522760+fedekunze@users.noreply.github.com> Co-authored-by: Marko Baricevic Co-authored-by: Giuliano Co-authored-by: Jordan Sexton Co-authored-by: MengXiangJian <805442788@qq.com> Co-authored-by: Yixin Luo <18810541851@163.com> Co-authored-by: crypto-facs <84574577+crypto-facs@users.noreply.github.com> Co-authored-by: Giuliano Co-authored-by: Mateusz Górski Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: William Banfield Co-authored-by: simon <71435388+simonshampoo@users.noreply.github.com> Co-authored-by: Sergio Mena Co-authored-by: Jasmina Malicevic Co-authored-by: Rootul P Co-authored-by: Lasaro Camargos --- crypto/merkle/doc.go | 25 ++++++++++++------------- crypto/merkle/proof_value.go | 4 ++-- crypto/merkle/tree.go | 8 ++++---- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crypto/merkle/doc.go b/crypto/merkle/doc.go index 865c302..fe50b34 100644 --- a/crypto/merkle/doc.go +++ b/crypto/merkle/doc.go @@ -12,20 +12,19 @@ second pre-image attacks. Hence, use this library with caution. Otherwise you might run into similar issues as, e.g., in early Bitcoin: https://bitcointalk.org/?topic=102395 - * - / \ - / \ - / \ - / \ - * * - / \ / \ - / \ / \ - / \ / \ - * * * h6 - / \ / \ / \ - h0 h1 h2 h3 h4 h5 + * + / \ + / \ + / \ + / \ + * * + / \ / \ + / \ / \ + / \ / \ + * * * h6 + / \ / \ / \ + h0 h1 h2 h3 h4 h5 TODO(ismail): add 2nd pre-image protection or clarify further on how we use this and why this secure. - */ package merkle diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index ab77621..842dc82 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -85,8 +85,8 @@ func (op ValueOp) Run(args [][]byte) ([][]byte, error) { bz := new(bytes.Buffer) // Wrap to hash the KVPair. - encodeByteSlice(bz, op.key) // nolint: errcheck // does not error - encodeByteSlice(bz, vhash) // nolint: errcheck // does not error + encodeByteSlice(bz, op.key) //nolint: errcheck // does not error + encodeByteSlice(bz, vhash) //nolint: errcheck // does not error kvhash := leafHash(bz.Bytes()) if !bytes.Equal(kvhash, op.Proof.LeafHash) { diff --git a/crypto/merkle/tree.go b/crypto/merkle/tree.go index 466c434..089c2f8 100644 --- a/crypto/merkle/tree.go +++ b/crypto/merkle/tree.go @@ -47,10 +47,10 @@ func HashFromByteSlices(items [][]byte) []byte { // // These preliminary results suggest: // -// 1. The performance of the HashFromByteSlice is pretty good -// 2. Go has low overhead for recursive functions -// 3. The performance of the HashFromByteSlice routine is dominated -// by the actual hashing of data +// 1. The performance of the HashFromByteSlice is pretty good +// 2. Go has low overhead for recursive functions +// 3. The performance of the HashFromByteSlice routine is dominated +// by the actual hashing of data // // Although this work is in no way exhaustive, point #3 suggests that // optimization of this routine would need to take an alternative From 06de1ec0b31563038763adc1508c6b57cb8593ba Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Wed, 14 Dec 2022 09:19:51 -0600 Subject: [PATCH 75/86] feat: new method to verify using keys instead of keypath (#912) * feat: new method to verify using keys * fix: add additional needed methods --- crypto/merkle/proof_op.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 038037c..ef5eef3 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -68,6 +68,33 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er return nil } +func (poz ProofOperators) VerifyKeys(root []byte, keys [][]byte, args [][]byte) (err error) { + for i, op := range poz { + key := op.GetKey() + if len(key) != 0 { + if len(keys) == 0 { + return fmt.Errorf("key path has insufficient # of parts: expected no more keys but got %+v", string(key)) + } + lastKey := keys[len(keys)-1] + if !bytes.Equal(lastKey, key) { + return fmt.Errorf("key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) + } + keys = keys[:len(keys)-1] + } + args, err = op.Run(args) + if err != nil { + return + } + } + if !bytes.Equal(root, args[0]) { + return fmt.Errorf("calculated root hash is invalid: expected %X but got %X", root, args[0]) + } + if len(keys) != 0 { + return errors.New("keypath not consumed all") + } + return nil +} + //---------------------------------------- // ProofRuntime - main entrypoint @@ -115,6 +142,10 @@ func (prt *ProofRuntime) VerifyValue(proof *tmcrypto.ProofOps, root []byte, keyp return prt.Verify(proof, root, keypath, [][]byte{value}) } +func (prt *ProofRuntime) VerifyValueKeys(proof *tmcrypto.ProofOps, root []byte, keys [][]byte, value []byte) (err error) { + return prt.VerifyKeys(proof, root, keys, [][]byte{value}) +} + // TODO In the long run we'll need a method of classifcation of ops, // whether existence or absence or perhaps a third? func (prt *ProofRuntime) VerifyAbsence(proof *tmcrypto.ProofOps, root []byte, keypath string) (err error) { @@ -129,6 +160,14 @@ func (prt *ProofRuntime) Verify(proof *tmcrypto.ProofOps, root []byte, keypath s return poz.Verify(root, keypath, args) } +func (prt *ProofRuntime) VerifyKeys(proof *tmcrypto.ProofOps, root []byte, keys [][]byte, args [][]byte) (err error) { + poz, err := prt.DecodeProof(proof) + if err != nil { + return fmt.Errorf("decoding proof: %w", err) + } + return poz.VerifyKeys(root, keys, args) +} + // DefaultProofRuntime only knows about value proofs. // To use e.g. IAVL proofs, register op-decoders as // defined in the IAVL package. From 901b299fc8f581f115b08694230473f3e0a771d1 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Fri, 16 Dec 2022 05:26:52 -0600 Subject: [PATCH 76/86] chore!: add docs and tests to new VerifyFromKeys methods (#914) * chore! add docs and tests to new VerifyFromKeys methods * chore: use assert.Error/assert.NoError --- crypto/merkle/proof_op.go | 18 +++++++---- crypto/merkle/proof_test.go | 64 +++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index ef5eef3..2e7b190 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -68,7 +68,10 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er return nil } -func (poz ProofOperators) VerifyKeys(root []byte, keys [][]byte, args [][]byte) (err error) { +// VerifyFromKeys performs the same verification logic as the normal Verify +// method, except it does not perform any processing on the keypath. This is +// useful when using keys that have split or escape points as a part of the key. +func (poz ProofOperators) VerifyFromKeys(root []byte, keys [][]byte, args [][]byte) (err error) { for i, op := range poz { key := op.GetKey() if len(key) != 0 { @@ -90,7 +93,7 @@ func (poz ProofOperators) VerifyKeys(root []byte, keys [][]byte, args [][]byte) return fmt.Errorf("calculated root hash is invalid: expected %X but got %X", root, args[0]) } if len(keys) != 0 { - return errors.New("keypath not consumed all") + return fmt.Errorf("keypath not consumed all: %s", string(bytes.Join(keys, []byte("/")))) } return nil } @@ -142,8 +145,8 @@ func (prt *ProofRuntime) VerifyValue(proof *tmcrypto.ProofOps, root []byte, keyp return prt.Verify(proof, root, keypath, [][]byte{value}) } -func (prt *ProofRuntime) VerifyValueKeys(proof *tmcrypto.ProofOps, root []byte, keys [][]byte, value []byte) (err error) { - return prt.VerifyKeys(proof, root, keys, [][]byte{value}) +func (prt *ProofRuntime) VerifyValueFromKeys(proof *tmcrypto.ProofOps, root []byte, keys [][]byte, value []byte) (err error) { + return prt.VerifyFromKeys(proof, root, keys, [][]byte{value}) } // TODO In the long run we'll need a method of classifcation of ops, @@ -160,12 +163,15 @@ func (prt *ProofRuntime) Verify(proof *tmcrypto.ProofOps, root []byte, keypath s return poz.Verify(root, keypath, args) } -func (prt *ProofRuntime) VerifyKeys(proof *tmcrypto.ProofOps, root []byte, keys [][]byte, args [][]byte) (err error) { +// VerifyFromKeys performs the same verification logic as the normal Verify +// method, except it does not perform any processing on the keypath. This is +// useful when using keys that have split or escape points as a part of the key. +func (prt *ProofRuntime) VerifyFromKeys(proof *tmcrypto.ProofOps, root []byte, keys [][]byte, args [][]byte) (err error) { poz, err := prt.DecodeProof(proof) if err != nil { return fmt.Errorf("decoding proof: %w", err) } - return poz.VerifyKeys(root, keys, args) + return poz.VerifyFromKeys(root, keys, args) } // DefaultProofRuntime only knows about value proofs. diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 22ab900..fcd4a94 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -133,6 +133,70 @@ func TestProofOperators(t *testing.T) { assert.NotNil(t, err) } +func TestProofOperatorsFromKeys(t *testing.T) { + var err error + + // ProofRuntime setup + // TODO test this somehow. + + // ProofOperators setup + op1 := NewDominoOp("KEY1", "INPUT1", "INPUT2") + op2 := NewDominoOp("KEY%2", "INPUT2", "INPUT3") + op3 := NewDominoOp("", "INPUT3", "INPUT4") + op4 := NewDominoOp("KEY/4", "INPUT4", "OUTPUT4") + + // add characters to the keys that would otherwise result in bad keypath if + // processed + keys1 := [][]byte{bz("KEY/4"), bz("KEY%2"), bz("KEY1")} + badkeys1 := [][]byte{bz("WrongKey"), bz("KEY%2"), bz("KEY1")} + keys2 := [][]byte{bz("KEY3"), bz("KEY%2"), bz("KEY1")} + keys3 := [][]byte{bz("KEY2"), bz("KEY1")} + + // Good + popz := ProofOperators([]ProofOperator{op1, op2, op3, op4}) + err = popz.VerifyFromKeys(bz("OUTPUT4"), keys1, [][]byte{bz("INPUT1")}) + assert.NoError(t, err) + + // BAD INPUT + err = popz.VerifyFromKeys(bz("OUTPUT4"), keys1, [][]byte{bz("INPUT1_WRONG")}) + assert.Error(t, err) + + // BAD KEY 1 + err = popz.VerifyFromKeys(bz("OUTPUT4"), keys2, [][]byte{bz("INPUT1")}) + assert.Error(t, err) + + // BAD KEY 2 + err = popz.VerifyFromKeys(bz("OUTPUT4"), badkeys1, [][]byte{bz("INPUT1")}) + assert.Error(t, err) + + // BAD KEY 5 + err = popz.VerifyFromKeys(bz("OUTPUT4"), keys3, [][]byte{bz("INPUT1")}) + assert.Error(t, err) + + // BAD OUTPUT 1 + err = popz.VerifyFromKeys(bz("OUTPUT4_WRONG"), keys1, [][]byte{bz("INPUT1")}) + assert.Error(t, err) + + // BAD OUTPUT 2 + err = popz.VerifyFromKeys(bz(""), keys1, [][]byte{bz("INPUT1")}) + assert.Error(t, err) + + // BAD POPZ 1 + popz = []ProofOperator{op1, op2, op4} + err = popz.VerifyFromKeys(bz("OUTPUT4"), keys1, [][]byte{bz("INPUT1")}) + assert.Error(t, err) + + // BAD POPZ 2 + popz = []ProofOperator{op4, op3, op2, op1} + err = popz.VerifyFromKeys(bz("OUTPUT4"), keys1, [][]byte{bz("INPUT1")}) + assert.Error(t, err) + + // BAD POPZ 3 + popz = []ProofOperator{} + err = popz.VerifyFromKeys(bz("OUTPUT4"), keys1, [][]byte{bz("INPUT1")}) + assert.Error(t, err) +} + func bz(s string) []byte { return []byte(s) } From ceb9c8da833a822dbac4b253dbb04eb5f5240718 Mon Sep 17 00:00:00 2001 From: Lasaro Date: Thu, 19 Jan 2023 09:27:29 -0300 Subject: [PATCH 77/86] replace tm for cmt for v0.34.x (#108) * Renaming of variables and aliases prefixed with tm to use prefix cmt * Replacing references to alias in the documentation * Placate linter * Replacing missed references * reverting typos --- crypto/merkle/proof.go | 8 ++++---- crypto/merkle/proof_op.go | 16 ++++++++-------- crypto/merkle/proof_test.go | 8 ++++---- crypto/merkle/proof_value.go | 12 ++++++------ crypto/merkle/tree_test.go | 10 +++++----- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index ab43f30..e3dd0f9 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" - tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ( @@ -116,11 +116,11 @@ func (sp *Proof) ValidateBasic() error { return nil } -func (sp *Proof) ToProto() *tmcrypto.Proof { +func (sp *Proof) ToProto() *cmtcrypto.Proof { if sp == nil { return nil } - pb := new(tmcrypto.Proof) + pb := new(cmtcrypto.Proof) pb.Total = sp.Total pb.Index = sp.Index @@ -130,7 +130,7 @@ func (sp *Proof) ToProto() *tmcrypto.Proof { return pb } -func ProofFromProto(pb *tmcrypto.Proof) (*Proof, error) { +func ProofFromProto(pb *cmtcrypto.Proof) (*Proof, error) { if pb == nil { return nil, errors.New("nil proof") } diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 038037c..2731df6 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) //---------------------------------------- @@ -21,7 +21,7 @@ import ( type ProofOperator interface { Run([][]byte) ([][]byte, error) GetKey() []byte - ProofOp() tmcrypto.ProofOp + ProofOp() cmtcrypto.ProofOp } //---------------------------------------- @@ -71,7 +71,7 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er //---------------------------------------- // ProofRuntime - main entrypoint -type OpDecoder func(tmcrypto.ProofOp) (ProofOperator, error) +type OpDecoder func(cmtcrypto.ProofOp) (ProofOperator, error) type ProofRuntime struct { decoders map[string]OpDecoder @@ -91,7 +91,7 @@ func (prt *ProofRuntime) RegisterOpDecoder(typ string, dec OpDecoder) { prt.decoders[typ] = dec } -func (prt *ProofRuntime) Decode(pop tmcrypto.ProofOp) (ProofOperator, error) { +func (prt *ProofRuntime) Decode(pop cmtcrypto.ProofOp) (ProofOperator, error) { decoder := prt.decoders[pop.Type] if decoder == nil { return nil, fmt.Errorf("unrecognized proof type %v", pop.Type) @@ -99,7 +99,7 @@ func (prt *ProofRuntime) Decode(pop tmcrypto.ProofOp) (ProofOperator, error) { return decoder(pop) } -func (prt *ProofRuntime) DecodeProof(proof *tmcrypto.ProofOps) (ProofOperators, error) { +func (prt *ProofRuntime) DecodeProof(proof *cmtcrypto.ProofOps) (ProofOperators, error) { poz := make(ProofOperators, 0, len(proof.Ops)) for _, pop := range proof.Ops { operator, err := prt.Decode(pop) @@ -111,17 +111,17 @@ func (prt *ProofRuntime) DecodeProof(proof *tmcrypto.ProofOps) (ProofOperators, return poz, nil } -func (prt *ProofRuntime) VerifyValue(proof *tmcrypto.ProofOps, root []byte, keypath string, value []byte) (err error) { +func (prt *ProofRuntime) VerifyValue(proof *cmtcrypto.ProofOps, root []byte, keypath string, value []byte) (err error) { return prt.Verify(proof, root, keypath, [][]byte{value}) } // TODO In the long run we'll need a method of classifcation of ops, // whether existence or absence or perhaps a third? -func (prt *ProofRuntime) VerifyAbsence(proof *tmcrypto.ProofOps, root []byte, keypath string) (err error) { +func (prt *ProofRuntime) VerifyAbsence(proof *cmtcrypto.ProofOps, root []byte, keypath string) (err error) { return prt.Verify(proof, root, keypath, nil) } -func (prt *ProofRuntime) Verify(proof *tmcrypto.ProofOps, root []byte, keypath string, args [][]byte) (err error) { +func (prt *ProofRuntime) Verify(proof *cmtcrypto.ProofOps, root []byte, keypath string, args [][]byte) (err error) { poz, err := prt.DecodeProof(proof) if err != nil { return fmt.Errorf("decoding proof: %w", err) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 22ab900..b07c853 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ProofOpDomino = "test:domino" @@ -29,8 +29,8 @@ func NewDominoOp(key, input, output string) DominoOp { } } -func (dop DominoOp) ProofOp() tmcrypto.ProofOp { - dopb := tmcrypto.DominoOp{ +func (dop DominoOp) ProofOp() cmtcrypto.ProofOp { + dopb := cmtcrypto.DominoOp{ Key: dop.key, Input: dop.Input, Output: dop.Output, @@ -40,7 +40,7 @@ func (dop DominoOp) ProofOp() tmcrypto.ProofOp { panic(err) } - return tmcrypto.ProofOp{ + return cmtcrypto.ProofOp{ Type: ProofOpDomino, Key: []byte(dop.key), Data: bz, diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 842dc82..6471b33 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" - tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ProofOpValue = "simple:v" @@ -37,11 +37,11 @@ func NewValueOp(key []byte, proof *Proof) ValueOp { } } -func ValueOpDecoder(pop tmcrypto.ProofOp) (ProofOperator, error) { +func ValueOpDecoder(pop cmtcrypto.ProofOp) (ProofOperator, error) { if pop.Type != ProofOpValue { return nil, fmt.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpValue) } - var pbop tmcrypto.ValueOp // a bit strange as we'll discard this, but it works. + var pbop cmtcrypto.ValueOp // a bit strange as we'll discard this, but it works. err := pbop.Unmarshal(pop.Data) if err != nil { return nil, fmt.Errorf("decoding ProofOp.Data into ValueOp: %w", err) @@ -54,8 +54,8 @@ func ValueOpDecoder(pop tmcrypto.ProofOp) (ProofOperator, error) { return NewValueOp(pop.Key, sp), nil } -func (op ValueOp) ProofOp() tmcrypto.ProofOp { - pbval := tmcrypto.ValueOp{ +func (op ValueOp) ProofOp() cmtcrypto.ProofOp { + pbval := cmtcrypto.ValueOp{ Key: op.key, Proof: op.Proof.ToProto(), } @@ -63,7 +63,7 @@ func (op ValueOp) ProofOp() tmcrypto.ProofOp { if err != nil { panic(err) } - return tmcrypto.ProofOp{ + return cmtcrypto.ProofOp{ Type: ProofOpValue, Key: op.key, Data: bz, diff --git a/crypto/merkle/tree_test.go b/crypto/merkle/tree_test.go index bdf4698..4c05b7a 100644 --- a/crypto/merkle/tree_test.go +++ b/crypto/merkle/tree_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmrand "github.com/tendermint/tendermint/libs/rand" + cmtrand "github.com/tendermint/tendermint/libs/rand" . "github.com/tendermint/tendermint/libs/test" "github.com/tendermint/tendermint/crypto/tmhash" @@ -54,7 +54,7 @@ func TestProof(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(tmrand.Bytes(tmhash.Size)) + items[i] = testItem(cmtrand.Bytes(tmhash.Size)) } rootHash = HashFromByteSlices(items) @@ -78,7 +78,7 @@ func TestProof(t *testing.T) { // Trail too long should make it fail origAunts := proof.Aunts - proof.Aunts = append(proof.Aunts, tmrand.Bytes(32)) + proof.Aunts = append(proof.Aunts, cmtrand.Bytes(32)) err = proof.Verify(rootHash, item) require.Error(t, err, "Expected verification to fail for wrong trail length") @@ -107,7 +107,7 @@ func TestHashAlternatives(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(tmrand.Bytes(tmhash.Size)) + items[i] = testItem(cmtrand.Bytes(tmhash.Size)) } rootHash1 := HashFromByteSlicesIterative(items) @@ -120,7 +120,7 @@ func BenchmarkHashAlternatives(b *testing.B) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(tmrand.Bytes(tmhash.Size)) + items[i] = testItem(cmtrand.Bytes(tmhash.Size)) } b.ResetTimer() From c96d101d3e6cbd278bf461d48b08bc4f105cd3bc Mon Sep 17 00:00:00 2001 From: Lasaro Date: Tue, 24 Jan 2023 07:57:00 -0300 Subject: [PATCH 78/86] 0.34.x: replace tendermint/tendermint for cometbft/cometbft (#156) * Replacing tendermint/tendermint for cometbft/cometbft and making necessary adjustments in Makefile, go.sum and go.mod to use the updated names * Fixing lint issues * trying to get more info on which files were not linted properly * Revert "trying to get more info on which files were not linted properly" This reverts commit 5677fa3c9292c5f4c2d14c9b0ef4819f3980ac28. * Fixing lint issues * Fixing lint issues --- crypto/merkle/hash.go | 2 +- crypto/merkle/proof.go | 4 ++-- crypto/merkle/proof_op.go | 2 +- crypto/merkle/proof_test.go | 2 +- crypto/merkle/proof_value.go | 4 ++-- crypto/merkle/rfc6962_test.go | 2 +- crypto/merkle/tree_test.go | 6 +++--- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index d45130f..e5f7318 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -1,7 +1,7 @@ package merkle import ( - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/cometbft/cometbft/crypto/tmhash" ) // TODO: make these have a large predefined capacity diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index e3dd0f9..51e107c 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" - "github.com/tendermint/tendermint/crypto/tmhash" - cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + "github.com/cometbft/cometbft/crypto/tmhash" + cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" ) const ( diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 2731df6..6282030 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" ) //---------------------------------------- diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index b07c853..677cf84 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" ) const ProofOpDomino = "test:domino" diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 6471b33..40eb607 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" - "github.com/tendermint/tendermint/crypto/tmhash" - cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + "github.com/cometbft/cometbft/crypto/tmhash" + cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" ) const ProofOpValue = "simple:v" diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go index c762cda..dd0c817 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/crypto/merkle/rfc6962_test.go @@ -20,7 +20,7 @@ import ( "encoding/hex" "testing" - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/cometbft/cometbft/crypto/tmhash" ) func TestRFC6962Hasher(t *testing.T) { diff --git a/crypto/merkle/tree_test.go b/crypto/merkle/tree_test.go index 4c05b7a..72f1402 100644 --- a/crypto/merkle/tree_test.go +++ b/crypto/merkle/tree_test.go @@ -7,10 +7,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - cmtrand "github.com/tendermint/tendermint/libs/rand" - . "github.com/tendermint/tendermint/libs/test" + cmtrand "github.com/cometbft/cometbft/libs/rand" + . "github.com/cometbft/cometbft/libs/test" - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/cometbft/cometbft/crypto/tmhash" ) type testItem []byte From 4358672261206c7eb7767fa58f352b42ee00fd08 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 7 Feb 2023 08:35:51 +0100 Subject: [PATCH 79/86] 0.34.x: Rename Tendermint to CometBFT: further actions (backport #224) (#255) * Rename Tendermint to CometBFT: further actions (#224) * rename: Tendermint -> CometBFT in README files * rename: Tendermint -> CometBFT in doc.go files * rename: Tendermint -> CometBFT in Go documentation * Removing unused/outdated CHANGELOGs * rename: using CometBFT in issues and PR templates * rename: references to tendermin in libs/events * rename: renaming on text files under rpc/ dir * rename: Tendermint-go -> CometBFT in proto docs * rename: Tendermint -> CometBFT in generated go docs * rename: renaming Tendermint -> CometBFT in scripts * rename: renaming to CometBFT in code comments * rename: general script files on repository root * rename: using CometBFT in github config files * rename: Go docs in .proto files, pb.go regenerated * rename: toml config file generated content * rename: avoiding unnecessary TM entries in comments * rename: renaming in log messages and strings * rename: upgrading instructions renamed, needs review * rename: fixing generated protobuf files * rename: fixing generated protobuf files * rename: fixing broken test, renamed string * Apply suggestions from code review * Update .github/PULL_REQUEST_TEMPLATE.md Co-authored-by: Thane Thomson * Update abci/README.md Co-authored-by: Thane Thomson * Update proto/README.md Co-authored-by: Thane Thomson * rename: removing unused files on repo root --------- Co-authored-by: Daniel Cason Co-authored-by: Thane Thomson (cherry picked from commit 1cb55d49baa86a227c4b9eadad2d7a344670eb51) # Conflicts: # .github/ISSUE_TEMPLATE/bug-report.md # .github/ISSUE_TEMPLATE/config.yml # .github/PULL_REQUEST_TEMPLATE.md # .github/issue_template.md # .github/workflows/janitor.yml # .github/workflows/lint.yml # .gitignore # .goreleaser.yml # UPGRADING.md # abci/README.md # abci/types/application.go # abci/types/mocks/base.go # cmd/cometbft/commands/version.go # config/config.go # config/toml.go # consensus/README.md # consensus/replay.go # consensus/replay_test.go # consensus/state.go # crypto/README.md # crypto/ed25519/ed25519.go # inspect/doc.go # inspect/inspect.go # internal/test/doc.go # libs/pubsub/query/query.go # libs/pubsub/query/syntax/doc.go # light/doc.go # networks/local/README.md # node/node.go # node/setup.go # p2p/README.md # proto/README.md # rpc/core/abci.go # rpc/core/blocks.go # rpc/core/consensus.go # rpc/core/doc.go # rpc/core/events.go # rpc/core/evidence.go # rpc/core/health.go # rpc/core/mempool.go # rpc/core/net.go # rpc/core/status.go # rpc/core/tx.go # rpc/jsonrpc/doc.go # rpc/openapi/openapi.yaml # scripts/metricsgen/metricsgen.go # test/fuzz/README.md # test/fuzz/oss-fuzz-build.sh # test/loadtime/cmd/load/main.go # tools/README.md # version/version.go * rename: fixing cherry-pick conflicts * rename: fixing cherry-pick conflicts Co-authored-by: Jasmina Malicevic * rename: fixing cherry-pick conflicts Co-authored-by: Jasmina Malicevic * rename: reverted changes on UPGRADING.md * rename: Tendermint Core -> CometBFT on intro * rename: further renaming Tendermint -> CometBFT * rename: Tendermint -> CometBFT under test/maverick/ * rename: further renaming Tendermint -> CometBFT * rename: removing copyright from internal package * rename: Tendermint -> CometBFT in /spec/abci * rename: further renaming Tendermint -> CometBFT * rename: undoing renaming of jTendermint * rename: removing reference to tendermint/go-amino * rename: tendermint -> cometbft on .circleci/ --------- Co-authored-by: Daniel Co-authored-by: Jasmina Malicevic --- crypto/merkle/proof_value.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 40eb607..7c267e7 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -13,7 +13,7 @@ const ProofOpValue = "simple:v" // ValueOp takes a key and a single value as argument and // produces the root hash. The corresponding tree structure is // the SimpleMap tree. SimpleMap takes a Hasher, and currently -// Tendermint uses tmhash. SimpleValueOp should support +// CometBFT uses tmhash. SimpleValueOp should support // the hash function as used in tmhash. TODO support // additional hash functions here as options/args to this // operator. From 5c007c2941268785195c809ebb49d6ab1659166f Mon Sep 17 00:00:00 2001 From: Lasaro Date: Tue, 21 Feb 2023 09:54:20 -0300 Subject: [PATCH 80/86] =?UTF-8?q?Rename=20package=20back=20to=20tendermint?= =?UTF-8?q?/tendermint=20and=20use=20replace=20to=20point=E2=80=A6=20(#351?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Rename package back to tendermint/tendermint and use replace to point to the current repo * Rename package back to tendermint/tendermint in the proto files and generated files * Rename package back to tendermint/tendermint in the mocks * Remove useless replace * Apply suggestions from code review * Apply suggestions from code review --- crypto/merkle/hash.go | 2 +- crypto/merkle/proof.go | 4 ++-- crypto/merkle/proof_op.go | 2 +- crypto/merkle/proof_test.go | 2 +- crypto/merkle/proof_value.go | 4 ++-- crypto/merkle/rfc6962_test.go | 2 +- crypto/merkle/tree_test.go | 6 +++--- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index e5f7318..d45130f 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -1,7 +1,7 @@ package merkle import ( - "github.com/cometbft/cometbft/crypto/tmhash" + "github.com/tendermint/tendermint/crypto/tmhash" ) // TODO: make these have a large predefined capacity diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 51e107c..e3dd0f9 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" - "github.com/cometbft/cometbft/crypto/tmhash" - cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/tmhash" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ( diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 6282030..2731df6 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) //---------------------------------------- diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 677cf84..b07c853 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ProofOpDomino = "test:domino" diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 7c267e7..bae6849 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" - "github.com/cometbft/cometbft/crypto/tmhash" - cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/tmhash" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ProofOpValue = "simple:v" diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go index dd0c817..c762cda 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/crypto/merkle/rfc6962_test.go @@ -20,7 +20,7 @@ import ( "encoding/hex" "testing" - "github.com/cometbft/cometbft/crypto/tmhash" + "github.com/tendermint/tendermint/crypto/tmhash" ) func TestRFC6962Hasher(t *testing.T) { diff --git a/crypto/merkle/tree_test.go b/crypto/merkle/tree_test.go index 72f1402..4c05b7a 100644 --- a/crypto/merkle/tree_test.go +++ b/crypto/merkle/tree_test.go @@ -7,10 +7,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - cmtrand "github.com/cometbft/cometbft/libs/rand" - . "github.com/cometbft/cometbft/libs/test" + cmtrand "github.com/tendermint/tendermint/libs/rand" + . "github.com/tendermint/tendermint/libs/test" - "github.com/cometbft/cometbft/crypto/tmhash" + "github.com/tendermint/tendermint/crypto/tmhash" ) type testItem []byte From 1bea554108e4f1a1685a6b004113b4586343e6f0 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Wed, 15 Mar 2023 18:32:48 +0100 Subject: [PATCH 81/86] Update to 'v0.34.27' of github.com:cometbft/cometbft (#980) --- crypto/merkle/proof.go | 8 ++++---- crypto/merkle/proof_op.go | 20 ++++++++++---------- crypto/merkle/proof_test.go | 8 ++++---- crypto/merkle/proof_value.go | 14 +++++++------- crypto/merkle/tree_test.go | 10 +++++----- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index ab43f30..e3dd0f9 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" - tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ( @@ -116,11 +116,11 @@ func (sp *Proof) ValidateBasic() error { return nil } -func (sp *Proof) ToProto() *tmcrypto.Proof { +func (sp *Proof) ToProto() *cmtcrypto.Proof { if sp == nil { return nil } - pb := new(tmcrypto.Proof) + pb := new(cmtcrypto.Proof) pb.Total = sp.Total pb.Index = sp.Index @@ -130,7 +130,7 @@ func (sp *Proof) ToProto() *tmcrypto.Proof { return pb } -func ProofFromProto(pb *tmcrypto.Proof) (*Proof, error) { +func ProofFromProto(pb *cmtcrypto.Proof) (*Proof, error) { if pb == nil { return nil, errors.New("nil proof") } diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 2e7b190..fdf987a 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) //---------------------------------------- @@ -21,7 +21,7 @@ import ( type ProofOperator interface { Run([][]byte) ([][]byte, error) GetKey() []byte - ProofOp() tmcrypto.ProofOp + ProofOp() cmtcrypto.ProofOp } //---------------------------------------- @@ -101,7 +101,7 @@ func (poz ProofOperators) VerifyFromKeys(root []byte, keys [][]byte, args [][]by //---------------------------------------- // ProofRuntime - main entrypoint -type OpDecoder func(tmcrypto.ProofOp) (ProofOperator, error) +type OpDecoder func(cmtcrypto.ProofOp) (ProofOperator, error) type ProofRuntime struct { decoders map[string]OpDecoder @@ -121,7 +121,7 @@ func (prt *ProofRuntime) RegisterOpDecoder(typ string, dec OpDecoder) { prt.decoders[typ] = dec } -func (prt *ProofRuntime) Decode(pop tmcrypto.ProofOp) (ProofOperator, error) { +func (prt *ProofRuntime) Decode(pop cmtcrypto.ProofOp) (ProofOperator, error) { decoder := prt.decoders[pop.Type] if decoder == nil { return nil, fmt.Errorf("unrecognized proof type %v", pop.Type) @@ -129,7 +129,7 @@ func (prt *ProofRuntime) Decode(pop tmcrypto.ProofOp) (ProofOperator, error) { return decoder(pop) } -func (prt *ProofRuntime) DecodeProof(proof *tmcrypto.ProofOps) (ProofOperators, error) { +func (prt *ProofRuntime) DecodeProof(proof *cmtcrypto.ProofOps) (ProofOperators, error) { poz := make(ProofOperators, 0, len(proof.Ops)) for _, pop := range proof.Ops { operator, err := prt.Decode(pop) @@ -141,21 +141,21 @@ func (prt *ProofRuntime) DecodeProof(proof *tmcrypto.ProofOps) (ProofOperators, return poz, nil } -func (prt *ProofRuntime) VerifyValue(proof *tmcrypto.ProofOps, root []byte, keypath string, value []byte) (err error) { +func (prt *ProofRuntime) VerifyValue(proof *cmtcrypto.ProofOps, root []byte, keypath string, value []byte) (err error) { return prt.Verify(proof, root, keypath, [][]byte{value}) } -func (prt *ProofRuntime) VerifyValueFromKeys(proof *tmcrypto.ProofOps, root []byte, keys [][]byte, value []byte) (err error) { +func (prt *ProofRuntime) VerifyValueFromKeys(proof *cmtcrypto.ProofOps, root []byte, keys [][]byte, value []byte) (err error) { return prt.VerifyFromKeys(proof, root, keys, [][]byte{value}) } // TODO In the long run we'll need a method of classifcation of ops, // whether existence or absence or perhaps a third? -func (prt *ProofRuntime) VerifyAbsence(proof *tmcrypto.ProofOps, root []byte, keypath string) (err error) { +func (prt *ProofRuntime) VerifyAbsence(proof *cmtcrypto.ProofOps, root []byte, keypath string) (err error) { return prt.Verify(proof, root, keypath, nil) } -func (prt *ProofRuntime) Verify(proof *tmcrypto.ProofOps, root []byte, keypath string, args [][]byte) (err error) { +func (prt *ProofRuntime) Verify(proof *cmtcrypto.ProofOps, root []byte, keypath string, args [][]byte) (err error) { poz, err := prt.DecodeProof(proof) if err != nil { return fmt.Errorf("decoding proof: %w", err) @@ -166,7 +166,7 @@ func (prt *ProofRuntime) Verify(proof *tmcrypto.ProofOps, root []byte, keypath s // VerifyFromKeys performs the same verification logic as the normal Verify // method, except it does not perform any processing on the keypath. This is // useful when using keys that have split or escape points as a part of the key. -func (prt *ProofRuntime) VerifyFromKeys(proof *tmcrypto.ProofOps, root []byte, keys [][]byte, args [][]byte) (err error) { +func (prt *ProofRuntime) VerifyFromKeys(proof *cmtcrypto.ProofOps, root []byte, keys [][]byte, args [][]byte) (err error) { poz, err := prt.DecodeProof(proof) if err != nil { return fmt.Errorf("decoding proof: %w", err) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index fcd4a94..ad9e483 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ProofOpDomino = "test:domino" @@ -29,8 +29,8 @@ func NewDominoOp(key, input, output string) DominoOp { } } -func (dop DominoOp) ProofOp() tmcrypto.ProofOp { - dopb := tmcrypto.DominoOp{ +func (dop DominoOp) ProofOp() cmtcrypto.ProofOp { + dopb := cmtcrypto.DominoOp{ Key: dop.key, Input: dop.Input, Output: dop.Output, @@ -40,7 +40,7 @@ func (dop DominoOp) ProofOp() tmcrypto.ProofOp { panic(err) } - return tmcrypto.ProofOp{ + return cmtcrypto.ProofOp{ Type: ProofOpDomino, Key: []byte(dop.key), Data: bz, diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 842dc82..bae6849 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/tendermint/tendermint/crypto/tmhash" - tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) const ProofOpValue = "simple:v" @@ -13,7 +13,7 @@ const ProofOpValue = "simple:v" // ValueOp takes a key and a single value as argument and // produces the root hash. The corresponding tree structure is // the SimpleMap tree. SimpleMap takes a Hasher, and currently -// Tendermint uses tmhash. SimpleValueOp should support +// CometBFT uses tmhash. SimpleValueOp should support // the hash function as used in tmhash. TODO support // additional hash functions here as options/args to this // operator. @@ -37,11 +37,11 @@ func NewValueOp(key []byte, proof *Proof) ValueOp { } } -func ValueOpDecoder(pop tmcrypto.ProofOp) (ProofOperator, error) { +func ValueOpDecoder(pop cmtcrypto.ProofOp) (ProofOperator, error) { if pop.Type != ProofOpValue { return nil, fmt.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpValue) } - var pbop tmcrypto.ValueOp // a bit strange as we'll discard this, but it works. + var pbop cmtcrypto.ValueOp // a bit strange as we'll discard this, but it works. err := pbop.Unmarshal(pop.Data) if err != nil { return nil, fmt.Errorf("decoding ProofOp.Data into ValueOp: %w", err) @@ -54,8 +54,8 @@ func ValueOpDecoder(pop tmcrypto.ProofOp) (ProofOperator, error) { return NewValueOp(pop.Key, sp), nil } -func (op ValueOp) ProofOp() tmcrypto.ProofOp { - pbval := tmcrypto.ValueOp{ +func (op ValueOp) ProofOp() cmtcrypto.ProofOp { + pbval := cmtcrypto.ValueOp{ Key: op.key, Proof: op.Proof.ToProto(), } @@ -63,7 +63,7 @@ func (op ValueOp) ProofOp() tmcrypto.ProofOp { if err != nil { panic(err) } - return tmcrypto.ProofOp{ + return cmtcrypto.ProofOp{ Type: ProofOpValue, Key: op.key, Data: bz, diff --git a/crypto/merkle/tree_test.go b/crypto/merkle/tree_test.go index bdf4698..4c05b7a 100644 --- a/crypto/merkle/tree_test.go +++ b/crypto/merkle/tree_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmrand "github.com/tendermint/tendermint/libs/rand" + cmtrand "github.com/tendermint/tendermint/libs/rand" . "github.com/tendermint/tendermint/libs/test" "github.com/tendermint/tendermint/crypto/tmhash" @@ -54,7 +54,7 @@ func TestProof(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(tmrand.Bytes(tmhash.Size)) + items[i] = testItem(cmtrand.Bytes(tmhash.Size)) } rootHash = HashFromByteSlices(items) @@ -78,7 +78,7 @@ func TestProof(t *testing.T) { // Trail too long should make it fail origAunts := proof.Aunts - proof.Aunts = append(proof.Aunts, tmrand.Bytes(32)) + proof.Aunts = append(proof.Aunts, cmtrand.Bytes(32)) err = proof.Verify(rootHash, item) require.Error(t, err, "Expected verification to fail for wrong trail length") @@ -107,7 +107,7 @@ func TestHashAlternatives(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(tmrand.Bytes(tmhash.Size)) + items[i] = testItem(cmtrand.Bytes(tmhash.Size)) } rootHash1 := HashFromByteSlicesIterative(items) @@ -120,7 +120,7 @@ func BenchmarkHashAlternatives(b *testing.B) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(tmrand.Bytes(tmhash.Size)) + items[i] = testItem(cmtrand.Bytes(tmhash.Size)) } b.ResetTimer() From 0a18ec3dc067670c2be033a38c0620bc6dd21d33 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 13 Apr 2023 08:28:38 -0300 Subject: [PATCH 82/86] crypto/merkle: Add error handling (backport #558) (#710) * crypto/merkle: Add error handling (#558) * porting pr * Disallow nil root hashes * linting * making computeRootHash private * Update release notes. * Update .changelog/unreleased/breaking-changes/558-tm10011.md Co-authored-by: Thane Thomson * Wrapping the private function into a public one with the old signature. * update comment * remove dependency on amino * Update .changelog/unreleased/breaking-changes/558-tm10011.md Co-authored-by: Thane Thomson --------- Co-authored-by: Thane Thomson (cherry picked from commit d067b740ec543718ba36865375acdfbb7d1116cd) # Conflicts: # crypto/merkle/proof.go # crypto/merkle/proof_test.go * fix conflicts --------- Co-authored-by: Lasaro --- crypto/merkle/proof.go | 48 ++++++++++++++++++++++++------------ crypto/merkle/proof_test.go | 25 +++++++++++++++++++ crypto/merkle/proof_value.go | 6 ++++- 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index e3dd0f9..1084bdf 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -50,25 +50,40 @@ func ProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*Proof) { // Verify that the Proof proves the root hash. // Check sp.Index/sp.Total manually if needed func (sp *Proof) Verify(rootHash []byte, leaf []byte) error { - leafHash := leafHash(leaf) + if rootHash == nil { + return fmt.Errorf("invalid root hash: cannot be nil") + } if sp.Total < 0 { return errors.New("proof total must be positive") } if sp.Index < 0 { return errors.New("proof index cannot be negative") } + leafHash := leafHash(leaf) if !bytes.Equal(sp.LeafHash, leafHash) { return fmt.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) } - computedHash := sp.ComputeRootHash() + computedHash, err := sp.computeRootHash() + if err != nil { + return fmt.Errorf("compute root hash: %w", err) + } if !bytes.Equal(computedHash, rootHash) { return fmt.Errorf("invalid root hash: wanted %X got %X", rootHash, computedHash) } return nil } -// Compute the root hash given a leaf hash. Does not verify the result. +// Compute the root hash given a leaf hash. Panics in case of errors. func (sp *Proof) ComputeRootHash() []byte { + computedHash, err := sp.computeRootHash() + if err != nil { + panic(fmt.Errorf("ComputeRootHash errored %w", err)) + } + return computedHash +} + +// Compute the root hash given a leaf hash. +func (sp *Proof) computeRootHash() ([]byte, error) { return computeHashFromAunts( sp.Index, sp.Total, @@ -148,35 +163,36 @@ func ProofFromProto(pb *cmtcrypto.Proof) (*Proof, error) { // Use the leafHash and innerHashes to get the root merkle hash. // If the length of the innerHashes slice isn't exactly correct, the result is nil. // Recursive impl. -func computeHashFromAunts(index, total int64, leafHash []byte, innerHashes [][]byte) []byte { +func computeHashFromAunts(index, total int64, leafHash []byte, innerHashes [][]byte) ([]byte, error) { if index >= total || index < 0 || total <= 0 { - return nil + return nil, fmt.Errorf("invalid index %d and/or total %d", index, total) } switch total { case 0: panic("Cannot call computeHashFromAunts() with 0 total") case 1: if len(innerHashes) != 0 { - return nil + return nil, fmt.Errorf("unexpected inner hashes") } - return leafHash + return leafHash, nil default: if len(innerHashes) == 0 { - return nil + return nil, fmt.Errorf("expected at least one inner hash") } numLeft := getSplitPoint(total) if index < numLeft { - leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) - if leftHash == nil { - return nil + leftHash, err := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if err != nil { + return nil, err } - return innerHash(leftHash, innerHashes[len(innerHashes)-1]) + + return innerHash(leftHash, innerHashes[len(innerHashes)-1]), nil } - rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) - if rightHash == nil { - return nil + rightHash, err := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if err != nil { + return nil, err } - return innerHash(innerHashes[len(innerHashes)-1], rightHash) + return innerHash(innerHashes[len(innerHashes)-1], rightHash), nil } } diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index b07c853..b865e0b 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -1,6 +1,7 @@ package merkle import ( + "bytes" "errors" "fmt" "testing" @@ -8,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto/tmhash" cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) @@ -198,3 +200,26 @@ func TestVoteProtobuf(t *testing.T) { } } } + +// TestVsa2022_100 verifies https://blog.verichains.io/p/vsa-2022-100-tendermint-forging-membership-proof +func TestVsa2022_100(t *testing.T) { + // a fake key-value pair and its hash + key := []byte{0x13} + value := []byte{0x37} + vhash := tmhash.Sum(value) + bz := new(bytes.Buffer) + _ = encodeByteSlice(bz, key) + _ = encodeByteSlice(bz, vhash) + kvhash := tmhash.Sum(append([]byte{0}, bz.Bytes()...)) + + // the malicious `op` + op := NewValueOp( + key, + &Proof{LeafHash: kvhash}, + ) + + // the nil root + var root []byte + + assert.NotNil(t, ProofOperators{op}.Verify(root, "/"+string(key), [][]byte{value})) +} diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index bae6849..9648d07 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -93,8 +93,12 @@ func (op ValueOp) Run(args [][]byte) ([][]byte, error) { return nil, fmt.Errorf("leaf hash mismatch: want %X got %X", op.Proof.LeafHash, kvhash) } + rootHash, err := op.Proof.computeRootHash() + if err != nil { + return nil, err + } return [][]byte{ - op.Proof.ComputeRootHash(), + rootHash, }, nil } From 6b4bd9df89e16642b8e6048e4d731db62791e513 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Mon, 24 Jul 2023 12:59:05 +0200 Subject: [PATCH 83/86] Update to use Go 1.20 (backport #405) (#419) * Update to use Go 1.20 (#405) * Bump minimum Go version to 1.20 in go.mod Signed-off-by: Thane Thomson * Bump minimum Go version in CI/test infra Signed-off-by: Thane Thomson * Bump Go version in docs Signed-off-by: Thane Thomson * Add changelog entry Signed-off-by: Thane Thomson * Ignore lint Signed-off-by: Thane Thomson * Fix math/rand usages in tests Signed-off-by: Thane Thomson * Fix lint: use local random source in test Signed-off-by: Thane Thomson * Rename variable for clarity Signed-off-by: Thane Thomson * ci: Bump version of golangci-lint to latest Signed-off-by: Thane Thomson --------- Signed-off-by: Thane Thomson (cherry picked from commit 200784acb3d9b3b955b237d038411534a5480b17) * Resolve conflicts Signed-off-by: Thane Thomson * Remove use of deprecated global random seed Signed-off-by: Thane Thomson * Attempt to appease linter Signed-off-by: Thane Thomson * Attempt 2 to appease linter Signed-off-by: Thane Thomson --------- Signed-off-by: Thane Thomson Co-authored-by: Thane Thomson --- crypto/merkle/proof_key_path_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crypto/merkle/proof_key_path_test.go b/crypto/merkle/proof_key_path_test.go index 22e3e21..25a61af 100644 --- a/crypto/merkle/proof_key_path_test.go +++ b/crypto/merkle/proof_key_path_test.go @@ -3,6 +3,7 @@ package merkle import ( // it is ok to use math/rand here: we do not need a cryptographically secure random // number generator here and we can run the tests a bit faster + crand "crypto/rand" "math/rand" "testing" @@ -26,7 +27,7 @@ func TestKeyPath(t *testing.T) { keys[i][j] = alphanum[rand.Intn(len(alphanum))] } case KeyEncodingHex: - rand.Read(keys[i]) + _, _ = crand.Read(keys[i]) default: panic("Unexpected encoding") } From 44cc9b79a9dad393d0b14a6155fa98e2cdcbd50e Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 22 Aug 2023 19:10:16 +0200 Subject: [PATCH 84/86] chore!: migrate to cometbft fork --- crypto/merkle/hash.go | 2 +- crypto/merkle/proof.go | 4 ++-- crypto/merkle/proof_op.go | 2 +- crypto/merkle/proof_test.go | 4 ++-- crypto/merkle/proof_value.go | 4 ++-- crypto/merkle/rfc6962_test.go | 2 +- crypto/merkle/tree_test.go | 6 +++--- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index d45130f..e5f7318 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -1,7 +1,7 @@ package merkle import ( - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/cometbft/cometbft/crypto/tmhash" ) // TODO: make these have a large predefined capacity diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 1084bdf..85b2db1 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" - "github.com/tendermint/tendermint/crypto/tmhash" - cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + "github.com/cometbft/cometbft/crypto/tmhash" + cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" ) const ( diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index fdf987a..17c107a 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" ) //---------------------------------------- diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 52eda9b..07af062 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/crypto/tmhash" - cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + "github.com/cometbft/cometbft/crypto/tmhash" + cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" ) const ProofOpDomino = "test:domino" diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 9648d07..5cc188c 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" - "github.com/tendermint/tendermint/crypto/tmhash" - cmtcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + "github.com/cometbft/cometbft/crypto/tmhash" + cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" ) const ProofOpValue = "simple:v" diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go index c762cda..dd0c817 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/crypto/merkle/rfc6962_test.go @@ -20,7 +20,7 @@ import ( "encoding/hex" "testing" - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/cometbft/cometbft/crypto/tmhash" ) func TestRFC6962Hasher(t *testing.T) { diff --git a/crypto/merkle/tree_test.go b/crypto/merkle/tree_test.go index 4c05b7a..72f1402 100644 --- a/crypto/merkle/tree_test.go +++ b/crypto/merkle/tree_test.go @@ -7,10 +7,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - cmtrand "github.com/tendermint/tendermint/libs/rand" - . "github.com/tendermint/tendermint/libs/test" + cmtrand "github.com/cometbft/cometbft/libs/rand" + . "github.com/cometbft/cometbft/libs/test" - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/cometbft/cometbft/crypto/tmhash" ) type testItem []byte From ba69e0d7c53ac95393d81bc7f6c93fdf797191e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90=E1=BB=97=20Vi=E1=BB=87t=20Ho=C3=A0ng?= Date: Thu, 14 Dec 2023 02:23:56 +0700 Subject: [PATCH 85/86] chore: fix typos (#1149) ## Description This PR will fix every typos exist on current repo. After this we can add another CI to check for typo per PR submit. #### PR checklist - [ ] Tests written/updated - [ ] Changelog entry added in `.changelog` (we use [unclog](https://github.com/informalsystems/unclog) to manage our changelog) - [x] Updated relevant documentation (`docs/` or `spec/`) and code comments --------- Co-authored-by: Rootul P --- crypto/merkle/proof.go | 2 +- crypto/merkle/proof_op.go | 2 +- crypto/merkle/tree_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 85b2db1..08dc2c2 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -12,7 +12,7 @@ import ( const ( // MaxAunts is the maximum number of aunts that can be included in a Proof. // This corresponds to a tree of size 2^100, which should be sufficient for all conceivable purposes. - // This maximum helps prevent Denial-of-Service attacks by limitting the size of the proofs. + // This maximum helps prevent Denial-of-Service attacks by limiting the size of the proofs. MaxAunts = 100 ) diff --git a/crypto/merkle/proof_op.go b/crypto/merkle/proof_op.go index 17c107a..36fdc06 100644 --- a/crypto/merkle/proof_op.go +++ b/crypto/merkle/proof_op.go @@ -149,7 +149,7 @@ func (prt *ProofRuntime) VerifyValueFromKeys(proof *cmtcrypto.ProofOps, root []b return prt.VerifyFromKeys(proof, root, keys, [][]byte{value}) } -// TODO In the long run we'll need a method of classifcation of ops, +// TODO In the long run we'll need a method of classification of ops, // whether existence or absence or perhaps a third? func (prt *ProofRuntime) VerifyAbsence(proof *cmtcrypto.ProofOps, root []byte, keypath string) (err error) { return prt.Verify(proof, root, keypath, nil) diff --git a/crypto/merkle/tree_test.go b/crypto/merkle/tree_test.go index 72f1402..4b90186 100644 --- a/crypto/merkle/tree_test.go +++ b/crypto/merkle/tree_test.go @@ -68,7 +68,7 @@ func TestProof(t *testing.T) { proof := proofs[i] // Check total/index - require.EqualValues(t, proof.Index, i, "Unmatched indicies: %d vs %d", proof.Index, i) + require.EqualValues(t, proof.Index, i, "Unmatched indices: %d vs %d", proof.Index, i) require.EqualValues(t, proof.Total, total, "Unmatched totals: %d vs %d", proof.Total, total) From fdf1e920e58a838598c959315a5c50b7f0ee54ce Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 19 Dec 2023 14:55:15 +0100 Subject: [PATCH 86/86] init go mod, copy proto files and modify directory path --- crypto/merkle/hash.go | 26 - go.work | 6 + go.work.sum | 3 + {crypto/merkle => merkle}/README.md | 4 +- {crypto/merkle => merkle}/doc.go | 0 merkle/go.mod | 15 + merkle/go.sum | 18 + merkle/hash.go | 31 ++ {crypto/merkle => merkle}/proof.go | 18 +- {crypto/merkle => merkle}/proof_key_path.go | 0 .../merkle => merkle}/proof_key_path_test.go | 0 {crypto/merkle => merkle}/proof_op.go | 23 +- {crypto/merkle => merkle}/proof_test.go | 17 +- {crypto/merkle => merkle}/proof_value.go | 21 +- merkle/proto/buf.gen.yaml | 6 + merkle/proto/gen/merkle/v1/proof.pb.go | 478 ++++++++++++++++++ merkle/proto/merkle/v1/proof.proto | 39 ++ {crypto/merkle => merkle}/rfc6962_test.go | 11 +- {crypto/merkle => merkle}/tree.go | 0 {crypto/merkle => merkle}/tree_test.go | 49 +- {crypto/merkle => merkle}/types.go | 0 21 files changed, 683 insertions(+), 82 deletions(-) delete mode 100644 crypto/merkle/hash.go create mode 100644 go.work create mode 100644 go.work.sum rename {crypto/merkle => merkle}/README.md (67%) rename {crypto/merkle => merkle}/doc.go (100%) create mode 100644 merkle/go.mod create mode 100644 merkle/go.sum create mode 100644 merkle/hash.go rename {crypto/merkle => merkle}/proof.go (94%) rename {crypto/merkle => merkle}/proof_key_path.go (100%) rename {crypto/merkle => merkle}/proof_key_path_test.go (100%) rename {crypto/merkle => merkle}/proof_op.go (84%) rename {crypto/merkle => merkle}/proof_test.go (95%) rename {crypto/merkle => merkle}/proof_value.go (82%) create mode 100644 merkle/proto/buf.gen.yaml create mode 100644 merkle/proto/gen/merkle/v1/proof.pb.go create mode 100644 merkle/proto/merkle/v1/proof.proto rename {crypto/merkle => merkle}/rfc6962_test.go (95%) rename {crypto/merkle => merkle}/tree.go (100%) rename {crypto/merkle => merkle}/tree_test.go (79%) rename {crypto/merkle => merkle}/types.go (100%) diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go deleted file mode 100644 index e5f7318..0000000 --- a/crypto/merkle/hash.go +++ /dev/null @@ -1,26 +0,0 @@ -package merkle - -import ( - "github.com/cometbft/cometbft/crypto/tmhash" -) - -// TODO: make these have a large predefined capacity -var ( - leafPrefix = []byte{0} - innerPrefix = []byte{1} -) - -// returns tmhash() -func emptyHash() []byte { - return tmhash.Sum([]byte{}) -} - -// returns tmhash(0x00 || leaf) -func leafHash(leaf []byte) []byte { - return tmhash.Sum(append(leafPrefix, leaf...)) -} - -// returns tmhash(0x01 || left || right) -func innerHash(left []byte, right []byte) []byte { - return tmhash.Sum(append(innerPrefix, append(left, right...)...)) -} diff --git a/go.work b/go.work new file mode 100644 index 0000000..b9dc24f --- /dev/null +++ b/go.work @@ -0,0 +1,6 @@ +go 1.21.5 + +use ( + . + ./merkle +) \ No newline at end of file diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..c5b581a --- /dev/null +++ b/go.work.sum @@ -0,0 +1,3 @@ +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= diff --git a/crypto/merkle/README.md b/merkle/README.md similarity index 67% rename from crypto/merkle/README.md rename to merkle/README.md index 16b1abb..00d6124 100644 --- a/crypto/merkle/README.md +++ b/merkle/README.md @@ -1,4 +1,6 @@ # Merkle Tree -For smaller static data structures that don't require immutable snapshots or mutability; +For smaller static data structures that don't require immutable snapshots or mutability; for instance the transactions and validation signatures of a block can be hashed using this simple merkle tree logic. + +This is a forked copy of github.com/cometbft/cometbft/crypto/merkle diff --git a/crypto/merkle/doc.go b/merkle/doc.go similarity index 100% rename from crypto/merkle/doc.go rename to merkle/doc.go diff --git a/merkle/go.mod b/merkle/go.mod new file mode 100644 index 0000000..3acc43d --- /dev/null +++ b/merkle/go.mod @@ -0,0 +1,15 @@ +module github.com/celestiaorg/go-square/merkle + +go 1.21.5 + +require ( + github.com/stretchr/testify v1.8.4 + google.golang.org/protobuf v1.31.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/merkle/go.sum b/merkle/go.sum new file mode 100644 index 0000000..0d44549 --- /dev/null +++ b/merkle/go.sum @@ -0,0 +1,18 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/merkle/hash.go b/merkle/hash.go new file mode 100644 index 0000000..973629e --- /dev/null +++ b/merkle/hash.go @@ -0,0 +1,31 @@ +package merkle + +import ( + "crypto/sha256" +) + +// TODO: make these have a large predefined capacity +var ( + leafPrefix = []byte{0} + innerPrefix = []byte{1} +) + +// returns empty sha256 hash +func emptyHash() []byte { + return hash([]byte{}) +} + +// returns sha256(0x00 || leaf) +func leafHash(leaf []byte) []byte { + return hash(append(leafPrefix, leaf...)) +} + +// returns sha256(0x01 || left || right) +func innerHash(left []byte, right []byte) []byte { + return hash(append(innerPrefix, append(left, right...)...)) +} + +func hash(bz []byte) []byte { + h := sha256.Sum256(bz) + return h[:] +} diff --git a/crypto/merkle/proof.go b/merkle/proof.go similarity index 94% rename from crypto/merkle/proof.go rename to merkle/proof.go index 08dc2c2..2491b47 100644 --- a/crypto/merkle/proof.go +++ b/merkle/proof.go @@ -2,11 +2,11 @@ package merkle import ( "bytes" + "crypto/sha256" "errors" "fmt" - "github.com/cometbft/cometbft/crypto/tmhash" - cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" + wire "github.com/celestiaorg/go-square/merkle/proto/gen/merkle/v1" ) const ( @@ -117,25 +117,25 @@ func (sp *Proof) ValidateBasic() error { if sp.Index < 0 { return errors.New("negative Index") } - if len(sp.LeafHash) != tmhash.Size { - return fmt.Errorf("expected LeafHash size to be %d, got %d", tmhash.Size, len(sp.LeafHash)) + if len(sp.LeafHash) != sha256.Size { + return fmt.Errorf("expected LeafHash size to be %d, got %d", sha256.Size, len(sp.LeafHash)) } if len(sp.Aunts) > MaxAunts { return fmt.Errorf("expected no more than %d aunts, got %d", MaxAunts, len(sp.Aunts)) } for i, auntHash := range sp.Aunts { - if len(auntHash) != tmhash.Size { - return fmt.Errorf("expected Aunts#%d size to be %d, got %d", i, tmhash.Size, len(auntHash)) + if len(auntHash) != sha256.Size { + return fmt.Errorf("expected Aunts#%d size to be %d, got %d", i, sha256.Size, len(auntHash)) } } return nil } -func (sp *Proof) ToProto() *cmtcrypto.Proof { +func (sp *Proof) ToProto() *wire.Proof { if sp == nil { return nil } - pb := new(cmtcrypto.Proof) + pb := new(wire.Proof) pb.Total = sp.Total pb.Index = sp.Index @@ -145,7 +145,7 @@ func (sp *Proof) ToProto() *cmtcrypto.Proof { return pb } -func ProofFromProto(pb *cmtcrypto.Proof) (*Proof, error) { +func ProofFromProto(pb *wire.Proof) (*Proof, error) { if pb == nil { return nil, errors.New("nil proof") } diff --git a/crypto/merkle/proof_key_path.go b/merkle/proof_key_path.go similarity index 100% rename from crypto/merkle/proof_key_path.go rename to merkle/proof_key_path.go diff --git a/crypto/merkle/proof_key_path_test.go b/merkle/proof_key_path_test.go similarity index 100% rename from crypto/merkle/proof_key_path_test.go rename to merkle/proof_key_path_test.go diff --git a/crypto/merkle/proof_op.go b/merkle/proof_op.go similarity index 84% rename from crypto/merkle/proof_op.go rename to merkle/proof_op.go index 36fdc06..122b600 100644 --- a/crypto/merkle/proof_op.go +++ b/merkle/proof_op.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" + wire "github.com/celestiaorg/go-square/merkle/proto/gen/merkle/v1" ) //---------------------------------------- @@ -21,7 +21,7 @@ import ( type ProofOperator interface { Run([][]byte) ([][]byte, error) GetKey() []byte - ProofOp() cmtcrypto.ProofOp + ProofOp() wire.ProofOp } //---------------------------------------- @@ -101,7 +101,7 @@ func (poz ProofOperators) VerifyFromKeys(root []byte, keys [][]byte, args [][]by //---------------------------------------- // ProofRuntime - main entrypoint -type OpDecoder func(cmtcrypto.ProofOp) (ProofOperator, error) +type OpDecoder func(*wire.ProofOp) (ProofOperator, error) type ProofRuntime struct { decoders map[string]OpDecoder @@ -121,15 +121,18 @@ func (prt *ProofRuntime) RegisterOpDecoder(typ string, dec OpDecoder) { prt.decoders[typ] = dec } -func (prt *ProofRuntime) Decode(pop cmtcrypto.ProofOp) (ProofOperator, error) { +func (prt *ProofRuntime) Decode(pop *wire.ProofOp) (ProofOperator, error) { decoder := prt.decoders[pop.Type] if decoder == nil { return nil, fmt.Errorf("unrecognized proof type %v", pop.Type) } + if pop == nil { + return nil, errors.New("nil ProofOp") + } return decoder(pop) } -func (prt *ProofRuntime) DecodeProof(proof *cmtcrypto.ProofOps) (ProofOperators, error) { +func (prt *ProofRuntime) DecodeProof(proof *wire.ProofOps) (ProofOperators, error) { poz := make(ProofOperators, 0, len(proof.Ops)) for _, pop := range proof.Ops { operator, err := prt.Decode(pop) @@ -141,21 +144,21 @@ func (prt *ProofRuntime) DecodeProof(proof *cmtcrypto.ProofOps) (ProofOperators, return poz, nil } -func (prt *ProofRuntime) VerifyValue(proof *cmtcrypto.ProofOps, root []byte, keypath string, value []byte) (err error) { +func (prt *ProofRuntime) VerifyValue(proof *wire.ProofOps, root []byte, keypath string, value []byte) (err error) { return prt.Verify(proof, root, keypath, [][]byte{value}) } -func (prt *ProofRuntime) VerifyValueFromKeys(proof *cmtcrypto.ProofOps, root []byte, keys [][]byte, value []byte) (err error) { +func (prt *ProofRuntime) VerifyValueFromKeys(proof *wire.ProofOps, root []byte, keys [][]byte, value []byte) (err error) { return prt.VerifyFromKeys(proof, root, keys, [][]byte{value}) } // TODO In the long run we'll need a method of classification of ops, // whether existence or absence or perhaps a third? -func (prt *ProofRuntime) VerifyAbsence(proof *cmtcrypto.ProofOps, root []byte, keypath string) (err error) { +func (prt *ProofRuntime) VerifyAbsence(proof *wire.ProofOps, root []byte, keypath string) (err error) { return prt.Verify(proof, root, keypath, nil) } -func (prt *ProofRuntime) Verify(proof *cmtcrypto.ProofOps, root []byte, keypath string, args [][]byte) (err error) { +func (prt *ProofRuntime) Verify(proof *wire.ProofOps, root []byte, keypath string, args [][]byte) (err error) { poz, err := prt.DecodeProof(proof) if err != nil { return fmt.Errorf("decoding proof: %w", err) @@ -166,7 +169,7 @@ func (prt *ProofRuntime) Verify(proof *cmtcrypto.ProofOps, root []byte, keypath // VerifyFromKeys performs the same verification logic as the normal Verify // method, except it does not perform any processing on the keypath. This is // useful when using keys that have split or escape points as a part of the key. -func (prt *ProofRuntime) VerifyFromKeys(proof *cmtcrypto.ProofOps, root []byte, keys [][]byte, args [][]byte) (err error) { +func (prt *ProofRuntime) VerifyFromKeys(proof *wire.ProofOps, root []byte, keys [][]byte, args [][]byte) (err error) { poz, err := prt.DecodeProof(proof) if err != nil { return fmt.Errorf("decoding proof: %w", err) diff --git a/crypto/merkle/proof_test.go b/merkle/proof_test.go similarity index 95% rename from crypto/merkle/proof_test.go rename to merkle/proof_test.go index 07af062..abaea7d 100644 --- a/crypto/merkle/proof_test.go +++ b/merkle/proof_test.go @@ -6,11 +6,10 @@ import ( "fmt" "testing" + wire "github.com/celestiaorg/go-square/merkle/proto/gen/merkle/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/cometbft/cometbft/crypto/tmhash" - cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" + "google.golang.org/protobuf/proto" ) const ProofOpDomino = "test:domino" @@ -31,18 +30,18 @@ func NewDominoOp(key, input, output string) DominoOp { } } -func (dop DominoOp) ProofOp() cmtcrypto.ProofOp { - dopb := cmtcrypto.DominoOp{ +func (dop DominoOp) ProofOp() wire.ProofOp { + dopb := &wire.DominoOp{ Key: dop.key, Input: dop.Input, Output: dop.Output, } - bz, err := dopb.Marshal() + bz, err := proto.Marshal(dopb) if err != nil { panic(err) } - return cmtcrypto.ProofOp{ + return wire.ProofOp{ Type: ProofOpDomino, Key: []byte(dop.key), Data: bz, @@ -270,11 +269,11 @@ func TestVsa2022_100(t *testing.T) { // a fake key-value pair and its hash key := []byte{0x13} value := []byte{0x37} - vhash := tmhash.Sum(value) + vhash := hash(value) bz := new(bytes.Buffer) _ = encodeByteSlice(bz, key) _ = encodeByteSlice(bz, vhash) - kvhash := tmhash.Sum(append([]byte{0}, bz.Bytes()...)) + kvhash := hash(append([]byte{0}, bz.Bytes()...)) // the malicious `op` op := NewValueOp( diff --git a/crypto/merkle/proof_value.go b/merkle/proof_value.go similarity index 82% rename from crypto/merkle/proof_value.go rename to merkle/proof_value.go index 5cc188c..5ef8933 100644 --- a/crypto/merkle/proof_value.go +++ b/merkle/proof_value.go @@ -2,10 +2,11 @@ package merkle import ( "bytes" + "crypto/sha256" "fmt" - "github.com/cometbft/cometbft/crypto/tmhash" - cmtcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" + wire "github.com/celestiaorg/go-square/merkle/proto/gen/merkle/v1" + "google.golang.org/protobuf/proto" ) const ProofOpValue = "simple:v" @@ -37,12 +38,12 @@ func NewValueOp(key []byte, proof *Proof) ValueOp { } } -func ValueOpDecoder(pop cmtcrypto.ProofOp) (ProofOperator, error) { +func ValueOpDecoder(pop *wire.ProofOp) (ProofOperator, error) { if pop.Type != ProofOpValue { return nil, fmt.Errorf("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpValue) } - var pbop cmtcrypto.ValueOp // a bit strange as we'll discard this, but it works. - err := pbop.Unmarshal(pop.Data) + var pbop wire.ValueOp // a bit strange as we'll discard this, but it works. + err := proto.Unmarshal(pop.Data, &pbop) if err != nil { return nil, fmt.Errorf("decoding ProofOp.Data into ValueOp: %w", err) } @@ -54,16 +55,16 @@ func ValueOpDecoder(pop cmtcrypto.ProofOp) (ProofOperator, error) { return NewValueOp(pop.Key, sp), nil } -func (op ValueOp) ProofOp() cmtcrypto.ProofOp { - pbval := cmtcrypto.ValueOp{ +func (op ValueOp) ProofOp() wire.ProofOp { + pbval := &wire.ValueOp{ Key: op.key, Proof: op.Proof.ToProto(), } - bz, err := pbval.Marshal() + bz, err := proto.Marshal(pbval) if err != nil { panic(err) } - return cmtcrypto.ProofOp{ + return wire.ProofOp{ Type: ProofOpValue, Key: op.key, Data: bz, @@ -79,7 +80,7 @@ func (op ValueOp) Run(args [][]byte) ([][]byte, error) { return nil, fmt.Errorf("expected 1 arg, got %v", len(args)) } value := args[0] - hasher := tmhash.New() + hasher := sha256.New() hasher.Write(value) vhash := hasher.Sum(nil) diff --git a/merkle/proto/buf.gen.yaml b/merkle/proto/buf.gen.yaml new file mode 100644 index 0000000..d887b47 --- /dev/null +++ b/merkle/proto/buf.gen.yaml @@ -0,0 +1,6 @@ +version: v1 +plugins: + - plugin: buf.build/protocolbuffers/go + out: gen + opt: + - paths=source_relative \ No newline at end of file diff --git a/merkle/proto/gen/merkle/v1/proof.pb.go b/merkle/proto/gen/merkle/v1/proof.pb.go new file mode 100644 index 0000000..f26aa80 --- /dev/null +++ b/merkle/proto/gen/merkle/v1/proof.pb.go @@ -0,0 +1,478 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc (unknown) +// source: merkle/v1/proof.proto + +package merkle + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Proof struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Total int64 `protobuf:"varint,1,opt,name=total,proto3" json:"total,omitempty"` + Index int64 `protobuf:"varint,2,opt,name=index,proto3" json:"index,omitempty"` + LeafHash []byte `protobuf:"bytes,3,opt,name=leaf_hash,json=leafHash,proto3" json:"leaf_hash,omitempty"` + Aunts [][]byte `protobuf:"bytes,4,rep,name=aunts,proto3" json:"aunts,omitempty"` +} + +func (x *Proof) Reset() { + *x = Proof{} + if protoimpl.UnsafeEnabled { + mi := &file_merkle_v1_proof_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Proof) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Proof) ProtoMessage() {} + +func (x *Proof) ProtoReflect() protoreflect.Message { + mi := &file_merkle_v1_proof_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Proof.ProtoReflect.Descriptor instead. +func (*Proof) Descriptor() ([]byte, []int) { + return file_merkle_v1_proof_proto_rawDescGZIP(), []int{0} +} + +func (x *Proof) GetTotal() int64 { + if x != nil { + return x.Total + } + return 0 +} + +func (x *Proof) GetIndex() int64 { + if x != nil { + return x.Index + } + return 0 +} + +func (x *Proof) GetLeafHash() []byte { + if x != nil { + return x.LeafHash + } + return nil +} + +func (x *Proof) GetAunts() [][]byte { + if x != nil { + return x.Aunts + } + return nil +} + +type ValueOp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Encoded in ProofOp.Key. + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // To encode in ProofOp.Data + Proof *Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` +} + +func (x *ValueOp) Reset() { + *x = ValueOp{} + if protoimpl.UnsafeEnabled { + mi := &file_merkle_v1_proof_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ValueOp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ValueOp) ProtoMessage() {} + +func (x *ValueOp) ProtoReflect() protoreflect.Message { + mi := &file_merkle_v1_proof_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ValueOp.ProtoReflect.Descriptor instead. +func (*ValueOp) Descriptor() ([]byte, []int) { + return file_merkle_v1_proof_proto_rawDescGZIP(), []int{1} +} + +func (x *ValueOp) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} + +func (x *ValueOp) GetProof() *Proof { + if x != nil { + return x.Proof + } + return nil +} + +type DominoOp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Input string `protobuf:"bytes,2,opt,name=input,proto3" json:"input,omitempty"` + Output string `protobuf:"bytes,3,opt,name=output,proto3" json:"output,omitempty"` +} + +func (x *DominoOp) Reset() { + *x = DominoOp{} + if protoimpl.UnsafeEnabled { + mi := &file_merkle_v1_proof_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DominoOp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DominoOp) ProtoMessage() {} + +func (x *DominoOp) ProtoReflect() protoreflect.Message { + mi := &file_merkle_v1_proof_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DominoOp.ProtoReflect.Descriptor instead. +func (*DominoOp) Descriptor() ([]byte, []int) { + return file_merkle_v1_proof_proto_rawDescGZIP(), []int{2} +} + +func (x *DominoOp) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *DominoOp) GetInput() string { + if x != nil { + return x.Input + } + return "" +} + +func (x *DominoOp) GetOutput() string { + if x != nil { + return x.Output + } + return "" +} + +// ProofOp defines an operation used for calculating Merkle root +// The data could be arbitrary format, providing necessary data +// for example neighbouring node hash +type ProofOp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *ProofOp) Reset() { + *x = ProofOp{} + if protoimpl.UnsafeEnabled { + mi := &file_merkle_v1_proof_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProofOp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProofOp) ProtoMessage() {} + +func (x *ProofOp) ProtoReflect() protoreflect.Message { + mi := &file_merkle_v1_proof_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProofOp.ProtoReflect.Descriptor instead. +func (*ProofOp) Descriptor() ([]byte, []int) { + return file_merkle_v1_proof_proto_rawDescGZIP(), []int{3} +} + +func (x *ProofOp) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *ProofOp) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} + +func (x *ProofOp) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +// ProofOps is Merkle proof defined by the list of ProofOps +type ProofOps struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Ops []*ProofOp `protobuf:"bytes,1,rep,name=ops,proto3" json:"ops,omitempty"` +} + +func (x *ProofOps) Reset() { + *x = ProofOps{} + if protoimpl.UnsafeEnabled { + mi := &file_merkle_v1_proof_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProofOps) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProofOps) ProtoMessage() {} + +func (x *ProofOps) ProtoReflect() protoreflect.Message { + mi := &file_merkle_v1_proof_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProofOps.ProtoReflect.Descriptor instead. +func (*ProofOps) Descriptor() ([]byte, []int) { + return file_merkle_v1_proof_proto_rawDescGZIP(), []int{4} +} + +func (x *ProofOps) GetOps() []*ProofOp { + if x != nil { + return x.Ops + } + return nil +} + +var File_merkle_v1_proof_proto protoreflect.FileDescriptor + +var file_merkle_v1_proof_proto_rawDesc = []byte{ + 0x0a, 0x15, 0x6d, 0x65, 0x72, 0x6b, 0x6c, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x6f, + 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x2e, + 0x6d, 0x65, 0x72, 0x6b, 0x6c, 0x65, 0x22, 0x66, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x6f, 0x66, 0x12, + 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, + 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x1b, 0x0a, 0x09, 0x6c, + 0x65, 0x61, 0x66, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, + 0x6c, 0x65, 0x61, 0x66, 0x48, 0x61, 0x73, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x75, 0x6e, 0x74, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x05, 0x61, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x47, + 0x0a, 0x07, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x70, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x70, + 0x72, 0x6f, 0x6f, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x71, 0x75, + 0x61, 0x72, 0x65, 0x2e, 0x6d, 0x65, 0x72, 0x6b, 0x6c, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x6f, 0x66, + 0x52, 0x05, 0x70, 0x72, 0x6f, 0x6f, 0x66, 0x22, 0x4a, 0x0a, 0x08, 0x44, 0x6f, 0x6d, 0x69, 0x6e, + 0x6f, 0x4f, 0x70, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6f, + 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, + 0x70, 0x75, 0x74, 0x22, 0x43, 0x0a, 0x07, 0x50, 0x72, 0x6f, 0x6f, 0x66, 0x4f, 0x70, 0x12, 0x12, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x34, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x6f, + 0x66, 0x4f, 0x70, 0x73, 0x12, 0x28, 0x0a, 0x03, 0x6f, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x2e, 0x6d, 0x65, 0x72, 0x6b, 0x6c, + 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x6f, 0x66, 0x4f, 0x70, 0x52, 0x03, 0x6f, 0x70, 0x73, 0x42, 0x29, + 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x65, 0x6c, + 0x65, 0x73, 0x74, 0x69, 0x61, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x6f, 0x2d, 0x73, 0x71, 0x75, 0x61, + 0x72, 0x65, 0x2f, 0x6d, 0x65, 0x72, 0x6b, 0x6c, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_merkle_v1_proof_proto_rawDescOnce sync.Once + file_merkle_v1_proof_proto_rawDescData = file_merkle_v1_proof_proto_rawDesc +) + +func file_merkle_v1_proof_proto_rawDescGZIP() []byte { + file_merkle_v1_proof_proto_rawDescOnce.Do(func() { + file_merkle_v1_proof_proto_rawDescData = protoimpl.X.CompressGZIP(file_merkle_v1_proof_proto_rawDescData) + }) + return file_merkle_v1_proof_proto_rawDescData +} + +var file_merkle_v1_proof_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_merkle_v1_proof_proto_goTypes = []interface{}{ + (*Proof)(nil), // 0: square.merkle.Proof + (*ValueOp)(nil), // 1: square.merkle.ValueOp + (*DominoOp)(nil), // 2: square.merkle.DominoOp + (*ProofOp)(nil), // 3: square.merkle.ProofOp + (*ProofOps)(nil), // 4: square.merkle.ProofOps +} +var file_merkle_v1_proof_proto_depIdxs = []int32{ + 0, // 0: square.merkle.ValueOp.proof:type_name -> square.merkle.Proof + 3, // 1: square.merkle.ProofOps.ops:type_name -> square.merkle.ProofOp + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_merkle_v1_proof_proto_init() } +func file_merkle_v1_proof_proto_init() { + if File_merkle_v1_proof_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_merkle_v1_proof_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Proof); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_merkle_v1_proof_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ValueOp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_merkle_v1_proof_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DominoOp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_merkle_v1_proof_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProofOp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_merkle_v1_proof_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProofOps); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_merkle_v1_proof_proto_rawDesc, + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_merkle_v1_proof_proto_goTypes, + DependencyIndexes: file_merkle_v1_proof_proto_depIdxs, + MessageInfos: file_merkle_v1_proof_proto_msgTypes, + }.Build() + File_merkle_v1_proof_proto = out.File + file_merkle_v1_proof_proto_rawDesc = nil + file_merkle_v1_proof_proto_goTypes = nil + file_merkle_v1_proof_proto_depIdxs = nil +} diff --git a/merkle/proto/merkle/v1/proof.proto b/merkle/proto/merkle/v1/proof.proto new file mode 100644 index 0000000..47fe221 --- /dev/null +++ b/merkle/proto/merkle/v1/proof.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; +package square.merkle; + +option go_package = "github.com/celestiaorg/go-square/merkle"; + +message Proof { + int64 total = 1; + int64 index = 2; + bytes leaf_hash = 3; + repeated bytes aunts = 4; +} + +message ValueOp { + // Encoded in ProofOp.Key. + bytes key = 1; + + // To encode in ProofOp.Data + Proof proof = 2; +} + +message DominoOp { + string key = 1; + string input = 2; + string output = 3; +} + +// ProofOp defines an operation used for calculating Merkle root +// The data could be arbitrary format, providing necessary data +// for example neighbouring node hash +message ProofOp { + string type = 1; + bytes key = 2; + bytes data = 3; +} + +// ProofOps is Merkle proof defined by the list of ProofOps +message ProofOps { + repeated ProofOp ops = 1; +} diff --git a/crypto/merkle/rfc6962_test.go b/merkle/rfc6962_test.go similarity index 95% rename from crypto/merkle/rfc6962_test.go rename to merkle/rfc6962_test.go index dd0c817..ae4b54a 100644 --- a/crypto/merkle/rfc6962_test.go +++ b/merkle/rfc6962_test.go @@ -17,10 +17,9 @@ package merkle // and consequently fall under the above license. import ( "bytes" + "crypto/sha256" "encoding/hex" "testing" - - "github.com/cometbft/cometbft/crypto/tmhash" ) func TestRFC6962Hasher(t *testing.T) { @@ -39,7 +38,7 @@ func TestRFC6962Hasher(t *testing.T) { // echo -n '' | sha256sum { desc: "RFC6962 Empty Tree", - want: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"[:tmhash.Size*2], + want: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"[:sha256.Size*2], got: emptyTreeHash, }, @@ -47,19 +46,19 @@ func TestRFC6962Hasher(t *testing.T) { // echo -n 00 | xxd -r -p | sha256sum { desc: "RFC6962 Empty Leaf", - want: "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d"[:tmhash.Size*2], + want: "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d"[:sha256.Size*2], got: emptyLeafHash, }, // echo -n 004C313233343536 | xxd -r -p | sha256sum { desc: "RFC6962 Leaf", - want: "395aa064aa4c29f7010acfe3f25db9485bbd4b91897b6ad7ad547639252b4d56"[:tmhash.Size*2], + want: "395aa064aa4c29f7010acfe3f25db9485bbd4b91897b6ad7ad547639252b4d56"[:sha256.Size*2], got: leafHash, }, // echo -n 014E3132334E343536 | xxd -r -p | sha256sum { desc: "RFC6962 Node", - want: "aa217fe888e47007fa15edab33c2b492a722cb106c64667fc2b044444de66bbb"[:tmhash.Size*2], + want: "aa217fe888e47007fa15edab33c2b492a722cb106c64667fc2b044444de66bbb"[:sha256.Size*2], got: innerHash([]byte("N123"), []byte("N456")), }, } { diff --git a/crypto/merkle/tree.go b/merkle/tree.go similarity index 100% rename from crypto/merkle/tree.go rename to merkle/tree.go diff --git a/crypto/merkle/tree_test.go b/merkle/tree_test.go similarity index 79% rename from crypto/merkle/tree_test.go rename to merkle/tree_test.go index 4b90186..78c1783 100644 --- a/crypto/merkle/tree_test.go +++ b/merkle/tree_test.go @@ -1,16 +1,14 @@ package merkle import ( + crand "crypto/rand" + "crypto/sha256" "encoding/hex" + "math/rand" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - cmtrand "github.com/cometbft/cometbft/libs/rand" - . "github.com/cometbft/cometbft/libs/test" - - "github.com/cometbft/cometbft/crypto/tmhash" ) type testItem []byte @@ -54,7 +52,7 @@ func TestProof(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(cmtrand.Bytes(tmhash.Size)) + items[i] = testItem(randBytes(sha256.Size)) } rootHash = HashFromByteSlices(items) @@ -78,7 +76,7 @@ func TestProof(t *testing.T) { // Trail too long should make it fail origAunts := proof.Aunts - proof.Aunts = append(proof.Aunts, cmtrand.Bytes(32)) + proof.Aunts = append(proof.Aunts, randBytes(32)) err = proof.Verify(rootHash, item) require.Error(t, err, "Expected verification to fail for wrong trail length") @@ -92,11 +90,11 @@ func TestProof(t *testing.T) { proof.Aunts = origAunts // Mutating the itemHash should make it fail. - err = proof.Verify(rootHash, MutateByteSlice(item)) + err = proof.Verify(rootHash, mutateByteSlice(item)) require.Error(t, err, "Expected verification to fail for mutated leaf hash") // Mutating the rootHash should make it fail. - err = proof.Verify(MutateByteSlice(rootHash), item) + err = proof.Verify(mutateByteSlice(rootHash), item) require.Error(t, err, "Expected verification to fail for mutated root hash") } } @@ -107,7 +105,7 @@ func TestHashAlternatives(t *testing.T) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(cmtrand.Bytes(tmhash.Size)) + items[i] = testItem(randBytes(sha256.Size)) } rootHash1 := HashFromByteSlicesIterative(items) @@ -120,7 +118,7 @@ func BenchmarkHashAlternatives(b *testing.B) { items := make([][]byte, total) for i := 0; i < total; i++ { - items[i] = testItem(cmtrand.Bytes(tmhash.Size)) + items[i] = testItem(randBytes(sha256.Size)) } b.ResetTimer() @@ -159,3 +157,32 @@ func Test_getSplitPoint(t *testing.T) { require.EqualValues(t, tt.want, got, "getSplitPoint(%d) = %v, want %v", tt.length, got, tt.want) } } + +func randBytes(size int) []byte { + b := make([]byte, size) + _, _ = crand.Read(b) + return b +} + +// Contract: !bytes.Equal(input, output) && len(input) >= len(output) +func mutateByteSlice(bytez []byte) []byte { + // If bytez is empty, panic + if len(bytez) == 0 { + panic("Cannot mutate an empty bytez") + } + + // Copy bytez + mBytez := make([]byte, len(bytez)) + copy(mBytez, bytez) + bytez = mBytez + + // Try a random mutation + switch rand.Int() % 2 { + case 0: // Mutate a single byte + bytez[rand.Int()%len(bytez)] += byte(rand.Int()%255 + 1) + case 1: // Remove an arbitrary byte + pos := rand.Int() % len(bytez) + bytez = append(bytez[:pos], bytez[pos+1:]...) + } + return bytez +} diff --git a/crypto/merkle/types.go b/merkle/types.go similarity index 100% rename from crypto/merkle/types.go rename to merkle/types.go