increase the 252 per-block transaction limit

.gitignore

@@ -7,3 +7,4 @@ test-log
 lwd-api.html
 *.orig
 __debug_bin
+.vscode

common/cache_test.go

@@ -46,10 +46,13 @@ func TestCache(t *testing.T) {
 	for _, test := range compactTests {
 		blockData, _ := hex.DecodeString(test.Full)
 		block := parser.NewBlock()
-		_, err = block.ParseFromSlice(blockData)
+		blockData, err = block.ParseFromSlice(blockData)
 		if err != nil {
 			t.Fatal(err)
 		}
+		if len(blockData) > 0 {
+			t.Error("Extra data remaining")
+		}
 		compacts = append(compacts, block.ToCompact())
 	}
 

common/darkside.go

@@ -178,11 +178,32 @@ func DarksideApplyStaged(height int) error {
 			return errors.New("transaction height too high")
 		}
 		block := state.activeBlocks[tx.height-state.startHeight]
-		if block[1487] == 253 {
-			return errors.New("too many transactions in a block (max 253)")
+		// The next one or 3 bytes encode the number of transactions to follow,
+		// little endian.
+		nTxFirstByte := block[1487]
+		switch {
+		case nTxFirstByte < 252:
+			block[1487]++
+		case nTxFirstByte == 252:
+			// incrementing to 253, requires "253" followed by 2-byte length,
+			// extend the block by two bytes, shift existing transaction bytes
+			block = append(block, 0, 0)
+			copy(block[1490:], block[1488:len(block)-2])
+			block[1487] = 253
+			block[1488] = 253
+			block[1489] = 0
+		case nTxFirstByte == 253:
+			block[1488]++
+			if block[1488] == 0 {
+				// wrapped around
+				block[1489]++
+			}
+		default:
+			// no need to worry about more than 64k transactions
+			Log.Fatal("unexpected compact transaction count ", nTxFirstByte,
+				", can't support more than 64k transactions in a block")
 		}
-		block[1487]++ // one more transaction
-		block[68]++   // hack HashFinalSaplingRoot to mod the block hash
+		block[68]++ // hack HashFinalSaplingRoot to mod the block hash
 		block = append(block, tx.bytes...)
 		state.activeBlocks[tx.height-state.startHeight] = block
 	}
@@ -244,7 +265,7 @@ func DarksideStageBlockStream(blockHex string) error {
 	if !state.resetted {
 		return errors.New("please call Reset first")
 	}
-	Log.Info("StageBlocks()")
+	Log.Info("StageBlocksStream()")
 	blockBytes, err := hex.DecodeString(blockHex)
 	if err != nil {
 		return err
@@ -445,7 +466,7 @@ func DarksideStageTransactionsURL(height int, url string) error {
 	if !state.resetted {
 		return errors.New("please call Reset first")
 	}
-	Log.Info("StageTransactionsURL(height=", height, "url=", url, ")")
+	Log.Info("StageTransactionsURL(height=", height, " url=", url, ")")
 	resp, err := http.Get(url)
 	if err != nil {
 		return err

docs/rtd/index.html

@@ -738,8 +738,8 @@ <h3 id="cash.z.wallet.sdk.rpc.DarksideStreamer">DarksideStreamer</h3>
                 <td><p>StageBlocksCreate is like the previous two, except it creates &#39;count&#39;
 empty blocks at consecutive heights starting at height &#39;height&#39;. The
 &#39;nonce&#39; is part of the header, so it contributes to the block hash; this
-lets you create two fake blocks with the same transactions (or no
-transactions) and same height, with two different hashes.</p></td>
+lets you create identical blocks (same transactions and height), but with
+different hashes.</p></td>
               </tr>
 
               <tr>
@@ -757,10 +757,9 @@ <h3 id="cash.z.wallet.sdk.rpc.DarksideStreamer">DarksideStreamer</h3>
                 <td>StageTransactions</td>
                 <td><a href="#cash.z.wallet.sdk.rpc.DarksideTransactionsURL">DarksideTransactionsURL</a></td>
                 <td><a href="#cash.z.wallet.sdk.rpc.Empty">Empty</a></td>
-                <td><p>StageTransactions is the same except the transactions are fetched
-from the given url. They are all staged into the block at the given
-height. Staging transactions at multiple different heights requires
-multiple calls.</p></td>
+                <td><p>StageTransactions is the same except the transactions are fetched from
+the given url. They are all staged into the block at the given height.
+Staging transactions to different heights requires multiple calls.</p></td>
               </tr>
 
               <tr>
@@ -770,12 +769,14 @@ <h3 id="cash.z.wallet.sdk.rpc.DarksideStreamer">DarksideStreamer</h3>
                 <td><p>ApplyStaged iterates the list of blocks that were staged by the
 StageBlocks*() gRPCs, in the order they were staged, and &#34;merges&#34; each
 into the active, working blocks list that the mock zcashd is presenting
-to lightwalletd. The resulting working block list can&#39;t have gaps; if the
-working block range is 1000-1006, and the staged block range is 1003-1004,
-the resulting range is 1000-1004, with 1000-1002 unchanged, blocks
-1003-1004 from the new range, and 1005-1006 dropped. After merging all
-blocks, ApplyStaged() appends staged transactions (in the order received)
-into each one&#39;s corresponding block. The staging area is then cleared.
+to lightwalletd. Even as each block is applied, the active list can&#39;t
+have gaps; if the active block range is 1000-1006, and the staged block
+range is 1003-1004, the resulting range is 1000-1004, with 1000-1002
+unchanged, blocks 1003-1004 from the new range, and 1005-1006 dropped.
+
+After merging all blocks, ApplyStaged() appends staged transactions (in
+the order received) into each one&#39;s corresponding (by height) block
+The staging area is then cleared.
 
 The argument specifies the latest block height that mock zcashd reports
 (i.e. what&#39;s returned by GetLatestBlock). Note that ApplyStaged() can

parser/block.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2019-2020 The Zcash developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://www.opensource.org/licenses/mit-license.php .
+
+// Package parser deserializes blocks from zcashd.
 package parser
 
 import (
@@ -11,24 +13,30 @@ import (
 	"github.com/zcash/lightwalletd/walletrpc"
 )
 
+// Block represents a full block (not a compact block).
 type Block struct {
 	hdr    *BlockHeader
 	vtx    []*Transaction
 	height int
 }
 
+// NewBlock constructs a block instance.
 func NewBlock() *Block {
 	return &Block{height: -1}
 }
 
+// GetVersion returns a block's version number (current 4)
 func (b *Block) GetVersion() int {
 	return int(b.hdr.Version)
 }
 
+// GetTxCount returns the number of transactions in the block,
+// including the coinbase transaction (minimum 1).
 func (b *Block) GetTxCount() int {
 	return len(b.vtx)
 }
 
+// Transactions returns the list of the block's transactions.
 func (b *Block) Transactions() []*Transaction {
 	// TODO: these should NOT be mutable
 	return b.vtx
@@ -46,10 +54,12 @@ func (b *Block) GetEncodableHash() []byte {
 	return b.hdr.GetEncodableHash()
 }
 
+// GetDisplayPrevHash returns the block's previous hash in big-endian format.
 func (b *Block) GetDisplayPrevHash() []byte {
 	return b.hdr.GetDisplayPrevHash()
 }
 
+// HasSaplingTransactions indicates if the block contains any Sapling tx.
 func (b *Block) HasSaplingTransactions() bool {
 	for _, tx := range b.vtx {
 		if tx.HasSaplingElements() {
@@ -62,7 +72,7 @@ func (b *Block) HasSaplingTransactions() bool {
 // see https://github.com/zcash/lightwalletd/issues/17#issuecomment-467110828
 const genesisTargetDifficulty = 520617983
 
-// GetHeight() extracts the block height from the coinbase transaction. See
+// GetHeight extracts the block height from the coinbase transaction. See
 // BIP34. Returns block height on success, or -1 on error.
 func (b *Block) GetHeight() int {
 	if b.height != -1 {
@@ -90,10 +100,12 @@ func (b *Block) GetHeight() int {
 	return int(blockHeight)
 }
 
+// GetPrevHash returns the hash of the block's previous block (little-endian).
 func (b *Block) GetPrevHash() []byte {
 	return b.hdr.HashPrevBlock
 }
 
+// ToCompact returns the compact representation of the full block.
 func (b *Block) ToCompact() *walletrpc.CompactBlock {
 	compactBlock := &walletrpc.CompactBlock{
 		//TODO ProtoVersion: 1,
@@ -114,6 +126,9 @@ func (b *Block) ToCompact() *walletrpc.CompactBlock {
 	return compactBlock
 }
 
+// ParseFromSlice deserializes a block from the given data stream
+// and returns a slice to the remaining data. The caller should verify
+// there is no remaining data if none is expected.
 func (b *Block) ParseFromSlice(data []byte) (rest []byte, err error) {
 	hdr := NewBlockHeader()
 	data, err = hdr.ParseFromSlice(data)
@@ -129,17 +144,19 @@ func (b *Block) ParseFromSlice(data []byte) (rest []byte, err error) {
 	data = []byte(s)
 
 	vtx := make([]*Transaction, 0, txCount)
-	for i := 0; len(data) > 0; i++ {
+	var i int
+	for i = 0; i < txCount && len(data) > 0; i++ {
 		tx := NewTransaction()
 		data, err = tx.ParseFromSlice(data)
 		if err != nil {
 			return nil, errors.Wrap(err, fmt.Sprintf("parsing transaction %d", i))
 		}
 		vtx = append(vtx, tx)
 	}
-
+	if i < txCount {
+		return nil, errors.New("parsing block transactions: not enough data")
+	}
 	b.hdr = hdr
 	b.vtx = vtx
-
 	return data, nil
 }

parser/block_header.go

@@ -1,6 +1,8 @@
 // Copyright (c) 2019-2020 The Zcash developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://www.opensource.org/licenses/mit-license.php .
+
+// Package parser deserializes the block header from zcashd.
 package parser
 
 import (
@@ -18,7 +20,8 @@ const (
 	equihashSizeMainnet             = 1344 // size of a mainnet / testnet Equihash solution in bytes
 )
 
-// A block header as defined in version 2018.0-beta-29 of the Zcash Protocol Spec.
+// RawBlockHeader implements the block header as defined in version
+// 2018.0-beta-29 of the Zcash Protocol Spec.
 type RawBlockHeader struct {
 	// The block version number indicates which set of block validation rules
 	// to follow. The current and only defined block version number for Zcash
@@ -58,6 +61,7 @@ type RawBlockHeader struct {
 	Solution []byte
 }
 
+// BlockHeader extends RawBlockHeader by adding a cache for the block hash.
 type BlockHeader struct {
 	*RawBlockHeader
 	cachedHash []byte
@@ -93,17 +97,18 @@ func WriteCompactLengthPrefixedLen(buf *bytes.Buffer, length int) {
 	}
 }
 
-func WriteCompactLengthPrefixed(buf *bytes.Buffer, val []byte) {
+func writeCompactLengthPrefixed(buf *bytes.Buffer, val []byte) {
 	WriteCompactLengthPrefixedLen(buf, len(val))
 	binary.Write(buf, binary.LittleEndian, val)
 }
 
-func (hdr *RawBlockHeader) GetSize() int {
+func (hdr *RawBlockHeader) getSize() int {
 	return serBlockHeaderMinusEquihashSize + CompactLengthPrefixedLen(len(hdr.Solution))
 }
 
+// MarshalBinary returns the block header in serialized form
 func (hdr *RawBlockHeader) MarshalBinary() ([]byte, error) {
-	headerSize := hdr.GetSize()
+	headerSize := hdr.getSize()
 	backing := make([]byte, 0, headerSize)
 	buf := bytes.NewBuffer(backing)
 	binary.Write(buf, binary.LittleEndian, hdr.Version)
@@ -113,7 +118,7 @@ func (hdr *RawBlockHeader) MarshalBinary() ([]byte, error) {
 	binary.Write(buf, binary.LittleEndian, hdr.Time)
 	binary.Write(buf, binary.LittleEndian, hdr.NBitsBytes)
 	binary.Write(buf, binary.LittleEndian, hdr.Nonce)
-	WriteCompactLengthPrefixed(buf, hdr.Solution)
+	writeCompactLengthPrefixed(buf, hdr.Solution)
 	return backing[:headerSize], nil
 }
 
@@ -229,6 +234,7 @@ func (hdr *BlockHeader) GetEncodableHash() []byte {
 	return digest[:]
 }
 
+// GetDisplayPrevHash returns the block hash in
 func (hdr *BlockHeader) GetDisplayPrevHash() []byte {
 	rhash := make([]byte, len(hdr.HashPrevBlock))
 	copy(rhash, hdr.HashPrevBlock)

parser/block_header_test.go

@@ -249,7 +249,7 @@ func TestWriteCompactLengthPrefixedLen(t *testing.T) {
 func TestWriteCompactLengthPrefixed(t *testing.T) {
 	var b bytes.Buffer
 	val := []byte{22, 33, 44}
-	WriteCompactLengthPrefixed(&b, val)
+	writeCompactLengthPrefixed(&b, val)
 	r := make([]byte, 4)
 	b.Read(r)
 	expected := []byte{3, 22, 33, 44}