ssz, tests: fix decoding empty slices into existing objects

decoder.go

@@ -292,7 +292,14 @@ func DecodeSliceOfBitsContent(dec *Decoder, bitlist *bitfield.Bitlist, maxBits u
 	// Compute the length of the encoded bits based on the seen offsets
 	size := dec.retrieveSize()
 	if size == 0 {
-		return // empty slice of objects
+		// Empty slice, remove anything extra
+		if len(*bitlist) == 0 {
+			(*bitlist) = make([]byte, 1)
+		} else {
+			*bitlist = (*bitlist)[:1]
+		}
+		(*bitlist)[0] = 0x01
+		return
 	}
 	// Verify that the byte size is reasonable, bits will need an extra step after decoding
 	if maxBytes := maxBits>>3 + 1; maxBytes < uint64(size) {
@@ -370,7 +377,9 @@ func DecodeSliceOfUint64sContent[T ~uint64](dec *Decoder, ns *[]T, maxItems uint
 	// Compute the length of the encoded binaries based on the seen offsets
 	size := dec.retrieveSize()
 	if size == 0 {
-		return // empty slice of objects
+		// Empty slice, remove anything extra
+		*ns = (*ns)[:0]
+		return
 	}
 	// Compute the number of items based on the item size of the type
 	if size&7 != 0 {
@@ -486,7 +495,9 @@ func DecodeSliceOfStaticBytesContent[T commonBytesLengths](dec *Decoder, blobs *
 	// Compute the length of the encoded binaries based on the seen offsets
 	size := dec.retrieveSize()
 	if size == 0 {
-		return // empty slice of objects
+		// Empty slice, remove anything extra
+		*blobs = (*blobs)[:0]
+		return
 	}
 	// Compute the number of items based on the item size of the type
 	var sizer T // SizeSSZ is on *U, objects is static, so nil T is fine
@@ -549,7 +560,9 @@ func DecodeSliceOfDynamicBytesContent(dec *Decoder, blobs *[][]byte, maxItems ui
 	// check for empty slice or possibly bad data (too short to encode anything)
 	size := dec.retrieveSize()
 	if size == 0 {
-		return // empty slice of blobs
+		// Empty slice, remove anything extra
+		*blobs = (*blobs)[:0]
+		return
 	}
 	if size < 4 {
 		dec.err = fmt.Errorf("%w: %d bytes available", ErrShortCounterOffset, size)
@@ -606,7 +619,9 @@ func DecodeSliceOfStaticObjectsContent[T newableStaticObject[U], U any](dec *Dec
 	// Compute the length of the encoded objects based on the seen offsets
 	size := dec.retrieveSize()
 	if size == 0 {
-		return // empty slice of objects
+		// Empty slice, remove anything extra
+		*objects = (*objects)[:0]
+		return
 	}
 	// Compute the number of items based on the item size of the type
 	var sizer T // SizeSSZ is on *U, objects is static, so nil T is fine
@@ -656,7 +671,9 @@ func DecodeSliceOfDynamicObjectsContent[T newableDynamicObject[U], U any](dec *D
 	// check for empty slice or possibly bad data (too short to encode anything)
 	size := dec.retrieveSize()
 	if size == 0 {
-		return // empty slice of blobs
+		// Empty slice, remove anything extra
+		*objects = (*objects)[:0]
+		return
 	}
 	if size < 4 {
 		dec.err = fmt.Errorf("%w: %d bytes available", ErrShortCounterOffset, size)

tests/consensus_specs_test.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"math/rand/v2"
 	"os"
 	"path/filepath"
 	"sync"
@@ -341,9 +342,27 @@ func FuzzConsensusSpecsBeaconBlock(f *testing.F) {
 func FuzzConsensusSpecsBeaconBlockBody(f *testing.F) {
 	fuzzConsensusSpecType[*types.BeaconBlockBody](f, "BeaconBlockBody")
 }
+func FuzzConsensusSpecsBeaconBlockBodyAltair(f *testing.F) {
+	fuzzConsensusSpecType[*types.BeaconBlockBodyAltair](f, "BeaconBlockBody")
+}
+func FuzzConsensusSpecsBeaconBlockBodyBellatrix(f *testing.F) {
+	fuzzConsensusSpecType[*types.BeaconBlockBodyBellatrix](f, "BeaconBlockBody")
+}
+func FuzzConsensusSpecsBeaconBlockBodyCapella(f *testing.F) {
+	fuzzConsensusSpecType[*types.BeaconBlockBodyCapella](f, "BeaconBlockBody")
+}
+func FuzzConsensusSpecsBeaconBlockBodyDeneb(f *testing.F) {
+	fuzzConsensusSpecType[*types.BeaconBlockBodyDeneb](f, "BeaconBlockBody")
+}
 func FuzzConsensusSpecsBeaconBlockHeader(f *testing.F) {
 	fuzzConsensusSpecType[*types.BeaconBlockHeader](f, "BeaconBlockHeader")
 }
+func FuzzConsensusSpecsBeaconState(f *testing.F) {
+	fuzzConsensusSpecType[*types.BeaconState](f, "BeaconState")
+}
+func FuzzConsensusSpecsBLSToExecutionChange(f *testing.F) {
+	fuzzConsensusSpecType[*types.BLSToExecutionChange](f, "BLSToExecutionChange")
+}
 func FuzzConsensusSpecsCheckpoint(f *testing.F) {
 	fuzzConsensusSpecType[*types.Checkpoint](f, "Checkpoint")
 }
@@ -353,6 +372,12 @@ func FuzzConsensusSpecsDeposit(f *testing.F) {
 func FuzzConsensusSpecsDepositData(f *testing.F) {
 	fuzzConsensusSpecType[*types.DepositData](f, "DepositData")
 }
+func FuzzConsensusSpecsDepositMessage(f *testing.F) {
+	fuzzConsensusSpecType[*types.DepositMessage](f, "DepositMessage")
+}
+func FuzzConsensusSpecsEth1Block(f *testing.F) {
+	fuzzConsensusSpecType[*types.Eth1Block](f, "Eth1Block")
+}
 func FuzzConsensusSpecsEth1Data(f *testing.F) {
 	fuzzConsensusSpecType[*types.Eth1Data](f, "Eth1Data")
 }
@@ -362,30 +387,60 @@ func FuzzConsensusSpecsExecutionPayload(f *testing.F) {
 func FuzzConsensusSpecsExecutionPayloadCapella(f *testing.F) {
 	fuzzConsensusSpecType[*types.ExecutionPayloadCapella](f, "ExecutionPayload")
 }
+func FuzzConsensusSpecsExecutionPayloadDeneb(f *testing.F) {
+	fuzzConsensusSpecType[*types.ExecutionPayloadDeneb](f, "ExecutionPayload")
+}
+func FuzzConsensusSpecsExecutionPayloadHeader(f *testing.F) {
+	fuzzConsensusSpecType[*types.ExecutionPayloadHeader](f, "ExecutionPayloadHeader")
+}
+func FuzzConsensusSpecsExecutionPayloadHeaderCapella(f *testing.F) {
+	fuzzConsensusSpecType[*types.ExecutionPayloadHeaderCapella](f, "ExecutionPayloadHeader")
+}
+func FuzzConsensusSpecsExecutionPayloadHeaderDeneb(f *testing.F) {
+	fuzzConsensusSpecType[*types.ExecutionPayloadHeaderDeneb](f, "ExecutionPayloadHeader")
+}
+func FuzzConsensusSpecsFork(f *testing.F) {
+	fuzzConsensusSpecType[*types.Fork](f, "Fork")
+}
 func FuzzConsensusSpecsHistoricalBatch(f *testing.F) {
 	fuzzConsensusSpecType[*types.HistoricalBatch](f, "HistoricalBatch")
 }
 func FuzzConsensusSpecsHistoricalBatchVariation(f *testing.F) {
 	fuzzConsensusSpecType[*types.HistoricalBatchVariation](f, "HistoricalBatch")
 }
+func FuzzConsensusSpecsHistoricalSummary(f *testing.F) {
+	fuzzConsensusSpecType[*types.HistoricalSummary](f, "HistoricalSummary")
+}
 func FuzzConsensusSpecsIndexedAttestation(f *testing.F) {
 	fuzzConsensusSpecType[*types.IndexedAttestation](f, "IndexedAttestation")
 }
+func FuzzConsensusSpecsPendingAttestation(f *testing.F) {
+	fuzzConsensusSpecType[*types.PendingAttestation](f, "PendingAttestation")
+}
 func FuzzConsensusSpecsProposerSlashing(f *testing.F) {
 	fuzzConsensusSpecType[*types.ProposerSlashing](f, "ProposerSlashing")
 }
 func FuzzConsensusSpecsSignedBeaconBlockHeader(f *testing.F) {
 	fuzzConsensusSpecType[*types.SignedBeaconBlockHeader](f, "SignedBeaconBlockHeader")
 }
+func FuzzConsensusSpecsSignedBLSToExecutionChange(f *testing.F) {
+	fuzzConsensusSpecType[*types.SignedBLSToExecutionChange](f, "SignedBLSToExecutionChange")
+}
 func FuzzConsensusSpecsSignedVoluntaryExit(f *testing.F) {
 	fuzzConsensusSpecType[*types.SignedVoluntaryExit](f, "SignedVoluntaryExit")
 }
-func FuzzConsensusSpecsVoluntaryExit(f *testing.F) {
-	fuzzConsensusSpecType[*types.VoluntaryExit](f, "VoluntaryExit")
+func FuzzConsensusSpecsSyncAggregate(f *testing.F) {
+	fuzzConsensusSpecType[*types.SyncAggregate](f, "SyncAggregate")
+}
+func FuzzConsensusSpecsSyncCommittee(f *testing.F) {
+	fuzzConsensusSpecType[*types.SyncCommittee](f, "SyncCommittee")
 }
 func FuzzConsensusSpecsValidator(f *testing.F) {
 	fuzzConsensusSpecType[*types.Validator](f, "Validator")
 }
+func FuzzConsensusSpecsVoluntaryExit(f *testing.F) {
+	fuzzConsensusSpecType[*types.VoluntaryExit](f, "VoluntaryExit")
+}
 func FuzzConsensusSpecsWithdrawal(f *testing.F) {
 	fuzzConsensusSpecType[*types.Withdrawal](f, "Withdrawal")
 }
@@ -394,14 +449,13 @@ func FuzzConsensusSpecsWithdrawalVariation(f *testing.F) {
 }
 
 func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string) {
-	// Iterate over all the forks and collect all the sample data. It's fine to
-	// have mismatching type version and test data, it's just going to skip on
-	// the first parse as bad data.
+	// Iterate over all the forks and collect all the sample data
 	forks, err := os.ReadDir(consensusSpecTestsRoot)
 	if err != nil {
 		f.Errorf("failed to walk spec collection %v: %v", consensusSpecTestsRoot, err)
 		return
 	}
+	var valids [][]byte
 	for _, fork := range forks {
 		// Skip test cases for types introduced in later forks
 		path := filepath.Join(consensusSpecTestsRoot, fork.Name(), "ssz_static", kind, "ssz_random")
@@ -413,7 +467,7 @@ func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string)
 			f.Errorf("failed to walk test collection %v: %v", path, err)
 			return
 		}
-		// Feed all the test data into the fuzzer
+		// Feed all the valid test data into the fuzzer
 		for _, test := range tests {
 			inSnappy, err := os.ReadFile(filepath.Join(path, test.Name(), "serialized.ssz_snappy"))
 			if err != nil {
@@ -423,11 +477,22 @@ func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string)
 			if err != nil {
 				f.Fatalf("failed to parse snappy ssz binary: %v", err)
 			}
-			f.Add(inSSZ)
+			obj := T(new(U))
+			if err := ssz.DecodeFromStream(bytes.NewReader(inSSZ), obj, uint32(len(inSSZ))); err == nil {
+				// Stash away all valid ssz streams so we can play with decoding
+				// into previously used objects
+				valids = append(valids, inSSZ)
+
+				// Add the valid ssz stream to the fuzzer
+				f.Add(inSSZ)
+			}
 		}
 	}
 	// Run the fuzzer
 	f.Fuzz(func(t *testing.T, inSSZ []byte) {
+		// Track whether the testcase is valid
+		var valid bool
+
 		// Try the stream encoder/decoder
 		obj := T(new(U))
 		if err := ssz.DecodeFromStream(bytes.NewReader(inSSZ), obj, uint32(len(inSSZ))); err == nil {
@@ -448,6 +513,7 @@ func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string)
 			if size := ssz.Size(obj); size != uint32(len(inSSZ)) {
 				t.Fatalf("reported/generated size mismatch: reported %v, generated %v", size, len(inSSZ))
 			}
+			valid = true
 		}
 		// Try the buffer encoder/decoder
 		obj = T(new(U))
@@ -470,5 +536,51 @@ func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string)
 				t.Fatalf("reported/generated size mismatch: reported %v, generated %v", size, len(inSSZ))
 			}
 		}
+		// If the testcase was valid, try decoding it into a used object
+		if valid {
+			// Pick a random starting object
+			vSSZ := valids[rand.N(len(valids))]
+
+			// Try the stream encoder/decoder into a prepped object
+			obj = T(new(U))
+			if err := ssz.DecodeFromBytes(vSSZ, obj); err != nil {
+				panic(err) // we've already decoded this, cannot fail
+			}
+			if err := ssz.DecodeFromStream(bytes.NewReader(inSSZ), obj, uint32(len(inSSZ))); err != nil {
+				t.Fatalf("failed to decode stream into used object: %v", err)
+			}
+			blob := new(bytes.Buffer)
+			if err := ssz.EncodeToStream(blob, obj); err != nil {
+				t.Fatalf("failed to re-encode stream from used object: %v", err)
+			}
+			if !bytes.Equal(blob.Bytes(), inSSZ) {
+				prefix := commonPrefix(blob.Bytes(), inSSZ)
+				t.Fatalf("re-encoded stream from used object mismatch: have %x, want %x, common prefix %d, have left %x, want left %x",
+					blob, inSSZ, len(prefix), blob.Bytes()[len(prefix):], inSSZ[len(prefix):])
+			}
+			if size := ssz.Size(obj); size != uint32(len(inSSZ)) {
+				t.Fatalf("reported/generated size mismatch: reported %v, generated %v", size, len(inSSZ))
+			}
+			// Try the buffer encoder/decoder into a prepped object
+			obj = T(new(U))
+			if err := ssz.DecodeFromBytes(vSSZ, obj); err != nil {
+				panic(err) // we've already decoded this, cannot fail
+			}
+			if err := ssz.DecodeFromBytes(inSSZ, obj); err != nil {
+				t.Fatalf("failed to decode buffer into used object: %v", err)
+			}
+			bin := make([]byte, ssz.Size(obj))
+			if err := ssz.EncodeToBytes(bin, obj); err != nil {
+				t.Fatalf("failed to re-encode buffer from used object: %v", err)
+			}
+			if !bytes.Equal(bin, inSSZ) {
+				prefix := commonPrefix(bin, inSSZ)
+				t.Fatalf("re-encoded buffer from used object mismatch: have %x, want %x, common prefix %d, have left %x, want left %x",
+					blob, inSSZ, len(prefix), bin[len(prefix):], inSSZ[len(prefix):])
+			}
+			if size := ssz.Size(obj); size != uint32(len(inSSZ)) {
+				t.Fatalf("reported/generated size mismatch: reported %v, generated %v", size, len(inSSZ))
+			}
+		}
 	})
 }

tests/testtypes/consensus-spec-tests/types_consensus.go

@@ -28,7 +28,7 @@ import (
 //go:generate go run ../../../cmd/sszgen -type HistoricalSummary -out gen_historical_summary_ssz.go
 //go:generate go run ../../../cmd/sszgen -type IndexedAttestation -out gen_indexed_attestation_ssz.go
 //go:generate go run ../../../cmd/sszgen -type PendingAttestation -out gen_pending_attestation_ssz.go
-//go:generate go run ../../../cmd/sszgen -type ProposerSlashing -out gen_proposed_slashing_ssz.go
+//go:generate go run ../../../cmd/sszgen -type ProposerSlashing -out gen_proposer_slashing_ssz.go
 //go:generate go run ../../../cmd/sszgen -type SignedBeaconBlockHeader -out gen_signed_beacon_block_header_ssz.go
 //go:generate go run ../../../cmd/sszgen -type SignedBLSToExecutionChange -out gen_signed_bls_to_execution_change_ssz.go
 //go:generate go run ../../../cmd/sszgen -type SignedVoluntaryExit -out gen_signed_voluntary_exit_ssz.go