From 52a59190a8eef39612cf12c2720d67a9450e6dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 10 Jul 2024 10:01:25 +0300 Subject: [PATCH] ssz, tests: fix decoding empty slices into existing objects --- decoder.go | 29 +++- tests/consensus_specs_test.go | 126 +++++++++++++++++- ...ng_ssz.go => gen_proposer_slashing_ssz.go} | 0 .../consensus-spec-tests/types_consensus.go | 2 +- 4 files changed, 143 insertions(+), 14 deletions(-) rename tests/testtypes/consensus-spec-tests/{gen_proposed_slashing_ssz.go => gen_proposer_slashing_ssz.go} (100%) diff --git a/decoder.go b/decoder.go index aea7c22..6f628c1 100644 --- a/decoder.go +++ b/decoder.go @@ -292,7 +292,14 @@ func DecodeSliceOfBitsContent(dec *Decoder, bitlist *bitfield.Bitlist, maxBits u // Compute the length of the encoded bits based on the seen offsets size := dec.retrieveSize() if size == 0 { - return // empty slice of objects + // Empty slice, remove anything extra + if len(*bitlist) == 0 { + (*bitlist) = make([]byte, 1) + } else { + *bitlist = (*bitlist)[:1] + } + (*bitlist)[0] = 0x01 + return } // Verify that the byte size is reasonable, bits will need an extra step after decoding if maxBytes := maxBits>>3 + 1; maxBytes < uint64(size) { @@ -370,7 +377,9 @@ func DecodeSliceOfUint64sContent[T ~uint64](dec *Decoder, ns *[]T, maxItems uint // Compute the length of the encoded binaries based on the seen offsets size := dec.retrieveSize() if size == 0 { - return // empty slice of objects + // Empty slice, remove anything extra + *ns = (*ns)[:0] + return } // Compute the number of items based on the item size of the type if size&7 != 0 { @@ -486,7 +495,9 @@ func DecodeSliceOfStaticBytesContent[T commonBytesLengths](dec *Decoder, blobs * // Compute the length of the encoded binaries based on the seen offsets size := dec.retrieveSize() if size == 0 { - return // empty slice of objects + // Empty slice, remove anything extra + *blobs = (*blobs)[:0] + return } // Compute the number of items based on the item size of the type var sizer T // SizeSSZ is on *U, objects is static, so nil T is fine @@ -549,7 +560,9 @@ func DecodeSliceOfDynamicBytesContent(dec *Decoder, blobs *[][]byte, maxItems ui // check for empty slice or possibly bad data (too short to encode anything) size := dec.retrieveSize() if size == 0 { - return // empty slice of blobs + // Empty slice, remove anything extra + *blobs = (*blobs)[:0] + return } if size < 4 { dec.err = fmt.Errorf("%w: %d bytes available", ErrShortCounterOffset, size) @@ -606,7 +619,9 @@ func DecodeSliceOfStaticObjectsContent[T newableStaticObject[U], U any](dec *Dec // Compute the length of the encoded objects based on the seen offsets size := dec.retrieveSize() if size == 0 { - return // empty slice of objects + // Empty slice, remove anything extra + *objects = (*objects)[:0] + return } // Compute the number of items based on the item size of the type var sizer T // SizeSSZ is on *U, objects is static, so nil T is fine @@ -656,7 +671,9 @@ func DecodeSliceOfDynamicObjectsContent[T newableDynamicObject[U], U any](dec *D // check for empty slice or possibly bad data (too short to encode anything) size := dec.retrieveSize() if size == 0 { - return // empty slice of blobs + // Empty slice, remove anything extra + *objects = (*objects)[:0] + return } if size < 4 { dec.err = fmt.Errorf("%w: %d bytes available", ErrShortCounterOffset, size) diff --git a/tests/consensus_specs_test.go b/tests/consensus_specs_test.go index 32272b0..fceb845 100644 --- a/tests/consensus_specs_test.go +++ b/tests/consensus_specs_test.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" "io" + "math/rand/v2" "os" "path/filepath" "sync" @@ -341,9 +342,27 @@ func FuzzConsensusSpecsBeaconBlock(f *testing.F) { func FuzzConsensusSpecsBeaconBlockBody(f *testing.F) { fuzzConsensusSpecType[*types.BeaconBlockBody](f, "BeaconBlockBody") } +func FuzzConsensusSpecsBeaconBlockBodyAltair(f *testing.F) { + fuzzConsensusSpecType[*types.BeaconBlockBodyAltair](f, "BeaconBlockBody") +} +func FuzzConsensusSpecsBeaconBlockBodyBellatrix(f *testing.F) { + fuzzConsensusSpecType[*types.BeaconBlockBodyBellatrix](f, "BeaconBlockBody") +} +func FuzzConsensusSpecsBeaconBlockBodyCapella(f *testing.F) { + fuzzConsensusSpecType[*types.BeaconBlockBodyCapella](f, "BeaconBlockBody") +} +func FuzzConsensusSpecsBeaconBlockBodyDeneb(f *testing.F) { + fuzzConsensusSpecType[*types.BeaconBlockBodyDeneb](f, "BeaconBlockBody") +} func FuzzConsensusSpecsBeaconBlockHeader(f *testing.F) { fuzzConsensusSpecType[*types.BeaconBlockHeader](f, "BeaconBlockHeader") } +func FuzzConsensusSpecsBeaconState(f *testing.F) { + fuzzConsensusSpecType[*types.BeaconState](f, "BeaconState") +} +func FuzzConsensusSpecsBLSToExecutionChange(f *testing.F) { + fuzzConsensusSpecType[*types.BLSToExecutionChange](f, "BLSToExecutionChange") +} func FuzzConsensusSpecsCheckpoint(f *testing.F) { fuzzConsensusSpecType[*types.Checkpoint](f, "Checkpoint") } @@ -353,6 +372,12 @@ func FuzzConsensusSpecsDeposit(f *testing.F) { func FuzzConsensusSpecsDepositData(f *testing.F) { fuzzConsensusSpecType[*types.DepositData](f, "DepositData") } +func FuzzConsensusSpecsDepositMessage(f *testing.F) { + fuzzConsensusSpecType[*types.DepositMessage](f, "DepositMessage") +} +func FuzzConsensusSpecsEth1Block(f *testing.F) { + fuzzConsensusSpecType[*types.Eth1Block](f, "Eth1Block") +} func FuzzConsensusSpecsEth1Data(f *testing.F) { fuzzConsensusSpecType[*types.Eth1Data](f, "Eth1Data") } @@ -362,30 +387,60 @@ func FuzzConsensusSpecsExecutionPayload(f *testing.F) { func FuzzConsensusSpecsExecutionPayloadCapella(f *testing.F) { fuzzConsensusSpecType[*types.ExecutionPayloadCapella](f, "ExecutionPayload") } +func FuzzConsensusSpecsExecutionPayloadDeneb(f *testing.F) { + fuzzConsensusSpecType[*types.ExecutionPayloadDeneb](f, "ExecutionPayload") +} +func FuzzConsensusSpecsExecutionPayloadHeader(f *testing.F) { + fuzzConsensusSpecType[*types.ExecutionPayloadHeader](f, "ExecutionPayloadHeader") +} +func FuzzConsensusSpecsExecutionPayloadHeaderCapella(f *testing.F) { + fuzzConsensusSpecType[*types.ExecutionPayloadHeaderCapella](f, "ExecutionPayloadHeader") +} +func FuzzConsensusSpecsExecutionPayloadHeaderDeneb(f *testing.F) { + fuzzConsensusSpecType[*types.ExecutionPayloadHeaderDeneb](f, "ExecutionPayloadHeader") +} +func FuzzConsensusSpecsFork(f *testing.F) { + fuzzConsensusSpecType[*types.Fork](f, "Fork") +} func FuzzConsensusSpecsHistoricalBatch(f *testing.F) { fuzzConsensusSpecType[*types.HistoricalBatch](f, "HistoricalBatch") } func FuzzConsensusSpecsHistoricalBatchVariation(f *testing.F) { fuzzConsensusSpecType[*types.HistoricalBatchVariation](f, "HistoricalBatch") } +func FuzzConsensusSpecsHistoricalSummary(f *testing.F) { + fuzzConsensusSpecType[*types.HistoricalSummary](f, "HistoricalSummary") +} func FuzzConsensusSpecsIndexedAttestation(f *testing.F) { fuzzConsensusSpecType[*types.IndexedAttestation](f, "IndexedAttestation") } +func FuzzConsensusSpecsPendingAttestation(f *testing.F) { + fuzzConsensusSpecType[*types.PendingAttestation](f, "PendingAttestation") +} func FuzzConsensusSpecsProposerSlashing(f *testing.F) { fuzzConsensusSpecType[*types.ProposerSlashing](f, "ProposerSlashing") } func FuzzConsensusSpecsSignedBeaconBlockHeader(f *testing.F) { fuzzConsensusSpecType[*types.SignedBeaconBlockHeader](f, "SignedBeaconBlockHeader") } +func FuzzConsensusSpecsSignedBLSToExecutionChange(f *testing.F) { + fuzzConsensusSpecType[*types.SignedBLSToExecutionChange](f, "SignedBLSToExecutionChange") +} func FuzzConsensusSpecsSignedVoluntaryExit(f *testing.F) { fuzzConsensusSpecType[*types.SignedVoluntaryExit](f, "SignedVoluntaryExit") } -func FuzzConsensusSpecsVoluntaryExit(f *testing.F) { - fuzzConsensusSpecType[*types.VoluntaryExit](f, "VoluntaryExit") +func FuzzConsensusSpecsSyncAggregate(f *testing.F) { + fuzzConsensusSpecType[*types.SyncAggregate](f, "SyncAggregate") +} +func FuzzConsensusSpecsSyncCommittee(f *testing.F) { + fuzzConsensusSpecType[*types.SyncCommittee](f, "SyncCommittee") } func FuzzConsensusSpecsValidator(f *testing.F) { fuzzConsensusSpecType[*types.Validator](f, "Validator") } +func FuzzConsensusSpecsVoluntaryExit(f *testing.F) { + fuzzConsensusSpecType[*types.VoluntaryExit](f, "VoluntaryExit") +} func FuzzConsensusSpecsWithdrawal(f *testing.F) { fuzzConsensusSpecType[*types.Withdrawal](f, "Withdrawal") } @@ -394,14 +449,13 @@ func FuzzConsensusSpecsWithdrawalVariation(f *testing.F) { } func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string) { - // Iterate over all the forks and collect all the sample data. It's fine to - // have mismatching type version and test data, it's just going to skip on - // the first parse as bad data. + // Iterate over all the forks and collect all the sample data forks, err := os.ReadDir(consensusSpecTestsRoot) if err != nil { f.Errorf("failed to walk spec collection %v: %v", consensusSpecTestsRoot, err) return } + var valids [][]byte for _, fork := range forks { // Skip test cases for types introduced in later forks path := filepath.Join(consensusSpecTestsRoot, fork.Name(), "ssz_static", kind, "ssz_random") @@ -413,7 +467,7 @@ func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string) f.Errorf("failed to walk test collection %v: %v", path, err) return } - // Feed all the test data into the fuzzer + // Feed all the valid test data into the fuzzer for _, test := range tests { inSnappy, err := os.ReadFile(filepath.Join(path, test.Name(), "serialized.ssz_snappy")) if err != nil { @@ -423,11 +477,22 @@ func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string) if err != nil { f.Fatalf("failed to parse snappy ssz binary: %v", err) } - f.Add(inSSZ) + obj := T(new(U)) + if err := ssz.DecodeFromStream(bytes.NewReader(inSSZ), obj, uint32(len(inSSZ))); err == nil { + // Stash away all valid ssz streams so we can play with decoding + // into previously used objects + valids = append(valids, inSSZ) + + // Add the valid ssz stream to the fuzzer + f.Add(inSSZ) + } } } // Run the fuzzer f.Fuzz(func(t *testing.T, inSSZ []byte) { + // Track whether the testcase is valid + var valid bool + // Try the stream encoder/decoder obj := T(new(U)) if err := ssz.DecodeFromStream(bytes.NewReader(inSSZ), obj, uint32(len(inSSZ))); err == nil { @@ -448,6 +513,7 @@ func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string) if size := ssz.Size(obj); size != uint32(len(inSSZ)) { t.Fatalf("reported/generated size mismatch: reported %v, generated %v", size, len(inSSZ)) } + valid = true } // Try the buffer encoder/decoder obj = T(new(U)) @@ -470,5 +536,51 @@ func fuzzConsensusSpecType[T newableObject[U], U any](f *testing.F, kind string) t.Fatalf("reported/generated size mismatch: reported %v, generated %v", size, len(inSSZ)) } } + // If the testcase was valid, try decoding it into a used object + if valid { + // Pick a random starting object + vSSZ := valids[rand.N(len(valids))] + + // Try the stream encoder/decoder into a prepped object + obj = T(new(U)) + if err := ssz.DecodeFromBytes(vSSZ, obj); err != nil { + panic(err) // we've already decoded this, cannot fail + } + if err := ssz.DecodeFromStream(bytes.NewReader(inSSZ), obj, uint32(len(inSSZ))); err != nil { + t.Fatalf("failed to decode stream into used object: %v", err) + } + blob := new(bytes.Buffer) + if err := ssz.EncodeToStream(blob, obj); err != nil { + t.Fatalf("failed to re-encode stream from used object: %v", err) + } + if !bytes.Equal(blob.Bytes(), inSSZ) { + prefix := commonPrefix(blob.Bytes(), inSSZ) + t.Fatalf("re-encoded stream from used object mismatch: have %x, want %x, common prefix %d, have left %x, want left %x", + blob, inSSZ, len(prefix), blob.Bytes()[len(prefix):], inSSZ[len(prefix):]) + } + if size := ssz.Size(obj); size != uint32(len(inSSZ)) { + t.Fatalf("reported/generated size mismatch: reported %v, generated %v", size, len(inSSZ)) + } + // Try the buffer encoder/decoder into a prepped object + obj = T(new(U)) + if err := ssz.DecodeFromBytes(vSSZ, obj); err != nil { + panic(err) // we've already decoded this, cannot fail + } + if err := ssz.DecodeFromBytes(inSSZ, obj); err != nil { + t.Fatalf("failed to decode buffer into used object: %v", err) + } + bin := make([]byte, ssz.Size(obj)) + if err := ssz.EncodeToBytes(bin, obj); err != nil { + t.Fatalf("failed to re-encode buffer from used object: %v", err) + } + if !bytes.Equal(bin, inSSZ) { + prefix := commonPrefix(bin, inSSZ) + t.Fatalf("re-encoded buffer from used object mismatch: have %x, want %x, common prefix %d, have left %x, want left %x", + blob, inSSZ, len(prefix), bin[len(prefix):], inSSZ[len(prefix):]) + } + if size := ssz.Size(obj); size != uint32(len(inSSZ)) { + t.Fatalf("reported/generated size mismatch: reported %v, generated %v", size, len(inSSZ)) + } + } }) } diff --git a/tests/testtypes/consensus-spec-tests/gen_proposed_slashing_ssz.go b/tests/testtypes/consensus-spec-tests/gen_proposer_slashing_ssz.go similarity index 100% rename from tests/testtypes/consensus-spec-tests/gen_proposed_slashing_ssz.go rename to tests/testtypes/consensus-spec-tests/gen_proposer_slashing_ssz.go diff --git a/tests/testtypes/consensus-spec-tests/types_consensus.go b/tests/testtypes/consensus-spec-tests/types_consensus.go index 1b5f422..531c8f3 100644 --- a/tests/testtypes/consensus-spec-tests/types_consensus.go +++ b/tests/testtypes/consensus-spec-tests/types_consensus.go @@ -28,7 +28,7 @@ import ( //go:generate go run ../../../cmd/sszgen -type HistoricalSummary -out gen_historical_summary_ssz.go //go:generate go run ../../../cmd/sszgen -type IndexedAttestation -out gen_indexed_attestation_ssz.go //go:generate go run ../../../cmd/sszgen -type PendingAttestation -out gen_pending_attestation_ssz.go -//go:generate go run ../../../cmd/sszgen -type ProposerSlashing -out gen_proposed_slashing_ssz.go +//go:generate go run ../../../cmd/sszgen -type ProposerSlashing -out gen_proposer_slashing_ssz.go //go:generate go run ../../../cmd/sszgen -type SignedBeaconBlockHeader -out gen_signed_beacon_block_header_ssz.go //go:generate go run ../../../cmd/sszgen -type SignedBLSToExecutionChange -out gen_signed_bls_to_execution_change_ssz.go //go:generate go run ../../../cmd/sszgen -type SignedVoluntaryExit -out gen_signed_voluntary_exit_ssz.go