-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
# Conflicts: # go.mod
- Loading branch information
Showing
5 changed files
with
279 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,113 @@ | ||
package block | ||
|
||
import ( | ||
"bytes" | ||
"fmt" | ||
"reflect" | ||
"strings" | ||
|
||
"github.com/streamingfast/substreams-gear/generated/convert1420" | ||
Check failure on line 9 in block/decoder.go GitHub Actions / build (1.22.x)
|
||
|
||
"github.com/centrifuge/go-substrate-rpc-client/v4/registry" | ||
"github.com/centrifuge/go-substrate-rpc-client/v4/scale" | ||
"github.com/centrifuge/go-substrate-rpc-client/v4/types" | ||
"github.com/gobeam/stringy" | ||
pbgear "github.com/streamingfast/firehose-gear/pb/sf/gear/type/v1" | ||
v1 "github.com/streamingfast/substreams-gear/pb/sf/substreams/gear/type/v1" | ||
Check failure on line 16 in block/decoder.go GitHub Actions / build (1.22.x)
|
||
"go.uber.org/zap" | ||
) | ||
|
||
type Decoder struct { | ||
callRegistry registry.CallRegistry | ||
logger *zap.Logger | ||
} | ||
|
||
func NewDecoder(logger *zap.Logger) *Decoder { | ||
return &Decoder{ | ||
logger: logger, | ||
} | ||
} | ||
|
||
func (d *Decoder) Decoded(block *pbgear.Block) (*v1.Block, error) { | ||
|
||
decodedExtrinsics, err := decodeExtrinsics(d.callRegistry, block.Extrinsics, d.logger) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to decode extrinsics: %w", err) | ||
} | ||
|
||
return &v1.Block{ | ||
Number: block.Number, | ||
Hash: block.Hash, | ||
Header: block.Header, | ||
DigestItems: block.DigestItems, | ||
Justification: block.Justification, | ||
Extrinsics: decodedExtrinsics, | ||
Events: nil, | ||
}, nil | ||
} | ||
|
||
var versionFuncMap map[string]map[string]reflect.Value | ||
|
||
func init() { | ||
versionFuncMap = map[string]map[string]reflect.Value{ | ||
"1420": convert1420.FuncMap, | ||
} | ||
} | ||
|
||
func decodeExtrinsics(version string, callRegistry registry.CallRegistry, extrinsics []*pbgear.Extrinsic, logger *zap.Logger) ([]*v1.Extrinsic, error) { | ||
var decodedExtrinsics []*v1.Extrinsic | ||
for i, extrinsic := range extrinsics { | ||
logger.Info("decoding extrinsic", zap.Int("index", i), zap.Uint32("section", extrinsic.Method.CallIndex.SectionIndex), zap.Uint32("method", extrinsic.Method.CallIndex.MethodIndex)) | ||
callName, decodedFields, err := decodeCallExtrinsics(callRegistry, extrinsic) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to decode extrinsic: %w", err) | ||
} | ||
|
||
parts := strings.Split(callName, ".") | ||
pallet := parts[0] | ||
call := parts[1] | ||
call = stringy.New(call).PascalCase().Get() | ||
structName := pallet + "_" + call + "Call" | ||
funcName := "To_" + structName | ||
|
||
funcMap := versionFuncMap[] | ||
if fn, found := funcMap[funcName]; found { | ||
o := fn.Call([]reflect.Value{reflect.ValueOf(decodedFields)}) | ||
|
||
e := o[0].Interface().(*v1.Extrinsic) | ||
decodedExtrinsics = append(decodedExtrinsics, e) | ||
|
||
} else { | ||
panic(fmt.Sprintf("unknown extrinsic call: %s", callName)) | ||
} | ||
} | ||
return decodedExtrinsics, nil | ||
} | ||
|
||
func decodeCallExtrinsics(callRegistry registry.CallRegistry, extrinsic *pbgear.Extrinsic) (string, registry.DecodedFields, error) { | ||
callIndex := extrinsic.Method.CallIndex | ||
args := extrinsic.Method.Args | ||
|
||
callDecoder, found := callRegistry[toCallIndex(callIndex)] | ||
if !found { | ||
return "", nil, fmt.Errorf("failed to get call decoder for call at index %d %d", callIndex.SectionIndex, callIndex.MethodIndex) | ||
} | ||
|
||
if args != nil { | ||
decoder := scale.NewDecoder(bytes.NewReader(args)) | ||
callFields, err := callDecoder.Decode(decoder) | ||
if err != nil { | ||
return "", nil, fmt.Errorf("failed to decode call: %w", err) | ||
} | ||
return callDecoder.Name, callFields, nil | ||
} | ||
|
||
return callDecoder.Name, nil, nil | ||
} | ||
|
||
func toCallIndex(ci *pbgear.CallIndex) types.CallIndex { | ||
return types.CallIndex{ | ||
SectionIndex: uint8(ci.SectionIndex), | ||
MethodIndex: uint8(ci.MethodIndex), | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,138 @@ | ||
package main | ||
|
||
import ( | ||
"encoding/hex" | ||
"encoding/json" | ||
"fmt" | ||
"io" | ||
"os" | ||
|
||
"github.com/centrifuge/go-substrate-rpc-client/v4/registry" | ||
|
||
"github.com/centrifuge/go-substrate-rpc-client/v4/types" | ||
"github.com/centrifuge/go-substrate-rpc-client/v4/types/codec" | ||
|
||
pbgear "github.com/streamingfast/firehose-gear/pb/sf/gear/type/v1" | ||
pbfirehose "github.com/streamingfast/pbgo/sf/firehose/v2" | ||
"google.golang.org/protobuf/proto" | ||
"google.golang.org/protobuf/types/known/anypb" | ||
|
||
"github.com/mostynb/go-grpc-compression/zstd" | ||
"google.golang.org/grpc" | ||
|
||
"github.com/streamingfast/cli/sflags" | ||
|
||
"github.com/streamingfast/firehose-core/firehose/client" | ||
|
||
"github.com/streamingfast/firehose-gear/block" | ||
|
||
"github.com/spf13/cobra" | ||
"github.com/streamingfast/logging" | ||
"go.uber.org/zap" | ||
) | ||
|
||
func NewDecoderCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command { | ||
cmd := &cobra.Command{ | ||
Use: "decode-block", | ||
Short: "Consume blocks from firehose and decode them", | ||
RunE: decodeRunE(logger, tracer), | ||
} | ||
cmd.Flags().Int64("first-streamable-block", 0, "first block to decode") | ||
cmd.Flags().BoolP("plaintext", "p", false, "Use plaintext connection to Firehose") | ||
cmd.Flags().BoolP("insecure", "k", false, "Use SSL connection to Firehose but skip SSL certificate validation") | ||
|
||
return cmd | ||
} | ||
|
||
func decodeRunE(logger *zap.Logger, tracer logging.Tracer) func(cmd *cobra.Command, args []string) error { | ||
return func(cmd *cobra.Command, args []string) error { | ||
ctx := cmd.Context() | ||
|
||
firehoseEndpoint := args[0] | ||
plaintext := sflags.MustGetBool(cmd, "plaintext") | ||
insecure := sflags.MustGetBool(cmd, "insecure") | ||
firstStreamableBlock := sflags.MustGetInt64(cmd, "first-streamable-block") | ||
|
||
decoder := block.NewDecoder(logger) | ||
jwt := os.Getenv("FIREHOSE_API_TOKEN") | ||
|
||
firehoseClient, connClose, grpcCallOpts, err := client.NewFirehoseClient(firehoseEndpoint, jwt, "", insecure, plaintext) | ||
if err != nil { | ||
return fmt.Errorf("creating firehose client: %w", err) | ||
} | ||
defer connClose() | ||
|
||
grpcCallOpts = append(grpcCallOpts, grpc.UseCompressor(zstd.Name)) | ||
|
||
//todo: load cursor here | ||
|
||
request := &pbfirehose.Request{ | ||
StartBlockNum: firstStreamableBlock, | ||
StopBlockNum: 0, | ||
FinalBlocksOnly: false, | ||
} | ||
|
||
stream, err := firehoseClient.Blocks(ctx, request, grpcCallOpts...) | ||
if err != nil { | ||
return fmt.Errorf("unable to start blocks stream: %w", err) | ||
} | ||
logger.Info("starting blocks stream") | ||
lastBlockNum := uint64(firstStreamableBlock) | ||
for { | ||
response, err := stream.Recv() | ||
if err != nil { | ||
if err == io.EOF { | ||
break | ||
} | ||
return fmt.Errorf("stream error while receiving: %w", err) | ||
} | ||
|
||
logger.Info("got response") | ||
block := &pbgear.Block{} | ||
if err := anypb.UnmarshalTo(response.Block, block, proto.UnmarshalOptions{}); err != nil { | ||
return fmt.Errorf("unmarshalling anypb: %w", err) | ||
} | ||
logger.Info("got block", zap.Uint64("block_num", block.Number)) | ||
|
||
if len(block.Header.UpdatedMetadata) > 0 { | ||
logger.Info("Updating metadata", zap.Uint64("block_num", block.Number), zap.Uint32("version", block.Header.SpecVersion)) | ||
md := LoadMetadata(block.Header.UpdatedMetadata) | ||
factory := registry.NewFactory() | ||
callRegistry, err := factory.CreateCallRegistry(md) | ||
if err != nil { | ||
return fmt.Errorf("creating call registry: %w", err) | ||
} | ||
decoder.SetCallRegistry(callRegistry) | ||
//todo: save has last metadata seen and reload it at startup | ||
} | ||
|
||
if lastBlockNum != 0 && (block.Number-lastBlockNum) != 1 { | ||
panic(fmt.Sprintf("incorrect number of blocks sequence. Expected %d got %d", lastBlockNum+1, block.Number)) | ||
} | ||
lastBlockNum = block.Number | ||
|
||
decodedBlock, err := decoder.Decoded(block) | ||
if err != nil { | ||
return fmt.Errorf("decoding block %d %s: %w", block.Number, hex.EncodeToString(block.Hash), err) | ||
} | ||
|
||
j, err := json.Marshal(decodedBlock) | ||
if err != nil { | ||
return fmt.Errorf("marshalling block %d: %w", block.Number, err) | ||
} | ||
fmt.Println(string(j)) | ||
//todo: print DM log | ||
//todo: save Cursor | ||
} | ||
return nil | ||
} | ||
} | ||
|
||
func LoadMetadata(data []byte) *types.Metadata { | ||
metadata := &types.Metadata{} | ||
err := codec.Decode(data, metadata) | ||
if err != nil { | ||
panic(err) | ||
} | ||
return metadata | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.