From cb2282c1789073aee15080dd398b83d2f505a675 Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Wed, 17 May 2023 17:23:01 +0200 Subject: [PATCH] Index transaction signature to CID --- cmd-x-index-sig2cid.go | 64 ++++++++++ cmd-x-index.go | 1 + cmd-x-verify-index-sig2cid.go | 38 ++++++ cmd-x-verify-index.go | 1 + index-sig-to-cid.go | 228 ++++++++++++++++++++++++++++++++++ index-slot-to-cid.go | 2 +- 6 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 cmd-x-index-sig2cid.go create mode 100644 cmd-x-verify-index-sig2cid.go create mode 100644 index-sig-to-cid.go diff --git a/cmd-x-index-sig2cid.go b/cmd-x-index-sig2cid.go new file mode 100644 index 00000000..df4fc8c7 --- /dev/null +++ b/cmd-x-index-sig2cid.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "time" + + "github.com/urfave/cli/v2" + "k8s.io/klog/v2" +) + +func newCmd_Index_sig2cid() *cli.Command { + var verify bool + return &cli.Command{ + Name: "sig-to-cid", + Description: "Given a CAR file containing a Solana epoch, create an index of the file that maps transaction signatures to CIDs.", + ArgsUsage: " ", + Before: func(c *cli.Context) error { + return nil + }, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "verify", + Usage: "verify the index after creating it", + Destination: &verify, + }, + }, + Subcommands: []*cli.Command{}, + Action: func(c *cli.Context) error { + carPath := c.Args().Get(0) + indexDir := c.Args().Get(1) + + { + startedAt := time.Now() + defer func() { + klog.Infof("Finished in %s", time.Since(startedAt)) + }() + klog.Infof("Creating Sig-to-CID index for %s", carPath) + indexFilepath, err := CreateIndex_sig2cid( + context.TODO(), + carPath, + indexDir, + ) + if err != nil { + panic(err) + } + klog.Info("Index created") + if verify { + klog.Infof("Verifying index for %s located at %s", carPath, indexFilepath) + startedAt := time.Now() + defer func() { + klog.Infof("Finished in %s", time.Since(startedAt)) + }() + err := VerifyIndex_sig2cid(context.TODO(), carPath, indexFilepath) + if err != nil { + return cli.Exit(err, 1) + } + klog.Info("Index verified") + return nil + } + } + return nil + }, + } +} diff --git a/cmd-x-index.go b/cmd-x-index.go index bc20a43a..8624366d 100644 --- a/cmd-x-index.go +++ b/cmd-x-index.go @@ -15,6 +15,7 @@ func newCmd_Index() *cli.Command { Subcommands: []*cli.Command{ newCmd_Index_cid2offset(), newCmd_Index_slot2cid(), + newCmd_Index_sig2cid(), }, } } diff --git a/cmd-x-verify-index-sig2cid.go b/cmd-x-verify-index-sig2cid.go new file mode 100644 index 00000000..da3a9bd5 --- /dev/null +++ b/cmd-x-verify-index-sig2cid.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "time" + + "github.com/urfave/cli/v2" + "k8s.io/klog/v2" +) + +func newCmd_VerifyIndex_sig2cid() *cli.Command { + return &cli.Command{ + Name: "sig-to-cid", + Description: "Verify the index of the CAR file that maps transaction signatures to CIDs.", + ArgsUsage: " ", + Before: func(c *cli.Context) error { + return nil + }, + Flags: []cli.Flag{}, + Action: func(c *cli.Context) error { + carPath := c.Args().Get(0) + indexFilePath := c.Args().Get(1) + { + startedAt := time.Now() + defer func() { + klog.Infof("Finished in %s", time.Since(startedAt)) + }() + klog.Infof("Verifying Sig-to-CID index for %s", carPath) + err := VerifyIndex_sig2cid(context.TODO(), carPath, indexFilePath) + if err != nil { + return err + } + klog.Info("Index verified successfully") + } + return nil + }, + } +} diff --git a/cmd-x-verify-index.go b/cmd-x-verify-index.go index e5375a55..a0c25f96 100644 --- a/cmd-x-verify-index.go +++ b/cmd-x-verify-index.go @@ -15,6 +15,7 @@ func newCmd_VerifyIndex() *cli.Command { Subcommands: []*cli.Command{ newCmd_VerifyIndex_cid2offset(), newCmd_VerifyIndex_slot2cid(), + newCmd_VerifyIndex_sig2cid(), }, } } diff --git a/index-sig-to-cid.go b/index-sig-to-cid.go new file mode 100644 index 00000000..2e67ca53 --- /dev/null +++ b/index-sig-to-cid.go @@ -0,0 +1,228 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" + "github.com/ipfs/go-cid" + carv2 "github.com/ipld/go-car/v2" + "github.com/rpcpool/yellowstone-faithful/compactindex36" + "go.firedancer.io/radiance/cmd/radiance/car/createcar/ipld/ipldbindcode" + "k8s.io/klog/v2" +) + +// CreateIndex_sig2cid creates an index file that maps transaction signatures to CIDs. +func CreateIndex_sig2cid(ctx context.Context, carPath string, indexDir string) (string, error) { + // Check if the CAR file exists: + exists, err := fileExists(carPath) + if err != nil { + return "", fmt.Errorf("failed to check if CAR file exists: %w", err) + } + if !exists { + return "", fmt.Errorf("CAR file %q does not exist", carPath) + } + + cr, err := carv2.OpenReader(carPath) + if err != nil { + return "", fmt.Errorf("failed to open CAR file: %w", err) + } + + // check it has 1 root + roots, err := cr.Roots() + if err != nil { + return "", fmt.Errorf("failed to get roots: %w", err) + } + // There should be only one root CID in the CAR file. + if len(roots) != 1 { + return "", fmt.Errorf("CAR file has %d roots, expected 1", len(roots)) + } + + // TODO: use another way to precisely count the number of solana Blocks in the CAR file. + klog.Infof("Counting items in car file...") + numItems, err := carCountItems(carPath) + if err != nil { + return "", fmt.Errorf("failed to count items in car file: %w", err) + } + klog.Infof("Found %d items in car file", numItems) + + klog.Infof("Creating builder with %d items", numItems) + c2o, err := compactindex36.NewBuilder( + "", + uint(numItems), // TODO: what if the number of real items is less than this? + (0), + ) + if err != nil { + return "", fmt.Errorf("failed to open index store: %w", err) + } + defer c2o.Close() + + numItemsIndexed := uint64(0) + klog.Infof("Indexing...") + + dr, err := cr.DataReader() + if err != nil { + return "", fmt.Errorf("failed to get data reader: %w", err) + } + + // Iterate over all Transactions in the CAR file and put them into the index, + // using the transaction signature as the key and the CID as the value. + err = FindTransactions( + ctx, + dr, + func(c cid.Cid, txNode *ipldbindcode.Transaction) error { + var tx solana.Transaction + if err := bin.UnmarshalBin(&tx, txNode.Data); err != nil { + return fmt.Errorf("failed to unmarshal transaction: %w", err) + } else if len(tx.Signatures) == 0 { + panic("no signatures") + } + sig := tx.Signatures[0] + + var buf [36]byte + copy(buf[:], c.Bytes()[:36]) + + err = c2o.Insert(sig[:], buf) + if err != nil { + return fmt.Errorf("failed to put cid to offset: %w", err) + } + + numItemsIndexed++ + if numItemsIndexed%10_000 == 0 { + printToStderr(".") + } + return nil + }) + if err != nil { + return "", fmt.Errorf("failed to index; error while iterating over blocks: %w", err) + } + + rootCID := roots[0] + + // Use the car file name and root CID to name the index file: + indexFilePath := filepath.Join(indexDir, fmt.Sprintf("%s.%s.sig-to-cid.index", filepath.Base(carPath), rootCID.String())) + + klog.Infof("Creating index file at %s", indexFilePath) + targetFile, err := os.Create(indexFilePath) + if err != nil { + return "", fmt.Errorf("failed to create index file: %w", err) + } + defer targetFile.Close() + + klog.Infof("Sealing index...") + if err = c2o.Seal(ctx, targetFile); err != nil { + return "", fmt.Errorf("failed to seal index: %w", err) + } + klog.Infof("Index created") + return indexFilePath, nil +} + +// VerifyIndex_sig2cid verifies that the index file is correct for the given car file. +// It does this by reading the car file and comparing the offsets in the index +// file to the offsets in the car file. +func VerifyIndex_sig2cid(ctx context.Context, carPath string, indexFilePath string) error { + // Check if the CAR file exists: + exists, err := fileExists(carPath) + if err != nil { + return fmt.Errorf("failed to check if CAR file exists: %w", err) + } + if !exists { + return fmt.Errorf("CAR file %s does not exist", carPath) + } + + // Check if the index file exists: + exists, err = fileExists(indexFilePath) + if err != nil { + return fmt.Errorf("failed to check if index file exists: %w", err) + } + if !exists { + return fmt.Errorf("index file %s does not exist", indexFilePath) + } + + cr, err := carv2.OpenReader(carPath) + if err != nil { + return fmt.Errorf("failed to open CAR file: %w", err) + } + + // check it has 1 root + roots, err := cr.Roots() + if err != nil { + return fmt.Errorf("failed to get roots: %w", err) + } + // There should be only one root CID in the CAR file. + if len(roots) != 1 { + return fmt.Errorf("CAR file has %d roots, expected 1", len(roots)) + } + + indexFile, err := os.Open(indexFilePath) + if err != nil { + return fmt.Errorf("failed to open index file: %w", err) + } + defer indexFile.Close() + + c2o, err := compactindex36.Open(indexFile) + if err != nil { + return fmt.Errorf("failed to open index: %w", err) + } + + dr, err := cr.DataReader() + if err != nil { + return fmt.Errorf("failed to get data reader: %w", err) + } + + numItems := uint64(0) + err = FindTransactions( + ctx, + dr, + func(c cid.Cid, txNode *ipldbindcode.Transaction) error { + var tx solana.Transaction + if err := bin.UnmarshalBin(&tx, txNode.Data); err != nil { + return fmt.Errorf("failed to unmarshal transaction: %w", err) + } else if len(tx.Signatures) == 0 { + panic("no signatures") + } + sig := tx.Signatures[0] + + got, err := findCidFromSig(c2o, sig) + if err != nil { + return fmt.Errorf("failed to put cid to offset: %w", err) + } + + if !got.Equals(c) { + return fmt.Errorf("sig %s: expected cid %s, got %s", sig, c, got) + } + + numItems++ + if numItems%10_000 == 0 { + printToStderr(".") + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to verify index; error while iterating over blocks: %w", err) + } + return nil +} + +func findCidFromSig(db *compactindex36.DB, sig solana.Signature) (cid.Cid, error) { + bucket, err := db.LookupBucket(sig[:]) + if err != nil { + return cid.Cid{}, fmt.Errorf("failed to lookup bucket for %s: %w", sig, err) + } + got, err := bucket.Lookup(sig[:]) + if err != nil { + return cid.Cid{}, fmt.Errorf("failed to lookup value for %s: %w", sig, err) + } + l, c, err := cid.CidFromBytes(got[:]) + if err != nil { + return cid.Cid{}, fmt.Errorf("failed to parse cid from bytes: %w", err) + } + if l != 36 { + return cid.Cid{}, fmt.Errorf("unexpected cid length %d", l) + } + return c, nil +} diff --git a/index-slot-to-cid.go b/index-slot-to-cid.go index b2fe989e..e715d7d2 100644 --- a/index-slot-to-cid.go +++ b/index-slot-to-cid.go @@ -183,7 +183,7 @@ func VerifyIndex_slot2cid(ctx context.Context, carPath string, indexFilePath str } if !got.Equals(c) { - return fmt.Errorf("slot %d: expected %s, got %s", slotNum, c, got) + return fmt.Errorf("slot %d: expected cid %s, got %s", slotNum, c, got) } numItems++