Add blockHeight to schema

cmd-rpc-server-car-getBlock.go

@@ -167,7 +167,7 @@ func (ser *rpcServer) handleGetBlock(ctx context.Context, conn *requestContext,
 	tim.time("get entries")
 
 	var allTransactions []GetTransactionResponse
-	var rewards any // TODO: implement rewards as in solana
+	var rewards any
 	hasRewards := !block.Rewards.(cidlink.Link).Cid.Equals(DummyCID)
 	if hasRewards {
 		rewardsNode, err := ser.GetRewardsByCid(ctx, block.Rewards.(cidlink.Link).Cid)
@@ -333,8 +333,14 @@ func (ser *rpcServer) handleGetBlock(ctx context.Context, conn *requestContext,
 	blockResp.BlockTime = &blocktime
 	blockResp.Blockhash = lastEntryHash.String()
 	blockResp.ParentSlot = uint64(block.Meta.Parent_slot)
-	blockResp.Rewards = rewards                                 // TODO: implement rewards as in solana
-	blockResp.BlockHeight = calcBlockHeight(uint64(block.Slot)) // TODO: implement block height
+	blockResp.Rewards = rewards
+
+	{
+		blockHeight, ok := block.GetBlockHeight()
+		if ok {
+			blockResp.BlockHeight = blockHeight
+		}
+	}
 	{
 		// get parent slot
 		parentSlot := uint64(block.Meta.Parent_slot)
@@ -352,27 +358,27 @@ func (ser *rpcServer) handleGetBlock(ctx context.Context, conn *requestContext,
 				return
 			}
 
-			lastEntryCidOfParent := parentBlock.Entries[len(parentBlock.Entries)-1]
-			parentEntryNode, err := ser.GetEntryByCid(ctx, lastEntryCidOfParent.(cidlink.Link).Cid)
-			if err != nil {
-				klog.Errorf("failed to decode Entry: %v", err)
-				conn.ReplyWithError(
-					ctx,
-					req.ID,
-					&jsonrpc2.Error{
-						Code:    jsonrpc2.CodeInternalError,
-						Message: "Internal error",
-					})
-				return
+			if len(parentBlock.Entries) > 0 {
+				lastEntryCidOfParent := parentBlock.Entries[len(parentBlock.Entries)-1]
+				parentEntryNode, err := ser.GetEntryByCid(ctx, lastEntryCidOfParent.(cidlink.Link).Cid)
+				if err != nil {
+					klog.Errorf("failed to decode Entry: %v", err)
+					conn.ReplyWithError(
+						ctx,
+						req.ID,
+						&jsonrpc2.Error{
+							Code:    jsonrpc2.CodeInternalError,
+							Message: "Internal error",
+						})
+					return
+				}
+				parentEntryHash := solana.HashFromBytes(parentEntryNode.Hash)
+				blockResp.PreviousBlockhash = parentEntryHash.String()
 			}
-			parentEntryHash := solana.HashFromBytes(parentEntryNode.Hash)
-			blockResp.PreviousBlockhash = parentEntryHash.String()
 		}
 	}
 	tim.time("get parent block")
 
-	// TODO: get all the transactions from the block
-	// reply with the data
 	err = conn.Reply(
 		ctx,
 		req.ID,

cmd-rpc-server-car-getSignaturesForAddress.go

@@ -207,7 +207,7 @@ func (ser *rpcServer) handleGetSignaturesForAddress(ctx context.Context, conn *r
 				slot := uint64(transactionNode.Slot)
 				response[ii]["slot"] = slot
 				response[ii]["blockTime"] = getBlockTime(slot)
-				response[ii]["confirmationStatus"] = "finalized" // TODO: is this correct?
+				response[ii]["confirmationStatus"] = "finalized"
 			}
 			return nil
 		})

cmd-rpc-server-car.go

@@ -944,8 +944,3 @@ func parseTransactionAndMetaFromNode(
 	}
 	return
 }
-
-func calcBlockHeight(slot uint64) uint64 {
-	// TODO: fix this
-	return 0
-}

cmd-x-index-all.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/dustin/go-humanize"
-	bin "github.com/gagliardetto/binary"
 	"github.com/gagliardetto/solana-go"
 	"github.com/ipfs/go-cid"
 	carv1 "github.com/ipld/go-car"
@@ -231,20 +230,10 @@ func createAllIndexes(
 					return nil, fmt.Errorf("failed to decode transaction: %w", err)
 				}
 
-				var tx solana.Transaction
-				txBuffer := new(bytes.Buffer)
-				txBuffer.Write(txNode.Data.Bytes())
-				if total, ok := txNode.Data.GetTotal(); ok && total > 1 {
-					// TODO: handle this case
-					klog.Infof("skipping transaction with %d partials", total)
-					continue
-				}
-				if err := bin.UnmarshalBin(&tx, txBuffer.Bytes()); err != nil {
-					return nil, fmt.Errorf("failed to unmarshal transaction: %w", err)
-				} else if len(tx.Signatures) == 0 {
-					panic("no signatures")
+				sig, err := readFirstSignature(txNode.Data.Bytes())
+				if err != nil {
+					return nil, fmt.Errorf("failed to read signature: %w", err)
 				}
-				sig := tx.Signatures[0]
 
 				err = sig_to_cid.Put(sig, _cid)
 				if err != nil {
@@ -616,20 +605,10 @@ func verifyAllIndexes(
 					return fmt.Errorf("failed to decode transaction: %w", err)
 				}
 
-				var tx solana.Transaction
-				txBuffer := new(bytes.Buffer)
-				txBuffer.Write(txNode.Data.Bytes())
-				if total, ok := txNode.Data.GetTotal(); ok && total > 1 {
-					// TODO: handle this case
-					klog.Infof("skipping transaction with %d partials", total)
-					continue
-				}
-				if err := bin.UnmarshalBin(&tx, txBuffer.Bytes()); err != nil {
-					return fmt.Errorf("failed to unmarshal transaction: %w", err)
-				} else if len(tx.Signatures) == 0 {
-					panic("no signatures")
+				sig, err := readFirstSignature(txNode.Data.Bytes())
+				if err != nil {
+					return fmt.Errorf("failed to read signature: %w", err)
 				}
-				sig := tx.Signatures[0]
 
 				got, err := sig_to_cid.Get(sig)
 				if err != nil {

go.mod

@@ -59,11 +59,12 @@ require (
 
 require (
 	github.com/ipld/go-car v0.5.0
+	github.com/mr-tron/base58 v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/ronanh/intcomp v1.1.0
 	github.com/tejzpr/ordered-concurrently/v3 v3.0.1
 	github.com/valyala/fasthttp v1.47.0
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/klog v1.0.0
 )
 
@@ -155,7 +156,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 // indirect
-	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
@@ -204,7 +204,6 @@ require (
 	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/tools v0.7.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
 	nhooyr.io/websocket v1.8.7 // indirect
 )

index-sig-to-cid.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"os"
@@ -82,20 +81,10 @@ func CreateIndex_sig2cid(ctx context.Context, tmpDir string, carPath string, ind
 		ctx,
 		dr,
 		func(c cid.Cid, txNode *ipldbindcode.Transaction) error {
-			var tx solana.Transaction
-			txBuffer := new(bytes.Buffer)
-			txBuffer.Write(txNode.Data.Bytes())
-			if total, ok := txNode.Data.GetTotal(); ok && total > 1 {
-				// TODO: handle this case
-				klog.Infof("skipping transaction with %d partials", total)
-				return nil
-			}
-			if err := bin.UnmarshalBin(&tx, txBuffer.Bytes()); err != nil {
-				return fmt.Errorf("failed to unmarshal transaction: %w", err)
-			} else if len(tx.Signatures) == 0 {
-				panic("no signatures")
+			sig, err := readFirstSignature(txNode.Data.Bytes())
+			if err != nil {
+				return fmt.Errorf("failed to read signature: %w", err)
 			}
-			sig := tx.Signatures[0]
 
 			var buf [36]byte
 			copy(buf[:], c.Bytes()[:36])
@@ -193,20 +182,10 @@ func VerifyIndex_sig2cid(ctx context.Context, carPath string, indexFilePath stri
 		ctx,
 		dr,
 		func(c cid.Cid, txNode *ipldbindcode.Transaction) error {
-			var tx solana.Transaction
-			txBuffer := new(bytes.Buffer)
-			txBuffer.Write(txNode.Data.Bytes())
-			if total, ok := txNode.Data.GetTotal(); ok && total > 1 {
-				// TODO: handle this case
-				klog.Infof("skipping transaction with %d partials", total)
-				return nil
-			}
-			if err := bin.UnmarshalBin(&tx, txBuffer.Bytes()); err != nil {
-				return fmt.Errorf("failed to unmarshal transaction: %w", err)
-			} else if len(tx.Signatures) == 0 {
-				panic("no signatures")
+			sig, err := readFirstSignature(txNode.Data.Bytes())
+			if err != nil {
+				return fmt.Errorf("failed to read signature: %w", err)
 			}
-			sig := tx.Signatures[0]
 
 			got, err := findCidFromSignature(c2o, sig)
 			if err != nil {
@@ -248,3 +227,25 @@ func findCidFromSignature(db *compactindex36.DB, sig solana.Signature) (cid.Cid,
 	}
 	return c, nil
 }
+
+func readFirstSignature(buf []byte) (solana.Signature, error) {
+	decoder := bin.NewCompactU16Decoder(buf)
+	numSigs, err := decoder.ReadCompactU16()
+	if err != nil {
+		return solana.Signature{}, err
+	}
+	if numSigs == 0 {
+		return solana.Signature{}, fmt.Errorf("no signatures")
+	}
+
+	// Read the first signature:
+	var sig solana.Signature
+	numRead, err := decoder.Read(sig[:])
+	if err != nil {
+		return solana.Signature{}, err
+	}
+	if numRead != 64 {
+		return solana.Signature{}, fmt.Errorf("unexpected signature length %d", numRead)
+	}
+	return sig, nil
+}

ipld/ipldbindcode/ledger.ipldsch

@@ -47,6 +47,8 @@ type SlotMeta struct {
   parent_slot         Int
   # Block time of this slot.
   blocktime           Int
+  # Block height of this slot.
+  block_height nullable optional Int
 } representation tuple
 
 type Shredding struct {
@@ -70,7 +72,7 @@ type Transaction struct {
   metadata DataFrame
   # The slot number where this transaction was created.
   slot     Int
-  # The index of this transaction in the the block (0-indexed).
+  # The index of the position of this transaction in the block (0-indexed).
   index nullable optional  Int
 } representation tuple
 

ipld/ipldbindcode/methods.go

@@ -109,3 +109,13 @@ func (n Transaction) GetPositionIndex() (int, bool) {
 	}
 	return **n.Index, true
 }
+
+// GetBlockHeight returns the 'block_height' field, which indicates
+// the height of the block, and
+// a flag indicating whether the field has a value.
+func (n Block) GetBlockHeight() (uint64, bool) {
+	if n.Meta.Block_height == nil || *n.Meta.Block_height == nil {
+		return 0, false
+	}
+	return uint64(**n.Meta.Block_height), true
+}

ipld/ipldbindcode/non-generated-types.go

@@ -0,0 +1,6 @@
+package ipldbindcode
+
+type (
+	Hash   []uint8
+	Buffer []uint8
+)

ipld/ipldbindcode/types.go

@@ -10,7 +10,6 @@ type (
 		Subsets List__Link
 	}
 )
-
 type Subset struct {
 	Kind   int
 	First  int
@@ -28,15 +27,15 @@ type (
 		Rewards   datamodel.Link
 	}
 )
-
 type Rewards struct {
 	Kind int
 	Slot int
 	Data DataFrame
 }
 type SlotMeta struct {
-	Parent_slot int
-	Blocktime   int
+	Parent_slot  int
+	Blocktime    int
+	Block_height **int
 }
 type Shredding struct {
 	EntryEndIdx int
@@ -63,8 +62,3 @@ type DataFrame struct {
 	Data  []uint8
 	Next  **List__Link
 }
-
-type (
-	Hash   []uint8
-	Buffer []uint8
-)