From dc8fb10535cd71423ca89baf2604b04f1ccdd357 Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Wed, 23 Aug 2023 14:39:20 +0200 Subject: [PATCH 1/6] Fix url filepath base --- http-range.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/http-range.go b/http-range.go index c66e6a0b..591d0d88 100644 --- a/http-range.go +++ b/http-range.go @@ -9,6 +9,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/goware/urlx" ) type ReaderAtCloser interface { @@ -71,10 +73,15 @@ func remoteHTTPFileAsIoReaderAt(ctx context.Context, url string) (ReaderAtCloser contentLength: contentLength, client: newHTTPClient(), } + parsedURL, err := urlx.Parse(url) + if err != nil { + return nil, err + } + name := filepath.Base(parsedURL.Path) rc := NewRangeCache( contentLength, - filepath.Base(url), + name, func(p []byte, off int64) (n int, err error) { return remoteReadAt(rr.client, rr.url, p, off) }) From 7e42397c54d0ce91179a958a2c5e92f7d1e22c5a Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Wed, 23 Aug 2023 14:47:28 +0200 Subject: [PATCH 2/6] Fix tests --- range-cache_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/range-cache_test.go b/range-cache_test.go index 2fd91fac..794e0a9e 100644 --- a/range-cache_test.go +++ b/range-cache_test.go @@ -13,9 +13,12 @@ func TestCache(t *testing.T) { v := []byte("hello") full := append(v, []byte(" world")...) rd := bytes.NewReader(full) - rc := NewRangeCache(int64(len(full)), func(p []byte, off int64) (n int, err error) { - return rd.ReadAt(p, off) - }) + rc := NewRangeCache( + int64(len(full)), + "test", + func(p []byte, off int64) (n int, err error) { + return rd.ReadAt(p, off) + }) { { From eb0e6f957a07ce99756750bdaf2f4886ad63008b Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Wed, 23 Aug 2023 14:56:06 +0200 Subject: [PATCH 3/6] If innerInstruction doesn't have index, then set to zero; closes #45 --- cmd-rpc-server-car-getTransaction.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd-rpc-server-car-getTransaction.go b/cmd-rpc-server-car-getTransaction.go index 5540b694..29ec8340 100644 --- a/cmd-rpc-server-car-getTransaction.go +++ b/cmd-rpc-server-car-getTransaction.go @@ -288,6 +288,10 @@ func adaptTransactionMetaToExpectedOutput(m map[string]any) map[string]any { if !ok { continue } + // If doesn't have `index`, then set it to 0 + if _, ok := innerInstruction["index"]; !ok { + innerInstruction["index"] = 0 + } instructionsAny, ok := innerInstruction["instructions"] if !ok { continue From 4f2c07d368ea6ce6280f5acf00509cbc86a887af Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Wed, 23 Aug 2023 16:57:15 +0200 Subject: [PATCH 4/6] Fix preload --- multiepoch-getBlock.go | 21 +++++++++++++++------ solana-block-rewards/rewards.go | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/multiepoch-getBlock.go b/multiepoch-getBlock.go index 33a08ef8..da821194 100644 --- a/multiepoch-getBlock.go +++ b/multiepoch-getBlock.go @@ -125,16 +125,25 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex } idealEntrySize := uint64(36190) - maybeOffsetOfLastEntry := parentOffset - idealEntrySize - length += idealEntrySize + var start uint64 + if parentIsInPreviousEpoch { + start = parentOffset + } else { + if parentOffset > idealEntrySize { + start = parentOffset - idealEntrySize + } else { + start = parentOffset + } + length += idealEntrySize + } - klog.Infof("prefetching CAR: start=%d length=%d (parent_offset=%d)", maybeOffsetOfLastEntry, length, parentOffset) - carSection, err := epochHandler.ReadAtFromCar(ctx, maybeOffsetOfLastEntry, length) + klog.Infof("prefetching CAR: start=%d length=%d (parent_offset=%d)", start, length, parentOffset) + carSection, err := epochHandler.ReadAtFromCar(ctx, start, length) if err != nil { return err } dr := bytes.NewReader(carSection) - { + if !parentIsInPreviousEpoch { dr.Seek(int64(idealEntrySize), io.SeekStart) } br := bufio.NewReader(dr) @@ -154,7 +163,7 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex if errors.Is(err, io.EOF) { break } - return err + return fmt.Errorf("failed to read node: %w", err) } if gotCid.Equals(blockCid) { break diff --git a/solana-block-rewards/rewards.go b/solana-block-rewards/rewards.go index 846d53aa..8bbd2024 100644 --- a/solana-block-rewards/rewards.go +++ b/solana-block-rewards/rewards.go @@ -1,8 +1,8 @@ package solanablockrewards import ( - "github.com/golang/protobuf/proto" "github.com/rpcpool/yellowstone-faithful/third_party/solana_proto/confirmed_block" + "google.golang.org/protobuf/proto" ) func ParseRewards(buf []byte) (*confirmed_block.Rewards, error) { From 6268deb2ae31128a4cd9cc876b6f0dbf20189dcb Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Wed, 23 Aug 2023 17:27:06 +0200 Subject: [PATCH 5/6] Fix rewards values --- multiepoch-getBlock.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/multiepoch-getBlock.go b/multiepoch-getBlock.go index da821194..514701a5 100644 --- a/multiepoch-getBlock.go +++ b/multiepoch-getBlock.go @@ -294,7 +294,13 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex rewardsAsArray := m["rewards"].([]any) for _, reward := range rewardsAsArray { rewardAsMap := reward.(map[string]any) - rewardAsMap["commission"] = nil + if _, ok := rewardAsMap["commission"]; !ok { + rewardAsMap["commission"] = nil + } + // if the commission field is a string, convert it to a float + if asString, ok := rewardAsMap["commission"].(string); ok { + rewardAsMap["commission"] = asFloat(asString) + } // if it has a post_balance field, convert it to postBalance if _, ok := rewardAsMap["post_balance"]; ok { @@ -442,6 +448,15 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex return nil, nil } +func asFloat(s string) float64 { + var f float64 + _, err := fmt.Sscanf(s, "%f", &f) + if err != nil { + panic(err) + } + return f +} + func mergeTxNodeSlices(slices [][]*ipldbindcode.Transaction) []*ipldbindcode.Transaction { var out []*ipldbindcode.Transaction for _, slice := range slices { From 1a12294ad1f75895ffbc897b256f46792cfff2b4 Mon Sep 17 00:00:00 2001 From: gagliardetto Date: Wed, 23 Aug 2023 17:34:55 +0200 Subject: [PATCH 6/6] Add lamports=0 field to rewards --- multiepoch-getBlock.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/multiepoch-getBlock.go b/multiepoch-getBlock.go index 514701a5..18650314 100644 --- a/multiepoch-getBlock.go +++ b/multiepoch-getBlock.go @@ -301,6 +301,10 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex if asString, ok := rewardAsMap["commission"].(string); ok { rewardAsMap["commission"] = asFloat(asString) } + // if no lamports field, add it and set it to 0 + if _, ok := rewardAsMap["lamports"]; !ok { + rewardAsMap["lamports"] = uint64(0) + } // if it has a post_balance field, convert it to postBalance if _, ok := rewardAsMap["post_balance"]; ok {