From f8b7420c22b3dab966dd5f08d3a01189667973cd Mon Sep 17 00:00:00 2001 From: Charles Billette Date: Wed, 14 Aug 2024 08:42:33 -0400 Subject: [PATCH 1/4] Add batch size flag for block fetching Introduce a new "--block-fetch-batch-size" flag to set the number of blocks fetched in parallel. Adjust poller logic to use the specified batch size, improving fetch operation efficiency. --- cmd/firevara/fetcher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/firevara/fetcher.go b/cmd/firevara/fetcher.go index 8c28b5c..bb4f8be 100644 --- a/cmd/firevara/fetcher.go +++ b/cmd/firevara/fetcher.go @@ -26,6 +26,7 @@ func NewFetchCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command { cmd.Flags().String("state-dir", "/data", "location to store the cursor.json") cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch") cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval between fetch when latest block is not available yet") + cmd.Flags().Int64("block-fetch-batch-size", 1, "number of block to fetch in parallel") return cmd } @@ -41,6 +42,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut } fetchInterval := sflags.MustGetDuration(cmd, "interval-between-fetch") + batchSize := sflags.MustGetInt(cmd, "block-fetch-batch-size") logger.Info( "launching firehose-gear poller", @@ -72,7 +74,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut ) // never use batch downloading for blocks - err = poller.Run(ctx, startBlock, 1) + err = poller.Run(ctx, startBlock, batchSize) if err != nil { return fmt.Errorf("running poller: %w", err) } From 2d53d4f4ec06cefe134cfde4c76d26485add0bb0 Mon Sep 17 00:00:00 2001 From: Charles Billette Date: Wed, 14 Aug 2024 08:48:58 -0400 Subject: [PATCH 2/4] Change block-fetch-batch-size flag type to uint64 --- cmd/firevara/fetcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/firevara/fetcher.go b/cmd/firevara/fetcher.go index bb4f8be..a8e0635 100644 --- a/cmd/firevara/fetcher.go +++ b/cmd/firevara/fetcher.go @@ -26,7 +26,7 @@ func NewFetchCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command { cmd.Flags().String("state-dir", "/data", "location to store the cursor.json") cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch") cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval between fetch when latest block is not available yet") - cmd.Flags().Int64("block-fetch-batch-size", 1, "number of block to fetch in parallel") + cmd.Flags().Uint64("block-fetch-batch-size", 1, "number of block to fetch in parallel") return cmd } @@ -42,7 +42,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut } fetchInterval := sflags.MustGetDuration(cmd, "interval-between-fetch") - batchSize := sflags.MustGetInt(cmd, "block-fetch-batch-size") + batchSize := sflags.MustGetUint64(cmd, "block-fetch-batch-size") logger.Info( "launching firehose-gear poller", From d9bcf49276cedc17d49933db6953c4a653712bfe Mon Sep 17 00:00:00 2001 From: Charles Billette Date: Wed, 14 Aug 2024 08:49:59 -0400 Subject: [PATCH 3/4] Change block-fetch-batch-size flag type to Int --- cmd/firevara/fetcher.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/firevara/fetcher.go b/cmd/firevara/fetcher.go index a8e0635..495d128 100644 --- a/cmd/firevara/fetcher.go +++ b/cmd/firevara/fetcher.go @@ -26,7 +26,7 @@ func NewFetchCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command { cmd.Flags().String("state-dir", "/data", "location to store the cursor.json") cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch") cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval between fetch when latest block is not available yet") - cmd.Flags().Uint64("block-fetch-batch-size", 1, "number of block to fetch in parallel") + cmd.Flags().Int("block-fetch-batch-size", 1, "number of block to fetch in parallel") return cmd } @@ -42,7 +42,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut } fetchInterval := sflags.MustGetDuration(cmd, "interval-between-fetch") - batchSize := sflags.MustGetUint64(cmd, "block-fetch-batch-size") + batchSize := sflags.MustGetInt(cmd, "block-fetch-batch-size") logger.Info( "launching firehose-gear poller", @@ -74,7 +74,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut ) // never use batch downloading for blocks - err = poller.Run(ctx, startBlock, batchSize) + err = poller.Run(ctx, startBlock, int(batchSize)) if err != nil { return fmt.Errorf("running poller: %w", err) } From 6022df074020fd8788f9f28e0f85ff0a30630ca3 Mon Sep 17 00:00:00 2001 From: Charles Billette Date: Wed, 14 Aug 2024 09:55:57 -0400 Subject: [PATCH 4/4] Reorganize contributing guidelines Moved contributing guidelines from README.md to a new CONTRIBUTING.md file for better maintainability and clarity. Updated instructions include steps for generating and publishing proto files. --- CONTRIBUTING.md | 27 +++++++++++++++++++++++++++ README.md | 9 --------- 2 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0d37957 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,27 @@ +# Contributing + +Report any protocol-specific issues in their +[respective repositories](https://github.com/streamingfast/streamingfast#protocols) + +**Please first refer to the general +[StreamingFast contribution guide](https://github.com/streamingfast/streamingfast/blob/master/CONTRIBUTING.md)**, +if you wish to contribute to this code base. + +This codebase uses unit tests extensively, please write and run tests. + +## Generate pb files from proto definition +```bash +cd proto +buf generate +``` + +## Publish proto definition and add it to `firecore` well known +```bash +cd proto +buf push +``` +### Once push go to `firecore` code and edit generator.go to add an entry to wellKnownProtoRepos list. +From root of `firecore` project run ... +```bash +go generate ./... +``` \ No newline at end of file diff --git a/README.md b/README.md index d0a9d9d..efb89fc 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,4 @@ The below command with start streaming Firehose Vara blocks, check `proto/sg/gea firevara fetch {FIRST_STREAMABLE_BLOCK} --endpoints {VARA_RPC_ENDPOINT} --state-dir {STATE_DIR} ``` -## Contributing -Report any protocol-specific issues in their -[respective repositories](https://github.com/streamingfast/streamingfast#protocols) - -**Please first refer to the general -[StreamingFast contribution guide](https://github.com/streamingfast/streamingfast/blob/master/CONTRIBUTING.md)**, -if you wish to contribute to this code base. - -This codebase uses unit tests extensively, please write and run tests.