Merge branch 'main' of github.com:streamingfast/firehose-gear

CONTRIBUTING.md

@@ -0,0 +1,27 @@
+# Contributing
+
+Report any protocol-specific issues in their
+[respective repositories](https://github.com/streamingfast/streamingfast#protocols)
+
+**Please first refer to the general
+[StreamingFast contribution guide](https://github.com/streamingfast/streamingfast/blob/master/CONTRIBUTING.md)**,
+if you wish to contribute to this code base.
+
+This codebase uses unit tests extensively, please write and run tests.
+
+## Generate pb files from proto definition
+```bash
+cd proto
+buf generate
+```
+
+## Publish proto definition and add it to `firecore` well known
+```bash
+cd proto
+buf push
+```
+### Once push go to `firecore` code and edit generator.go to add an entry to wellKnownProtoRepos list.
+From root of `firecore` project run  ...
+```bash
+go generate ./...
+```

README.md

@@ -32,12 +32,3 @@ curl --location 'https://vara-mainnet.public.blastapi.io' --header 'Content-Type
 ```
 
 ## Contributing
-
-Report any protocol-specific issues in their
-[respective repositories](https://github.com/streamingfast/streamingfast#protocols)
-
-**Please first refer to the general
-[StreamingFast contribution guide](https://github.com/streamingfast/streamingfast/blob/master/CONTRIBUTING.md)**,
-if you wish to contribute to this code base.
-
-This codebase uses unit tests extensively, please write and run tests.

cmd/firevara/fetcher.go

@@ -26,6 +26,7 @@ func NewFetchCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command {
 	cmd.Flags().String("state-dir", "/data", "location to store the cursor.json")
 	cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch")
 	cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval between fetch when latest block is not available yet")
+	cmd.Flags().Int("block-fetch-batch-size", 1, "number of block to fetch in parallel")
 
 	return cmd
 }
@@ -41,6 +42,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut
 		}
 
 		fetchInterval := sflags.MustGetDuration(cmd, "interval-between-fetch")
+		batchSize := sflags.MustGetInt(cmd, "block-fetch-batch-size")
 
 		logger.Info(
 			"launching firehose-gear poller",
@@ -72,7 +74,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut
 		)
 
 		// never use batch downloading for blocks
-		err = poller.Run(ctx, startBlock, 1)
+		err = poller.Run(ctx, startBlock, int(batchSize))
 		if err != nil {
 			return fmt.Errorf("running poller: %w", err)
 		}