refine code

.gitignore

@@ -0,0 +1,5 @@
+.DS_Store
+.idea/*
+build/*
+.local
+config/local/*

Makefile

@@ -10,16 +10,16 @@ ldflags = -X $(REPO)/version.AppVersion=$(VERSION) \
 
 build_syncer:
 ifeq ($(OS),Windows_NT)
-	go build -o build/blob-syncer.exe -ldflags="$(ldflags)" main.go
+	go build -o build/blob-syncer.exe -ldflags="$(ldflags)" cmd/blob-syncer/main.go
 else
-	go build -o build/blob-syncer -ldflags="$(ldflags)" main.go
+	go build -o build/blob-syncer -ldflags="$(ldflags)" cmd/blob-syncer/main.go
 endif
 
 build_server:
 ifeq ($(OS),Windows_NT)
-	go build $(BUILD_FLAGS) -o build/server.exe cmd/blob-syncer-server/main.go
+	go build -o build/blob-syncer-server.exe -ldflags="$(ldflags)" cmd/blob-syncer-server/main.go
 else
-	go build $(BUILD_FLAGS) -o build/server cmd/blob-syncer-server/main.go
+	go build -o build/blob-syncer-server -ldflags="$(ldflags)" cmd/blob-syncer-server/main.go
 endif
 
 install:

README.MD

@@ -0,0 +1,12 @@
+run the syncer instance:
+```shell
+./build/blob-syncer --config-type local --config-path config/config.json
+````
+
+run the server instance
+
+```shell
+./build/blob-syncer-server --config-path config/config.json --port 8080 
+```
+
+

cache/cache.go

@@ -0,0 +1,32 @@
+package cache
+
+import lru "github.com/hashicorp/golang-lru"
+
+type Cache interface {
+	Get(key string) (interface{}, bool)
+	Set(key string, value interface{})
+}
+
+const DefaultCacheSize = 1024
+
+type LocalCache struct {
+	*lru.Cache
+}
+
+func NewLocalCache(size uint64) (Cache, error) {
+	cache, err := lru.New(int(size))
+	if err != nil {
+		return nil, err
+	}
+	return &LocalCache{
+		cache,
+	}, nil
+}
+
+func (c *LocalCache) Get(key string) (interface{}, bool) {
+	return c.Cache.Get(key)
+}
+
+func (c *LocalCache) Set(key string, value interface{}) {
+	c.Cache.Add(key, value)
+}

main.go → cmd/blob-syncer/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
 	"github.com/bnb-chain/blob-syncer/config"
@@ -145,20 +144,5 @@ func main() {
 }
 
 func getDBPass(cfg *config.DBConfig) string {
-	if cfg.KeyType == config.KeyTypeAWSPrivateKey {
-		result, err := config.GetSecret(cfg.AWSSecretName, cfg.AWSRegion)
-		if err != nil {
-			panic(err)
-		}
-		type DBPass struct {
-			DbPass string `json:"db_pass"`
-		}
-		var dbPassword DBPass
-		err = json.Unmarshal([]byte(result), &dbPassword)
-		if err != nil {
-			panic(err)
-		}
-		return dbPassword.DbPass
-	}
 	return cfg.Password
 }

config/config.go

@@ -3,35 +3,53 @@ package config
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/bnb-chain/blob-syncer/cache"
 	"os"
 )
 
 type Config struct {
 	LogConfig    LogConfig    `json:"log_config"`
 	DBConfig     DBConfig     `json:"db_config"`
 	SyncerConfig SyncerConfig `json:"syncer_config"`
+	ServerConfig ServerConfig `json:"server_config"`
+	CacheConfig  CacheConfig  `json:"cache_config"`
 }
 
 type SyncerConfig struct {
+	BucketName             string   `json:"bucket_name"`              // BucketName is the identifier of bucket on Greenfield that store blob
+	StartSlot              uint64   `json:"start_slot"`               // StartSlot is used to init the syncer which slot of beacon chain to synced from
+	BundleServiceEndpoints []string `json:"bundle_service_endpoints"` // BundleServiceEndpoints is a list of bundle service address
+	BeaconRPCAddrs         []string `json:"beacon_rpc_addrs"`         // BeaconRPCAddrs is a list of beacon chain RPC address
+	ETHRPCAddrs            []string `json:"eth_rpc_addrs"`
+	TempFilePath           string   `json:"temp_file_path"` // TempFilePath used to create file for every blob.
+	PrivateKey             string   `json:"private_key"`
+}
+
+type ServerConfig struct {
 	BucketName         string   `json:"bucket_name"`
-	StartHeight        uint64   `json:"start_height"`
 	BundleServiceAddrs []string `json:"bundle_service_addrs"`
-	BeaconAddrs        []string `json:"beacon_addrs"`
-	ETHRPCAddrs        []string `json:"eth_rpc_addrs"`
-	TempFilePath       string   `json:"temp_file_path"` // used to create file for every blob, bundle service might support sending stream.
-	PrivateKey         string   `json:"private_key"`
+}
+
+type CacheConfig struct {
+	CacheType string `json:"cache_type"`
+	URL       string `json:"url"`
+	CacheSize uint64 `json:"cache_size"`
+}
+
+func (c *CacheConfig) GetCacheSize() uint64 {
+	if c.CacheSize != 0 {
+		return c.CacheSize
+	}
+	return cache.DefaultCacheSize
 }
 
 type DBConfig struct {
-	Dialect       string `json:"dialect"`
-	KeyType       string `json:"key_type"`
-	AWSRegion     string `json:"aws_region"`
-	AWSSecretName string `json:"aws_secret_name"`
-	Password      string `json:"password"`
-	Username      string `json:"username"`
-	Url           string `json:"url"`
-	MaxIdleConns  int    `json:"max_idle_conns"`
-	MaxOpenConns  int    `json:"max_open_conns"`
+	Dialect      string `json:"dialect"`
+	Username     string `json:"username"`
+	Password     string `json:"password"`
+	Url          string `json:"url"`
+	MaxIdleConns int    `json:"max_idle_conns"`
+	MaxOpenConns int    `json:"max_open_conns"`
 }
 
 func (cfg *DBConfig) Validate() {

db/dao.go

@@ -82,12 +82,12 @@ func (d *BlobSvcDB) UpdateBundleStatus(bundleName string, status InnerBundleStat
 
 func (d *BlobSvcDB) SaveBlockAndBlob(block *Block, blobs []*Blob) error {
 	return d.db.Transaction(func(dbTx *gorm.DB) error {
-		err := dbTx.Create(block).Error
+		err := dbTx.Save(block).Error
 		if err != nil {
 			return err
 		}
 		if len(blobs) != 0 {
-			err := dbTx.Create(blobs).Error
+			err = dbTx.Save(blobs).Error
 			if err != nil {
 				return err
 			}

syncer/beacon_client.go → external/beacon_client.go

@@ -1,4 +1,4 @@
-package syncer
+package external
 
 import (
 	"context"

syncer/beacon_client_test.go → external/beacon_client_test.go

@@ -1,4 +1,4 @@
-package syncer
+package external
 
 import (
 	"context"