From 19a890fca53e4d27333e5adab7abf828c0cb69b0 Mon Sep 17 00:00:00 2001
From: dakimura <34202807+dakimura@users.noreply.github.com>
Date: Wed, 30 Mar 2022 07:53:47 +0900
Subject: [PATCH] feat(alpacabkfeeder): retrieve tradable stocks from a remote
 json file (#580)

---
 contrib/alpacabkfeeder/README.md              |  6 +-
 contrib/alpacabkfeeder/alpacav2.go            | 46 +++++----
 contrib/alpacabkfeeder/api/client.go          | 18 ++--
 contrib/alpacabkfeeder/api/entities.go        |  8 +-
 contrib/alpacabkfeeder/api/v1/entities.go     |  2 +-
 contrib/alpacabkfeeder/configs/config.go      |  9 +-
 contrib/alpacabkfeeder/configs/config_test.go | 26 +++--
 contrib/alpacabkfeeder/configs/envconfig.go   | 18 +++-
 contrib/alpacabkfeeder/feed/backfill.go       |  4 +-
 contrib/alpacabkfeeder/feed/backfill_test.go  |  3 +-
 contrib/alpacabkfeeder/feed/worker.go         |  4 +-
 contrib/alpacabkfeeder/internal/mocks.go      |  1 -
 .../symbols/example_json_test.go              | 31 ++++++
 contrib/alpacabkfeeder/symbols/manager.go     |  2 +-
 .../alpacabkfeeder/symbols/manager_file.go    | 98 +++++++++++++++++++
 .../symbols/manager_file_test.go              | 89 +++++++++++++++++
 .../alpacabkfeeder/symbols/manager_test.go    |  5 +-
 contrib/alpacabkfeeder/writer/bar_writer.go   |  3 +-
 .../alpacabkfeeder/writer/bar_writer_test.go  |  1 -
 .../alpacabkfeeder/writer/snapshot_writer.go  |  3 +-
 contrib/xignitefeeder/api/client.go           |  4 +-
 contrib/xignitefeeder/symbols/manager.go      |  2 +-
 contrib/xignitefeeder/xignitefeeder.go        |  2 +-
 23 files changed, 319 insertions(+), 66 deletions(-)
 create mode 100644 contrib/alpacabkfeeder/symbols/example_json_test.go
 create mode 100644 contrib/alpacabkfeeder/symbols/manager_file.go
 create mode 100644 contrib/alpacabkfeeder/symbols/manager_file_test.go

diff --git a/contrib/alpacabkfeeder/README.md b/contrib/alpacabkfeeder/README.md
index d6df702a..3627e481 100644
--- a/contrib/alpacabkfeeder/README.md
+++ b/contrib/alpacabkfeeder/README.md
@@ -23,8 +23,10 @@ bgworkers:
         - NASDAQ
         # - NYSEARCA
         # - OTC
-      # time when target symbols in the exchanges are updated every day.
-      # this time is also used for the historical data back-fill.
+      # time when the list of target symbols (tradable stocks) are updated every day.
+      # This config can be manually overridden by "ALPACA_BROKER_FEEDER_SYMBOLS_UPDATE_TIME" environmental variable.
+      symbols_update_time: "13:00:00" # (UTC). = every day at 08:00:00 (EST)
+      # time when the historical data back-fill is run.
       # This config can be manually overridden by "ALPACA_BROKER_FEEDER_UPDATE_TIME" environmental variable.
       update_time: "13:30:00" # (UTC). = every day at 08:30:00 (EST)
       # Alpava Broker API Feeder writes data to "{symbol}/{timeframe}/TICK" TimeBucketKey
diff --git a/contrib/alpacabkfeeder/alpacav2.go b/contrib/alpacabkfeeder/alpacav2.go
index dc5be150..376bd488 100644
--- a/contrib/alpacabkfeeder/alpacav2.go
+++ b/contrib/alpacabkfeeder/alpacav2.go
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"time"
 
 	"github.com/pkg/errors"
@@ -18,6 +19,8 @@ import (
 	"github.com/alpacahq/marketstore/v4/utils/log"
 )
 
+const getJSONFileTimeout = 10 * time.Second
+
 // NewBgWorker returns the new instance of Alpaca Broker API Feeder.
 // See configs.Config for the details of available configurations.
 // nolint:deadcode // used as a plugin
@@ -28,18 +31,7 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) {
 	}
 	log.Info("loaded Alpaca Broker Feeder config...")
 
-	// init Alpaca API client
-	cred := &api.APIKey{
-		ID:           config.APIKeyID,
-		PolygonKeyID: config.APIKeyID,
-		Secret:       config.APISecretKey,
-		// OAuth:        os.Getenv(EnvApiOAuth),
-	}
-	if config.APIKeyID == "" || config.APISecretKey == "" {
-		// if empty, get from env vars
-		cred = api.Credentials()
-	}
-	apiClient := api.NewClient(cred)
+	apiCli := apiClient(config)
 
 	// init Market Time Checker
 	var timeChecker feed.MarketTimeChecker
@@ -73,12 +65,13 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) {
 	}
 
 	ctx := context.Background()
-	// init Symbols Manager to update symbols in the target exchanges
-
-	sm := symbols.NewManager(apiClient, config.Exchanges)
+	// init symbols Manager to update symbols in the target exchanges
+	sm := symbols.NewJSONFileManager(&http.Client{Timeout: getJSONFileTimeout},
+		config.StocksJSONURL, config.StocksJSONBasicAuth,
+	)
 	sm.UpdateSymbols()
-	timer.RunEveryDayAt(ctx, config.UpdateTime, sm.UpdateSymbols)
-	log.Info("updated symbols in the target exchanges")
+	timer.RunEveryDayAt(ctx, config.SymbolsUpdateTime, sm.UpdateSymbols)
+	log.Info("updated symbols using a remote json file.")
 
 	// init SnapshotWriter
 	var ssw writer.SnapshotWriter = writer.SnapshotWriterImpl{
@@ -97,7 +90,7 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) {
 	if config.Backfill.Enabled {
 		const maxBarsPerRequest = 1000
 		const maxSymbolsPerRequest = 100
-		bf := feed.NewBackfill(sm, apiClient, bw, time.Time(config.Backfill.Since),
+		bf := feed.NewBackfill(sm, apiCli, bw, time.Time(config.Backfill.Since),
 			maxBarsPerRequest, maxSymbolsPerRequest,
 		)
 		timer.RunEveryDayAt(ctx, config.UpdateTime, bf.UpdateSymbols)
@@ -105,7 +98,7 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) {
 
 	return &feed.Worker{
 		MarketTimeChecker: timeChecker,
-		APIClient:         apiClient,
+		APIClient:         apiCli,
 		SymbolManager:     sm,
 		SnapshotWriter:    ssw,
 		BarWriter:         bw,
@@ -113,4 +106,19 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) {
 	}, nil
 }
 
+func apiClient(config *configs.DefaultConfig) *api.Client {
+	// init Alpaca API client
+	cred := &api.APIKey{
+		ID:           config.APIKeyID,
+		PolygonKeyID: config.APIKeyID,
+		Secret:       config.APISecretKey,
+		// OAuth:        os.Getenv(EnvApiOAuth),
+	}
+	if config.APIKeyID == "" || config.APISecretKey == "" {
+		// if empty, get from env vars
+		cred = api.Credentials()
+	}
+	return api.NewClient(cred)
+}
+
 func main() {}
diff --git a/contrib/alpacabkfeeder/api/client.go b/contrib/alpacabkfeeder/api/client.go
index b4b85ad3..be1fbef5 100644
--- a/contrib/alpacabkfeeder/api/client.go
+++ b/contrib/alpacabkfeeder/api/client.go
@@ -22,7 +22,7 @@ const (
 
 var (
 	// DefaultClient is the default Alpaca client using the
-	// environment variable set credentials
+	// environment variable set credentials.
 	DefaultClient = NewClient(Credentials())
 	base          = "https://api.alpaca.markets"
 	dataURL       = "https://data.alpaca.markets"
@@ -35,10 +35,10 @@ func defaultDo(c *Client, req *http.Request) (*http.Response, error) {
 	if c.credentials.OAuth != "" {
 		req.Header.Set("Authorization", "Bearer "+c.credentials.OAuth)
 	} else {
-		if strings.Contains(req.URL.String(), "sandbox"){
+		if strings.Contains(req.URL.String(), "sandbox") {
 			// Add Basic Auth
 			req.SetBasicAuth(c.credentials.ID, c.credentials.Secret)
-		}else {
+		} else {
 			req.Header.Set("APCA-API-KEY-ID", c.credentials.ID)
 			req.Header.Set("APCA-API-SECRET-KEY", c.credentials.Secret)
 		}
@@ -71,7 +71,7 @@ func defaultDo(c *Client, req *http.Request) (*http.Response, error) {
 }
 
 const (
-	// v2MaxLimit is the maximum allowed limit parameter for all v2 endpoints
+	// v2MaxLimit is the maximum allowed limit parameter for all v2 endpoints.
 	v2MaxLimit = 10000
 )
 
@@ -102,7 +102,7 @@ func init() {
 }
 
 // APIError wraps the detailed code and message supplied
-// by Alpaca's API for debugging purposes
+// by Alpaca's API for debugging purposes.
 type APIError struct {
 	Code    int    `json:"code"`
 	Message string `json:"message"`
@@ -112,7 +112,7 @@ func (e *APIError) Error() string {
 	return e.Message
 }
 
-// Client is an Alpaca REST API client
+// Client is an Alpaca REST API client.
 type Client struct {
 	credentials *APIKey
 }
@@ -122,12 +122,12 @@ func SetBaseUrl(baseUrl string) {
 }
 
 // NewClient creates a new Alpaca client with specified
-// credentials
+// credentials.
 func NewClient(credentials *APIKey) *Client {
 	return &Client{credentials: credentials}
 }
 
-// GetSnapshots returns the snapshots for multiple symbol
+// GetSnapshots returns the snapshots for multiple symbol.
 func (c *Client) GetSnapshots(symbols []string) (map[string]*Snapshot, error) {
 	u, err := url.Parse(fmt.Sprintf("%s/v2/stocks/snapshots?symbols=%s",
 		dataURL, strings.Join(symbols, ",")))
@@ -194,7 +194,7 @@ func (c *Client) ListBars(symbols []string, opts v1.ListBarParams) (map[string][
 func (c *Client) ListAssets(status *string) ([]v1.Asset, error) {
 	// TODO: add tests
 	apiVer := apiVersion
-	if strings.Contains(base, "broker"){
+	if strings.Contains(base, "broker") {
 		apiVer = "v1"
 	}
 
diff --git a/contrib/alpacabkfeeder/api/entities.go b/contrib/alpacabkfeeder/api/entities.go
index f119024c..2c54a4ea 100644
--- a/contrib/alpacabkfeeder/api/entities.go
+++ b/contrib/alpacabkfeeder/api/entities.go
@@ -2,7 +2,7 @@ package api
 
 import "time"
 
-// Trade is a stock trade that happened on the market
+// Trade is a stock trade that happened on the market.
 type Trade struct {
 	ID         int64     `json:"i"`
 	Exchange   string    `json:"x"`
@@ -13,7 +13,7 @@ type Trade struct {
 	Tape       string    `json:"z"`
 }
 
-// Quote is a stock quote from the market
+// Quote is a stock quote from the market.
 type Quote struct {
 	BidExchange string    `json:"bx"`
 	BidPrice    float64   `json:"bp"`
@@ -26,7 +26,7 @@ type Quote struct {
 	Tape        string    `json:"z"`
 }
 
-// Bar is an aggregate of trades
+// Bar is an aggregate of trades.
 type Bar struct {
 	Open      float64   `json:"o"`
 	High      float64   `json:"h"`
@@ -36,7 +36,7 @@ type Bar struct {
 	Timestamp time.Time `json:"t"`
 }
 
-// Snapshot is a snapshot of a symbol
+// Snapshot is a snapshot of a symbol.
 type Snapshot struct {
 	LatestTrade  *Trade `json:"latestTrade"`
 	LatestQuote  *Quote `json:"latestQuote"`
diff --git a/contrib/alpacabkfeeder/api/v1/entities.go b/contrib/alpacabkfeeder/api/v1/entities.go
index f662088c..3b82e5a2 100644
--- a/contrib/alpacabkfeeder/api/v1/entities.go
+++ b/contrib/alpacabkfeeder/api/v1/entities.go
@@ -10,7 +10,7 @@ type Asset struct {
 	Symbol       string `json:"symbol"`
 	Status       string `json:"status"`
 	Tradable     bool   `json:"tradable"`
-	Marginable   bool   `json:"marginable"`
+	Marginal   bool   `json:"marginal"`
 	Shortable    bool   `json:"shortable"`
 	EasyToBorrow bool   `json:"easy_to_borrow"`
 }
diff --git a/contrib/alpacabkfeeder/configs/config.go b/contrib/alpacabkfeeder/configs/config.go
index 5c843691..a3aa2c44 100644
--- a/contrib/alpacabkfeeder/configs/config.go
+++ b/contrib/alpacabkfeeder/configs/config.go
@@ -23,7 +23,10 @@ var json = jsoniter.ConfigCompatibleWithStandardLibrary
 // marketstore's config file through bgworker extension.
 type DefaultConfig struct {
 	Exchanges           []Exchange `json:"exchanges"`
+	SymbolsUpdateTime   time.Time  `json:"symbols_update_time"`
 	UpdateTime          time.Time  `json:"update_time"`
+	StocksJSONURL       string     `json:"stocks_json_url"`
+	StocksJSONBasicAuth string     `json:"stocks_json_basic_auth"`
 	Timeframe           string     `json:"timeframe"`
 	APIKeyID            string     `json:"api_key_id"`
 	APISecretKey        string     `json:"api_secret_key"`
@@ -84,6 +87,7 @@ func (c *DefaultConfig) UnmarshalJSON(input []byte) error {
 	type Alias DefaultConfig
 
 	aux := &struct {
+		SymbolsUpdateTime   CustomTime  `json:"symbols_update_time"`
 		UpdateTime          CustomTime  `json:"update_time"`
 		OpenTime            CustomTime  `json:"openTime"`
 		CloseTime           CustomTime  `json:"closeTime"`
@@ -95,6 +99,7 @@ func (c *DefaultConfig) UnmarshalJSON(input []byte) error {
 	if err := json.Unmarshal(input, &aux); err != nil {
 		return err
 	}
+	c.SymbolsUpdateTime = time.Time(aux.SymbolsUpdateTime)
 	c.UpdateTime = time.Time(aux.UpdateTime)
 	c.OpenTime = time.Time(aux.OpenTime)
 	c.CloseTime = time.Time(aux.CloseTime)
@@ -108,7 +113,7 @@ func (c *DefaultConfig) UnmarshalJSON(input []byte) error {
 func convertTime(w []weekday) []time.Weekday {
 	d := make([]time.Weekday, len(w))
 	for i := range w {
-		d = append(d, time.Weekday(w[i]))
+		d[i] = time.Weekday(w[i])
 	}
 	return d
 }
@@ -116,7 +121,7 @@ func convertTime(w []weekday) []time.Weekday {
 func convertDate(cd []CustomDay) []time.Time {
 	d := make([]time.Time, len(cd))
 	for i := range cd {
-		d = append(d, time.Time(cd[i]))
+		d[i] = time.Time(cd[i])
 	}
 	return d
 }
diff --git a/contrib/alpacabkfeeder/configs/config_test.go b/contrib/alpacabkfeeder/configs/config_test.go
index 51267dd0..cc2a1a11 100644
--- a/contrib/alpacabkfeeder/configs/config_test.go
+++ b/contrib/alpacabkfeeder/configs/config_test.go
@@ -10,11 +10,13 @@ import (
 )
 
 var testConfig = map[string]interface{}{
-	"api_key_id":     "hello",
-	"api_secret_key": "world",
-	"update_time":    "12:34:56",
-	"exchanges":      []string{"AMEX", "ARCA", "BATS", "NYSE", "NASDAQ", "NYSEARCA", "OTC"},
-	"index_groups":   []string{"bar"},
+	"api_key_id":             "hello",
+	"api_secret_key":         "world",
+	"symbols_update_time":    "01:23:45",
+	"update_time":            "12:34:56",
+	"stocks_json_basic_auth": "foo:bar",
+	"exchanges":              []string{"AMEX", "ARCA", "BATS", "NYSE", "NASDAQ", "NYSEARCA", "OTC"},
+	"index_groups":           []string{"bar"},
 }
 
 var configWithInvalidExchange = map[string]interface{}{
@@ -30,12 +32,14 @@ func TestNewConfig(t *testing.T) {
 		want    *configs.DefaultConfig
 		wantErr bool
 	}{
-		"ok/ API key ID, API secret key and UpdateTime can be overridden by env vars": {
+		"ok/ API key ID, API secret key, UpdateTime, basicAuth can be overridden by env vars": {
 			config: testConfig,
 			envVars: map[string]string{
-				"ALPACA_BROKER_FEEDER_API_KEY_ID":     "yo",
-				"ALPACA_BROKER_FEEDER_API_SECRET_KEY": "yoyo",
-				"ALPACA_BROKER_FEEDER_UPDATE_TIME":    "20:00:00",
+				"ALPACA_BROKER_FEEDER_API_KEY_ID":             "yo",
+				"ALPACA_BROKER_FEEDER_API_SECRET_KEY":         "yoyo",
+				"ALPACA_BROKER_FEEDER_SYMBOLS_UPDATE_TIME":    "10:00:00",
+				"ALPACA_BROKER_FEEDER_UPDATE_TIME":            "20:00:00",
+				"ALPACA_BROKER_FEEDER_STOCKS_JSON_BASIC_AUTH": "akkie:mypassword",
 			},
 			want: &configs.DefaultConfig{
 				Exchanges: []configs.Exchange{
@@ -44,9 +48,11 @@ func TestNewConfig(t *testing.T) {
 				},
 				ClosedDaysOfTheWeek: []time.Weekday{},
 				ClosedDays:          []time.Time{},
+				SymbolsUpdateTime:   time.Date(0, 1, 1, 10, 0, 0, 0, time.UTC),
 				UpdateTime:          time.Date(0, 1, 1, 20, 0, 0, 0, time.UTC),
 				APIKeyID:            "yo",
 				APISecretKey:        "yoyo",
+				StocksJSONBasicAuth: "akkie:mypassword",
 			},
 			wantErr: false,
 		},
@@ -60,9 +66,11 @@ func TestNewConfig(t *testing.T) {
 				},
 				ClosedDaysOfTheWeek: []time.Weekday{},
 				ClosedDays:          []time.Time{},
+				SymbolsUpdateTime:   time.Date(0, 1, 1, 1, 23, 45, 0, time.UTC),
 				UpdateTime:          time.Date(0, 1, 1, 12, 34, 56, 0, time.UTC),
 				APIKeyID:            "hello",
 				APISecretKey:        "world",
+				StocksJSONBasicAuth: "foo:bar",
 			},
 			wantErr: false,
 		},
diff --git a/contrib/alpacabkfeeder/configs/envconfig.go b/contrib/alpacabkfeeder/configs/envconfig.go
index 1262fb3e..81f57098 100644
--- a/contrib/alpacabkfeeder/configs/envconfig.go
+++ b/contrib/alpacabkfeeder/configs/envconfig.go
@@ -5,12 +5,22 @@ import (
 	"time"
 )
 
-// APItoken and UpdateTime settings can be overridden by environment variables
+// APItoken, UpdateTime, and Basic Auth of stocksJsonURL settings can be overridden by environment variables
 // to flexibly re-run processes that are performed only at marketstore start-up/certain times of the day
 // and not to write security-related configs directly in the configuration file.
 
 // envOverride updates some configs by environment variables.
 func envOverride(config *DefaultConfig) (*DefaultConfig, error) {
+	// override SymbolsUpdateTime
+	symbolsUpdateTime := os.Getenv("ALPACA_BROKER_FEEDER_SYMBOLS_UPDATE_TIME")
+	if symbolsUpdateTime != "" {
+		t, err := time.Parse(ctLayout, symbolsUpdateTime)
+		if err != nil {
+			return nil, err
+		}
+		config.SymbolsUpdateTime = t
+	}
+
 	// override UpdateTime
 	updateTime := os.Getenv("ALPACA_BROKER_FEEDER_UPDATE_TIME")
 	if updateTime != "" {
@@ -32,5 +42,11 @@ func envOverride(config *DefaultConfig) (*DefaultConfig, error) {
 		config.APISecretKey = apiSecretKey
 	}
 
+	// override the basic Auth of Stocks Json URL
+	stocksJSONBasicAuth := os.Getenv("ALPACA_BROKER_FEEDER_STOCKS_JSON_BASIC_AUTH")
+	if stocksJSONBasicAuth != "" {
+		config.StocksJSONBasicAuth = stocksJSONBasicAuth
+	}
+
 	return config, nil
 }
diff --git a/contrib/alpacabkfeeder/feed/backfill.go b/contrib/alpacabkfeeder/feed/backfill.go
index a4b87c32..7a294754 100644
--- a/contrib/alpacabkfeeder/feed/backfill.go
+++ b/contrib/alpacabkfeeder/feed/backfill.go
@@ -1,9 +1,9 @@
 package feed
 
 import (
-	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 	"time"
 
+	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/symbols"
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/writer"
 	"github.com/alpacahq/marketstore/v4/utils/log"
@@ -61,7 +61,7 @@ func (b *Backfill) UpdateSymbols() {
 				log.Error("Alpaca Broker ListBars API call error. Err=%v", err)
 				return
 			}
-			log.Info("Alpaca ListBars API call: From=%v, To=%v, Symbols=%v",
+			log.Info("Alpaca ListBars API call: From=%v, To=%v, symbols=%v",
 				dateRange.From, dateRange.To, allSymbols[idx.From:idx.To],
 			)
 
diff --git a/contrib/alpacabkfeeder/feed/backfill_test.go b/contrib/alpacabkfeeder/feed/backfill_test.go
index bc9e2f9b..f828510a 100644
--- a/contrib/alpacabkfeeder/feed/backfill_test.go
+++ b/contrib/alpacabkfeeder/feed/backfill_test.go
@@ -7,11 +7,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 
+	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/feed"
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/internal"
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/writer"
-
-	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 )
 
 var (
diff --git a/contrib/alpacabkfeeder/feed/worker.go b/contrib/alpacabkfeeder/feed/worker.go
index 8bf4e623..c07a763b 100644
--- a/contrib/alpacabkfeeder/feed/worker.go
+++ b/contrib/alpacabkfeeder/feed/worker.go
@@ -62,7 +62,9 @@ func (w *Worker) try() error {
 	symbls := w.SymbolManager.GetAllSymbols()
 	snapshots, err := w.APIClient.GetSnapshots(symbls)
 	if err != nil {
-		return errors.Wrap(err, fmt.Sprintf("failed to get snapshot from Alpaca API. %v", symbls))
+		return fmt.Errorf("failed to get snapshot from Alpaca API. len(symbols)=%v: %w",
+			len(symbls), err,
+		)
 	}
 
 	// write SnapShot data
diff --git a/contrib/alpacabkfeeder/internal/mocks.go b/contrib/alpacabkfeeder/internal/mocks.go
index a834885b..9047118d 100644
--- a/contrib/alpacabkfeeder/internal/mocks.go
+++ b/contrib/alpacabkfeeder/internal/mocks.go
@@ -5,7 +5,6 @@ import (
 
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api"
 	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
-
 	"github.com/alpacahq/marketstore/v4/utils/io"
 )
 
diff --git a/contrib/alpacabkfeeder/symbols/example_json_test.go b/contrib/alpacabkfeeder/symbols/example_json_test.go
new file mode 100644
index 00000000..b13e660c
--- /dev/null
+++ b/contrib/alpacabkfeeder/symbols/example_json_test.go
@@ -0,0 +1,31 @@
+package symbols_test
+
+// AAPL, ACN, ADBE. restriction info should be ignored.
+const mockTradableStocksJSON = `
+{
+  "update_datetime": "2022-03-28T08:30:29.222960Z",
+  "data": {
+    "AAPL": {
+      "restriction": [
+        {
+          "from": "2022-03-17T11:48:31.246Z",
+          "to": "9999-12-31T14:59:59Z",
+          "stock_order_side": "",
+          "restriction_reason": "test",
+          "unrestriction_reason": "na"
+        }
+      ]
+    },
+    "ACN": {
+      "restriction": []
+    },
+    "ADBE": {
+      "restriction": []
+    }
+  }
+}
+`
+
+const unexpectedJSON = `
+{"abc": ["d", "e","f"]}aaaa
+`
diff --git a/contrib/alpacabkfeeder/symbols/manager.go b/contrib/alpacabkfeeder/symbols/manager.go
index 290813c7..30669389 100644
--- a/contrib/alpacabkfeeder/symbols/manager.go
+++ b/contrib/alpacabkfeeder/symbols/manager.go
@@ -2,8 +2,8 @@ package symbols
 
 import (
 	"fmt"
-	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 
+	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/configs"
 	"github.com/alpacahq/marketstore/v4/utils/log"
 )
diff --git a/contrib/alpacabkfeeder/symbols/manager_file.go b/contrib/alpacabkfeeder/symbols/manager_file.go
new file mode 100644
index 00000000..5e4fbca0
--- /dev/null
+++ b/contrib/alpacabkfeeder/symbols/manager_file.go
@@ -0,0 +1,98 @@
+package symbols
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/alpacahq/marketstore/v4/utils/log"
+)
+
+// JSONFileManager is a symbol manager that reads a json file to get the list of symbols.
+type JSONFileManager struct {
+	httpClient          *http.Client
+	stocksJSONURL       string
+	stocksJSONBasicAuth string
+	symbols             []string
+}
+
+// NewJSONFileManager initializes the SymbolManager object with the specified parameters.
+func NewJSONFileManager(hc *http.Client, stocksJSONURL, stocksJSONBasicAuth string) *JSONFileManager {
+	return &JSONFileManager{
+		httpClient:          hc,
+		stocksJSONURL:       stocksJSONURL,
+		stocksJSONBasicAuth: stocksJSONBasicAuth,
+		symbols:             []string{},
+	}
+}
+
+// GetAllSymbols returns symbols for all the target exchanges.
+func (m *JSONFileManager) GetAllSymbols() []string {
+	return m.symbols
+}
+
+// UpdateSymbols gets a remote json file, store the symbols in the file to the symbols map.
+func (m *JSONFileManager) UpdateSymbols() {
+	if symbols := m.downloadSymbols(context.Background()); symbols != nil {
+		// replace target symbols
+		m.symbols = symbols
+		log.Debug(fmt.Sprintf("Updated symbols. The number of symbols is %d", len(m.symbols)))
+	}
+}
+
+type Stocks struct {
+	Data map[string]interface{} `json:"data"`
+}
+
+func (m *JSONFileManager) downloadSymbols(ctx context.Context) []string {
+	req, err := http.NewRequestWithContext(ctx, "GET", m.stocksJSONURL, nil)
+	if err != nil {
+		log.Error(fmt.Sprintf("failed to create a http req for stocks Json(URL=%s). err=%v",
+			m.stocksJSONURL, err,
+		))
+		return nil
+	}
+	// set basic auth
+	if m.stocksJSONBasicAuth != "" {
+		req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(m.stocksJSONBasicAuth)))
+	}
+
+	resp, err := m.httpClient.Do(req)
+	if err != nil {
+		log.Error("failed to download stocks Json(URL=%s). err=%v",
+			m.stocksJSONURL, err,
+		)
+		return nil
+	}
+	defer func(Body io.ReadCloser) { _ = Body.Close() }(resp.Body)
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Error("failed to read body(URL=%s). err=%v",
+			m.stocksJSONURL, err,
+		)
+		return nil
+	}
+
+	var s Stocks
+	err = json.Unmarshal(b, &s)
+	if err != nil {
+		log.Error("failed to unmarshal json(URL=%s). err=%v",
+			m.stocksJSONURL, err,
+		)
+		return nil
+	}
+
+	symbols := make([]string, len(s.Data))
+	i := 0
+	for symbol := range s.Data {
+		symbols[i] = symbol
+		i++
+	}
+	log.Info(fmt.Sprintf("downloaded a json file(URL=%s), len(symbols)=%d", m.stocksJSONURL, len(symbols)))
+	return symbols
+}
diff --git a/contrib/alpacabkfeeder/symbols/manager_file_test.go b/contrib/alpacabkfeeder/symbols/manager_file_test.go
new file mode 100644
index 00000000..a8d64589
--- /dev/null
+++ b/contrib/alpacabkfeeder/symbols/manager_file_test.go
@@ -0,0 +1,89 @@
+package symbols_test
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/symbols"
+)
+
+type RoundTripFunc func(req *http.Request) *http.Response
+
+func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req), nil
+}
+
+func NewTestClient(fn RoundTripFunc) *http.Client {
+	return &http.Client{
+		Transport: fn,
+	}
+}
+
+func TestJsonFileManager_UpdateSymbols(t *testing.T) {
+	t.Parallel()
+
+	tests := map[string]struct {
+		mockResponse *http.Response
+		wantSymbols  []string
+	}{
+		"OK/tradable stocks are retrieved from the json file": {
+			mockResponse: &http.Response{
+				StatusCode: http.StatusOK,
+				Body:       io.NopCloser(bytes.NewBuffer([]byte(mockTradableStocksJSON))),
+				Header:     make(http.Header),
+			},
+			wantSymbols: []string{"AAPL", "ACN", "ADBE"},
+		},
+		"NG/json file is not found": {
+			mockResponse: &http.Response{
+				StatusCode: http.StatusNotFound,
+				Body:       nil,
+				Header:     make(http.Header),
+			},
+			wantSymbols: []string{},
+		},
+		"NG/json file has an unexpected format": {
+			mockResponse: &http.Response{
+				StatusCode: http.StatusOK,
+				Body:       io.NopCloser(bytes.NewBuffer([]byte(unexpectedJSON))),
+				Header:     make(http.Header),
+			},
+			wantSymbols: []string{},
+		},
+		"NG/unauthorized": {
+			mockResponse: &http.Response{
+				StatusCode: http.StatusUnauthorized,
+				Body:       nil,
+				Header:     make(http.Header),
+			},
+			wantSymbols: []string{},
+		},
+	}
+	for name, tt := range tests {
+		tt := tt
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			// --- given ---
+			httpClient := NewTestClient(func(req *http.Request) *http.Response { return tt.mockResponse })
+			m := symbols.NewJSONFileManager(httpClient, "test", "user:pass")
+
+			// --- when ---
+			m.UpdateSymbols()
+
+			// --- then ---
+			require.Equal(t, sortStrSlice(tt.wantSymbols), sortStrSlice(m.GetAllSymbols()))
+		})
+	}
+}
+
+func sortStrSlice(s []string) []string {
+	sort.SliceStable(s, func(i, j int) bool {
+		return s[i] < s[j]
+	})
+	return s
+}
diff --git a/contrib/alpacabkfeeder/symbols/manager_test.go b/contrib/alpacabkfeeder/symbols/manager_test.go
index 7c4881d3..084e0284 100644
--- a/contrib/alpacabkfeeder/symbols/manager_test.go
+++ b/contrib/alpacabkfeeder/symbols/manager_test.go
@@ -4,10 +4,9 @@ import (
 	"reflect"
 	"testing"
 
+	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/configs"
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/internal"
-
-	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 )
 
 type MockListAssetsAPIClient struct {
@@ -54,6 +53,6 @@ func TestManagerImpl_UpdateSymbols(t *testing.T) {
 		SUT.Symbols,
 		expectedSymbols,
 	) {
-		t.Errorf("Symbols: want=%v, got=%v", expectedSymbols, SUT.Symbols)
+		t.Errorf("symbols: want=%v, got=%v", expectedSymbols, SUT.Symbols)
 	}
 }
diff --git a/contrib/alpacabkfeeder/writer/bar_writer.go b/contrib/alpacabkfeeder/writer/bar_writer.go
index 1592a112..6396379f 100644
--- a/contrib/alpacabkfeeder/writer/bar_writer.go
+++ b/contrib/alpacabkfeeder/writer/bar_writer.go
@@ -6,10 +6,9 @@ import (
 
 	"github.com/pkg/errors"
 
+	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 	"github.com/alpacahq/marketstore/v4/utils/io"
 	"github.com/alpacahq/marketstore/v4/utils/log"
-
-	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
 )
 
 // BarWriter is an interface to write chart data to the marketstore.
diff --git a/contrib/alpacabkfeeder/writer/bar_writer_test.go b/contrib/alpacabkfeeder/writer/bar_writer_test.go
index 42257b6a..51bfa03e 100644
--- a/contrib/alpacabkfeeder/writer/bar_writer_test.go
+++ b/contrib/alpacabkfeeder/writer/bar_writer_test.go
@@ -5,7 +5,6 @@ import (
 	"time"
 
 	v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1"
-
 	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/internal"
 	"github.com/alpacahq/marketstore/v4/utils/io"
 )
diff --git a/contrib/alpacabkfeeder/writer/snapshot_writer.go b/contrib/alpacabkfeeder/writer/snapshot_writer.go
index 10481638..02528c0d 100644
--- a/contrib/alpacabkfeeder/writer/snapshot_writer.go
+++ b/contrib/alpacabkfeeder/writer/snapshot_writer.go
@@ -4,10 +4,9 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api"
 	"github.com/alpacahq/marketstore/v4/utils/io"
 	"github.com/alpacahq/marketstore/v4/utils/log"
-
-	"github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api"
 )
 
 // SnapshotWriter is an interface to write the realtime stock data to the marketstore.
diff --git a/contrib/xignitefeeder/api/client.go b/contrib/xignitefeeder/api/client.go
index 35e76d6f..f546ff6c 100644
--- a/contrib/xignitefeeder/api/client.go
+++ b/contrib/xignitefeeder/api/client.go
@@ -22,10 +22,10 @@ const (
 	// GetQuotesURL is the URL of Get Quotes endpoint
 	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/GetQuotes)
 	GetQuotesURL = XigniteBaseURL + "/QUICKEquityRealTime.json/GetQuotes"
-	// ListSymbolsURL is the URL of List Symbols endpoint
+	// ListSymbolsURL is the URL of List symbols endpoint
 	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/ListSymbols)
 	ListSymbolsURL = XigniteBaseURL + "/QUICKEquityRealTime.json/ListSymbols"
-	// ListIndexSymbolsURL is the URL of List Symbols endpoint
+	// ListIndexSymbolsURL is the URL of List symbols endpoint
 	// (https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexHistorical/Overview/ListSymbols)
 	// /QUICKEquityRealTime.json/ListSymbols : list symbols for a exchange
 	// /QUICKIndexHistorical.json/ListSymbols : list index symbols for an index group (ex. TOPIX).
diff --git a/contrib/xignitefeeder/symbols/manager.go b/contrib/xignitefeeder/symbols/manager.go
index e3e16e0e..ef47366c 100644
--- a/contrib/xignitefeeder/symbols/manager.go
+++ b/contrib/xignitefeeder/symbols/manager.go
@@ -69,7 +69,7 @@ func (m *ManagerImpl) UpdateSymbols(ctx context.Context) {
 
 		// if ListSymbols API returns an error, don't update the target symbols
 		if err != nil || resp.Outcome != "Success" {
-			log.Error(fmt.Sprintf("err=%v, List Symbols API response=%v", err, resp))
+			log.Error(fmt.Sprintf("err=%v, List symbols API response=%v", err, resp))
 			return
 		}
 
diff --git a/contrib/xignitefeeder/xignitefeeder.go b/contrib/xignitefeeder/xignitefeeder.go
index 50d5060e..972b3048 100644
--- a/contrib/xignitefeeder/xignitefeeder.go
+++ b/contrib/xignitefeeder/xignitefeeder.go
@@ -63,7 +63,7 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) {
 	}
 
 	ctx := context.Background()
-	// init Symbols Manager to...
+	// init symbols Manager to...
 	// 1. update symbols in the target exchanges
 	// 2. update index symbols in the target index groups
 	// every day