diff --git a/contrib/alpacabkfeeder/README.md b/contrib/alpacabkfeeder/README.md index d6df702a..3627e481 100644 --- a/contrib/alpacabkfeeder/README.md +++ b/contrib/alpacabkfeeder/README.md @@ -23,8 +23,10 @@ bgworkers: - NASDAQ # - NYSEARCA # - OTC - # time when target symbols in the exchanges are updated every day. - # this time is also used for the historical data back-fill. + # time when the list of target symbols (tradable stocks) are updated every day. + # This config can be manually overridden by "ALPACA_BROKER_FEEDER_SYMBOLS_UPDATE_TIME" environmental variable. + symbols_update_time: "13:00:00" # (UTC). = every day at 08:00:00 (EST) + # time when the historical data back-fill is run. # This config can be manually overridden by "ALPACA_BROKER_FEEDER_UPDATE_TIME" environmental variable. update_time: "13:30:00" # (UTC). = every day at 08:30:00 (EST) # Alpava Broker API Feeder writes data to "{symbol}/{timeframe}/TICK" TimeBucketKey diff --git a/contrib/alpacabkfeeder/alpacav2.go b/contrib/alpacabkfeeder/alpacav2.go index dc5be150..376bd488 100644 --- a/contrib/alpacabkfeeder/alpacav2.go +++ b/contrib/alpacabkfeeder/alpacav2.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "net/http" "time" "github.com/pkg/errors" @@ -18,6 +19,8 @@ import ( "github.com/alpacahq/marketstore/v4/utils/log" ) +const getJSONFileTimeout = 10 * time.Second + // NewBgWorker returns the new instance of Alpaca Broker API Feeder. // See configs.Config for the details of available configurations. // nolint:deadcode // used as a plugin @@ -28,18 +31,7 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) { } log.Info("loaded Alpaca Broker Feeder config...") - // init Alpaca API client - cred := &api.APIKey{ - ID: config.APIKeyID, - PolygonKeyID: config.APIKeyID, - Secret: config.APISecretKey, - // OAuth: os.Getenv(EnvApiOAuth), - } - if config.APIKeyID == "" || config.APISecretKey == "" { - // if empty, get from env vars - cred = api.Credentials() - } - apiClient := api.NewClient(cred) + apiCli := apiClient(config) // init Market Time Checker var timeChecker feed.MarketTimeChecker @@ -73,12 +65,13 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) { } ctx := context.Background() - // init Symbols Manager to update symbols in the target exchanges - - sm := symbols.NewManager(apiClient, config.Exchanges) + // init symbols Manager to update symbols in the target exchanges + sm := symbols.NewJSONFileManager(&http.Client{Timeout: getJSONFileTimeout}, + config.StocksJSONURL, config.StocksJSONBasicAuth, + ) sm.UpdateSymbols() - timer.RunEveryDayAt(ctx, config.UpdateTime, sm.UpdateSymbols) - log.Info("updated symbols in the target exchanges") + timer.RunEveryDayAt(ctx, config.SymbolsUpdateTime, sm.UpdateSymbols) + log.Info("updated symbols using a remote json file.") // init SnapshotWriter var ssw writer.SnapshotWriter = writer.SnapshotWriterImpl{ @@ -97,7 +90,7 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) { if config.Backfill.Enabled { const maxBarsPerRequest = 1000 const maxSymbolsPerRequest = 100 - bf := feed.NewBackfill(sm, apiClient, bw, time.Time(config.Backfill.Since), + bf := feed.NewBackfill(sm, apiCli, bw, time.Time(config.Backfill.Since), maxBarsPerRequest, maxSymbolsPerRequest, ) timer.RunEveryDayAt(ctx, config.UpdateTime, bf.UpdateSymbols) @@ -105,7 +98,7 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) { return &feed.Worker{ MarketTimeChecker: timeChecker, - APIClient: apiClient, + APIClient: apiCli, SymbolManager: sm, SnapshotWriter: ssw, BarWriter: bw, @@ -113,4 +106,19 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) { }, nil } +func apiClient(config *configs.DefaultConfig) *api.Client { + // init Alpaca API client + cred := &api.APIKey{ + ID: config.APIKeyID, + PolygonKeyID: config.APIKeyID, + Secret: config.APISecretKey, + // OAuth: os.Getenv(EnvApiOAuth), + } + if config.APIKeyID == "" || config.APISecretKey == "" { + // if empty, get from env vars + cred = api.Credentials() + } + return api.NewClient(cred) +} + func main() {} diff --git a/contrib/alpacabkfeeder/api/client.go b/contrib/alpacabkfeeder/api/client.go index b4b85ad3..be1fbef5 100644 --- a/contrib/alpacabkfeeder/api/client.go +++ b/contrib/alpacabkfeeder/api/client.go @@ -22,7 +22,7 @@ const ( var ( // DefaultClient is the default Alpaca client using the - // environment variable set credentials + // environment variable set credentials. DefaultClient = NewClient(Credentials()) base = "https://api.alpaca.markets" dataURL = "https://data.alpaca.markets" @@ -35,10 +35,10 @@ func defaultDo(c *Client, req *http.Request) (*http.Response, error) { if c.credentials.OAuth != "" { req.Header.Set("Authorization", "Bearer "+c.credentials.OAuth) } else { - if strings.Contains(req.URL.String(), "sandbox"){ + if strings.Contains(req.URL.String(), "sandbox") { // Add Basic Auth req.SetBasicAuth(c.credentials.ID, c.credentials.Secret) - }else { + } else { req.Header.Set("APCA-API-KEY-ID", c.credentials.ID) req.Header.Set("APCA-API-SECRET-KEY", c.credentials.Secret) } @@ -71,7 +71,7 @@ func defaultDo(c *Client, req *http.Request) (*http.Response, error) { } const ( - // v2MaxLimit is the maximum allowed limit parameter for all v2 endpoints + // v2MaxLimit is the maximum allowed limit parameter for all v2 endpoints. v2MaxLimit = 10000 ) @@ -102,7 +102,7 @@ func init() { } // APIError wraps the detailed code and message supplied -// by Alpaca's API for debugging purposes +// by Alpaca's API for debugging purposes. type APIError struct { Code int `json:"code"` Message string `json:"message"` @@ -112,7 +112,7 @@ func (e *APIError) Error() string { return e.Message } -// Client is an Alpaca REST API client +// Client is an Alpaca REST API client. type Client struct { credentials *APIKey } @@ -122,12 +122,12 @@ func SetBaseUrl(baseUrl string) { } // NewClient creates a new Alpaca client with specified -// credentials +// credentials. func NewClient(credentials *APIKey) *Client { return &Client{credentials: credentials} } -// GetSnapshots returns the snapshots for multiple symbol +// GetSnapshots returns the snapshots for multiple symbol. func (c *Client) GetSnapshots(symbols []string) (map[string]*Snapshot, error) { u, err := url.Parse(fmt.Sprintf("%s/v2/stocks/snapshots?symbols=%s", dataURL, strings.Join(symbols, ","))) @@ -194,7 +194,7 @@ func (c *Client) ListBars(symbols []string, opts v1.ListBarParams) (map[string][ func (c *Client) ListAssets(status *string) ([]v1.Asset, error) { // TODO: add tests apiVer := apiVersion - if strings.Contains(base, "broker"){ + if strings.Contains(base, "broker") { apiVer = "v1" } diff --git a/contrib/alpacabkfeeder/api/entities.go b/contrib/alpacabkfeeder/api/entities.go index f119024c..2c54a4ea 100644 --- a/contrib/alpacabkfeeder/api/entities.go +++ b/contrib/alpacabkfeeder/api/entities.go @@ -2,7 +2,7 @@ package api import "time" -// Trade is a stock trade that happened on the market +// Trade is a stock trade that happened on the market. type Trade struct { ID int64 `json:"i"` Exchange string `json:"x"` @@ -13,7 +13,7 @@ type Trade struct { Tape string `json:"z"` } -// Quote is a stock quote from the market +// Quote is a stock quote from the market. type Quote struct { BidExchange string `json:"bx"` BidPrice float64 `json:"bp"` @@ -26,7 +26,7 @@ type Quote struct { Tape string `json:"z"` } -// Bar is an aggregate of trades +// Bar is an aggregate of trades. type Bar struct { Open float64 `json:"o"` High float64 `json:"h"` @@ -36,7 +36,7 @@ type Bar struct { Timestamp time.Time `json:"t"` } -// Snapshot is a snapshot of a symbol +// Snapshot is a snapshot of a symbol. type Snapshot struct { LatestTrade *Trade `json:"latestTrade"` LatestQuote *Quote `json:"latestQuote"` diff --git a/contrib/alpacabkfeeder/api/v1/entities.go b/contrib/alpacabkfeeder/api/v1/entities.go index f662088c..3b82e5a2 100644 --- a/contrib/alpacabkfeeder/api/v1/entities.go +++ b/contrib/alpacabkfeeder/api/v1/entities.go @@ -10,7 +10,7 @@ type Asset struct { Symbol string `json:"symbol"` Status string `json:"status"` Tradable bool `json:"tradable"` - Marginable bool `json:"marginable"` + Marginal bool `json:"marginal"` Shortable bool `json:"shortable"` EasyToBorrow bool `json:"easy_to_borrow"` } diff --git a/contrib/alpacabkfeeder/configs/config.go b/contrib/alpacabkfeeder/configs/config.go index 5c843691..a3aa2c44 100644 --- a/contrib/alpacabkfeeder/configs/config.go +++ b/contrib/alpacabkfeeder/configs/config.go @@ -23,7 +23,10 @@ var json = jsoniter.ConfigCompatibleWithStandardLibrary // marketstore's config file through bgworker extension. type DefaultConfig struct { Exchanges []Exchange `json:"exchanges"` + SymbolsUpdateTime time.Time `json:"symbols_update_time"` UpdateTime time.Time `json:"update_time"` + StocksJSONURL string `json:"stocks_json_url"` + StocksJSONBasicAuth string `json:"stocks_json_basic_auth"` Timeframe string `json:"timeframe"` APIKeyID string `json:"api_key_id"` APISecretKey string `json:"api_secret_key"` @@ -84,6 +87,7 @@ func (c *DefaultConfig) UnmarshalJSON(input []byte) error { type Alias DefaultConfig aux := &struct { + SymbolsUpdateTime CustomTime `json:"symbols_update_time"` UpdateTime CustomTime `json:"update_time"` OpenTime CustomTime `json:"openTime"` CloseTime CustomTime `json:"closeTime"` @@ -95,6 +99,7 @@ func (c *DefaultConfig) UnmarshalJSON(input []byte) error { if err := json.Unmarshal(input, &aux); err != nil { return err } + c.SymbolsUpdateTime = time.Time(aux.SymbolsUpdateTime) c.UpdateTime = time.Time(aux.UpdateTime) c.OpenTime = time.Time(aux.OpenTime) c.CloseTime = time.Time(aux.CloseTime) @@ -108,7 +113,7 @@ func (c *DefaultConfig) UnmarshalJSON(input []byte) error { func convertTime(w []weekday) []time.Weekday { d := make([]time.Weekday, len(w)) for i := range w { - d = append(d, time.Weekday(w[i])) + d[i] = time.Weekday(w[i]) } return d } @@ -116,7 +121,7 @@ func convertTime(w []weekday) []time.Weekday { func convertDate(cd []CustomDay) []time.Time { d := make([]time.Time, len(cd)) for i := range cd { - d = append(d, time.Time(cd[i])) + d[i] = time.Time(cd[i]) } return d } diff --git a/contrib/alpacabkfeeder/configs/config_test.go b/contrib/alpacabkfeeder/configs/config_test.go index 51267dd0..cc2a1a11 100644 --- a/contrib/alpacabkfeeder/configs/config_test.go +++ b/contrib/alpacabkfeeder/configs/config_test.go @@ -10,11 +10,13 @@ import ( ) var testConfig = map[string]interface{}{ - "api_key_id": "hello", - "api_secret_key": "world", - "update_time": "12:34:56", - "exchanges": []string{"AMEX", "ARCA", "BATS", "NYSE", "NASDAQ", "NYSEARCA", "OTC"}, - "index_groups": []string{"bar"}, + "api_key_id": "hello", + "api_secret_key": "world", + "symbols_update_time": "01:23:45", + "update_time": "12:34:56", + "stocks_json_basic_auth": "foo:bar", + "exchanges": []string{"AMEX", "ARCA", "BATS", "NYSE", "NASDAQ", "NYSEARCA", "OTC"}, + "index_groups": []string{"bar"}, } var configWithInvalidExchange = map[string]interface{}{ @@ -30,12 +32,14 @@ func TestNewConfig(t *testing.T) { want *configs.DefaultConfig wantErr bool }{ - "ok/ API key ID, API secret key and UpdateTime can be overridden by env vars": { + "ok/ API key ID, API secret key, UpdateTime, basicAuth can be overridden by env vars": { config: testConfig, envVars: map[string]string{ - "ALPACA_BROKER_FEEDER_API_KEY_ID": "yo", - "ALPACA_BROKER_FEEDER_API_SECRET_KEY": "yoyo", - "ALPACA_BROKER_FEEDER_UPDATE_TIME": "20:00:00", + "ALPACA_BROKER_FEEDER_API_KEY_ID": "yo", + "ALPACA_BROKER_FEEDER_API_SECRET_KEY": "yoyo", + "ALPACA_BROKER_FEEDER_SYMBOLS_UPDATE_TIME": "10:00:00", + "ALPACA_BROKER_FEEDER_UPDATE_TIME": "20:00:00", + "ALPACA_BROKER_FEEDER_STOCKS_JSON_BASIC_AUTH": "akkie:mypassword", }, want: &configs.DefaultConfig{ Exchanges: []configs.Exchange{ @@ -44,9 +48,11 @@ func TestNewConfig(t *testing.T) { }, ClosedDaysOfTheWeek: []time.Weekday{}, ClosedDays: []time.Time{}, + SymbolsUpdateTime: time.Date(0, 1, 1, 10, 0, 0, 0, time.UTC), UpdateTime: time.Date(0, 1, 1, 20, 0, 0, 0, time.UTC), APIKeyID: "yo", APISecretKey: "yoyo", + StocksJSONBasicAuth: "akkie:mypassword", }, wantErr: false, }, @@ -60,9 +66,11 @@ func TestNewConfig(t *testing.T) { }, ClosedDaysOfTheWeek: []time.Weekday{}, ClosedDays: []time.Time{}, + SymbolsUpdateTime: time.Date(0, 1, 1, 1, 23, 45, 0, time.UTC), UpdateTime: time.Date(0, 1, 1, 12, 34, 56, 0, time.UTC), APIKeyID: "hello", APISecretKey: "world", + StocksJSONBasicAuth: "foo:bar", }, wantErr: false, }, diff --git a/contrib/alpacabkfeeder/configs/envconfig.go b/contrib/alpacabkfeeder/configs/envconfig.go index 1262fb3e..81f57098 100644 --- a/contrib/alpacabkfeeder/configs/envconfig.go +++ b/contrib/alpacabkfeeder/configs/envconfig.go @@ -5,12 +5,22 @@ import ( "time" ) -// APItoken and UpdateTime settings can be overridden by environment variables +// APItoken, UpdateTime, and Basic Auth of stocksJsonURL settings can be overridden by environment variables // to flexibly re-run processes that are performed only at marketstore start-up/certain times of the day // and not to write security-related configs directly in the configuration file. // envOverride updates some configs by environment variables. func envOverride(config *DefaultConfig) (*DefaultConfig, error) { + // override SymbolsUpdateTime + symbolsUpdateTime := os.Getenv("ALPACA_BROKER_FEEDER_SYMBOLS_UPDATE_TIME") + if symbolsUpdateTime != "" { + t, err := time.Parse(ctLayout, symbolsUpdateTime) + if err != nil { + return nil, err + } + config.SymbolsUpdateTime = t + } + // override UpdateTime updateTime := os.Getenv("ALPACA_BROKER_FEEDER_UPDATE_TIME") if updateTime != "" { @@ -32,5 +42,11 @@ func envOverride(config *DefaultConfig) (*DefaultConfig, error) { config.APISecretKey = apiSecretKey } + // override the basic Auth of Stocks Json URL + stocksJSONBasicAuth := os.Getenv("ALPACA_BROKER_FEEDER_STOCKS_JSON_BASIC_AUTH") + if stocksJSONBasicAuth != "" { + config.StocksJSONBasicAuth = stocksJSONBasicAuth + } + return config, nil } diff --git a/contrib/alpacabkfeeder/feed/backfill.go b/contrib/alpacabkfeeder/feed/backfill.go index a4b87c32..7a294754 100644 --- a/contrib/alpacabkfeeder/feed/backfill.go +++ b/contrib/alpacabkfeeder/feed/backfill.go @@ -1,9 +1,9 @@ package feed import ( - v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" "time" + v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/symbols" "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/writer" "github.com/alpacahq/marketstore/v4/utils/log" @@ -61,7 +61,7 @@ func (b *Backfill) UpdateSymbols() { log.Error("Alpaca Broker ListBars API call error. Err=%v", err) return } - log.Info("Alpaca ListBars API call: From=%v, To=%v, Symbols=%v", + log.Info("Alpaca ListBars API call: From=%v, To=%v, symbols=%v", dateRange.From, dateRange.To, allSymbols[idx.From:idx.To], ) diff --git a/contrib/alpacabkfeeder/feed/backfill_test.go b/contrib/alpacabkfeeder/feed/backfill_test.go index bc9e2f9b..f828510a 100644 --- a/contrib/alpacabkfeeder/feed/backfill_test.go +++ b/contrib/alpacabkfeeder/feed/backfill_test.go @@ -7,11 +7,10 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" + v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/feed" "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/internal" "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/writer" - - v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" ) var ( diff --git a/contrib/alpacabkfeeder/feed/worker.go b/contrib/alpacabkfeeder/feed/worker.go index 8bf4e623..c07a763b 100644 --- a/contrib/alpacabkfeeder/feed/worker.go +++ b/contrib/alpacabkfeeder/feed/worker.go @@ -62,7 +62,9 @@ func (w *Worker) try() error { symbls := w.SymbolManager.GetAllSymbols() snapshots, err := w.APIClient.GetSnapshots(symbls) if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to get snapshot from Alpaca API. %v", symbls)) + return fmt.Errorf("failed to get snapshot from Alpaca API. len(symbols)=%v: %w", + len(symbls), err, + ) } // write SnapShot data diff --git a/contrib/alpacabkfeeder/internal/mocks.go b/contrib/alpacabkfeeder/internal/mocks.go index a834885b..9047118d 100644 --- a/contrib/alpacabkfeeder/internal/mocks.go +++ b/contrib/alpacabkfeeder/internal/mocks.go @@ -5,7 +5,6 @@ import ( "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api" v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" - "github.com/alpacahq/marketstore/v4/utils/io" ) diff --git a/contrib/alpacabkfeeder/symbols/example_json_test.go b/contrib/alpacabkfeeder/symbols/example_json_test.go new file mode 100644 index 00000000..b13e660c --- /dev/null +++ b/contrib/alpacabkfeeder/symbols/example_json_test.go @@ -0,0 +1,31 @@ +package symbols_test + +// AAPL, ACN, ADBE. restriction info should be ignored. +const mockTradableStocksJSON = ` +{ + "update_datetime": "2022-03-28T08:30:29.222960Z", + "data": { + "AAPL": { + "restriction": [ + { + "from": "2022-03-17T11:48:31.246Z", + "to": "9999-12-31T14:59:59Z", + "stock_order_side": "", + "restriction_reason": "test", + "unrestriction_reason": "na" + } + ] + }, + "ACN": { + "restriction": [] + }, + "ADBE": { + "restriction": [] + } + } +} +` + +const unexpectedJSON = ` +{"abc": ["d", "e","f"]}aaaa +` diff --git a/contrib/alpacabkfeeder/symbols/manager.go b/contrib/alpacabkfeeder/symbols/manager.go index 290813c7..30669389 100644 --- a/contrib/alpacabkfeeder/symbols/manager.go +++ b/contrib/alpacabkfeeder/symbols/manager.go @@ -2,8 +2,8 @@ package symbols import ( "fmt" - v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" + v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/configs" "github.com/alpacahq/marketstore/v4/utils/log" ) diff --git a/contrib/alpacabkfeeder/symbols/manager_file.go b/contrib/alpacabkfeeder/symbols/manager_file.go new file mode 100644 index 00000000..5e4fbca0 --- /dev/null +++ b/contrib/alpacabkfeeder/symbols/manager_file.go @@ -0,0 +1,98 @@ +package symbols + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/alpacahq/marketstore/v4/utils/log" +) + +// JSONFileManager is a symbol manager that reads a json file to get the list of symbols. +type JSONFileManager struct { + httpClient *http.Client + stocksJSONURL string + stocksJSONBasicAuth string + symbols []string +} + +// NewJSONFileManager initializes the SymbolManager object with the specified parameters. +func NewJSONFileManager(hc *http.Client, stocksJSONURL, stocksJSONBasicAuth string) *JSONFileManager { + return &JSONFileManager{ + httpClient: hc, + stocksJSONURL: stocksJSONURL, + stocksJSONBasicAuth: stocksJSONBasicAuth, + symbols: []string{}, + } +} + +// GetAllSymbols returns symbols for all the target exchanges. +func (m *JSONFileManager) GetAllSymbols() []string { + return m.symbols +} + +// UpdateSymbols gets a remote json file, store the symbols in the file to the symbols map. +func (m *JSONFileManager) UpdateSymbols() { + if symbols := m.downloadSymbols(context.Background()); symbols != nil { + // replace target symbols + m.symbols = symbols + log.Debug(fmt.Sprintf("Updated symbols. The number of symbols is %d", len(m.symbols))) + } +} + +type Stocks struct { + Data map[string]interface{} `json:"data"` +} + +func (m *JSONFileManager) downloadSymbols(ctx context.Context) []string { + req, err := http.NewRequestWithContext(ctx, "GET", m.stocksJSONURL, nil) + if err != nil { + log.Error(fmt.Sprintf("failed to create a http req for stocks Json(URL=%s). err=%v", + m.stocksJSONURL, err, + )) + return nil + } + // set basic auth + if m.stocksJSONBasicAuth != "" { + req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(m.stocksJSONBasicAuth))) + } + + resp, err := m.httpClient.Do(req) + if err != nil { + log.Error("failed to download stocks Json(URL=%s). err=%v", + m.stocksJSONURL, err, + ) + return nil + } + defer func(Body io.ReadCloser) { _ = Body.Close() }(resp.Body) + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Error("failed to read body(URL=%s). err=%v", + m.stocksJSONURL, err, + ) + return nil + } + + var s Stocks + err = json.Unmarshal(b, &s) + if err != nil { + log.Error("failed to unmarshal json(URL=%s). err=%v", + m.stocksJSONURL, err, + ) + return nil + } + + symbols := make([]string, len(s.Data)) + i := 0 + for symbol := range s.Data { + symbols[i] = symbol + i++ + } + log.Info(fmt.Sprintf("downloaded a json file(URL=%s), len(symbols)=%d", m.stocksJSONURL, len(symbols))) + return symbols +} diff --git a/contrib/alpacabkfeeder/symbols/manager_file_test.go b/contrib/alpacabkfeeder/symbols/manager_file_test.go new file mode 100644 index 00000000..a8d64589 --- /dev/null +++ b/contrib/alpacabkfeeder/symbols/manager_file_test.go @@ -0,0 +1,89 @@ +package symbols_test + +import ( + "bytes" + "io" + "net/http" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/symbols" +) + +type RoundTripFunc func(req *http.Request) *http.Response + +func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func NewTestClient(fn RoundTripFunc) *http.Client { + return &http.Client{ + Transport: fn, + } +} + +func TestJsonFileManager_UpdateSymbols(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + mockResponse *http.Response + wantSymbols []string + }{ + "OK/tradable stocks are retrieved from the json file": { + mockResponse: &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer([]byte(mockTradableStocksJSON))), + Header: make(http.Header), + }, + wantSymbols: []string{"AAPL", "ACN", "ADBE"}, + }, + "NG/json file is not found": { + mockResponse: &http.Response{ + StatusCode: http.StatusNotFound, + Body: nil, + Header: make(http.Header), + }, + wantSymbols: []string{}, + }, + "NG/json file has an unexpected format": { + mockResponse: &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer([]byte(unexpectedJSON))), + Header: make(http.Header), + }, + wantSymbols: []string{}, + }, + "NG/unauthorized": { + mockResponse: &http.Response{ + StatusCode: http.StatusUnauthorized, + Body: nil, + Header: make(http.Header), + }, + wantSymbols: []string{}, + }, + } + for name, tt := range tests { + tt := tt + t.Run(name, func(t *testing.T) { + t.Parallel() + // --- given --- + httpClient := NewTestClient(func(req *http.Request) *http.Response { return tt.mockResponse }) + m := symbols.NewJSONFileManager(httpClient, "test", "user:pass") + + // --- when --- + m.UpdateSymbols() + + // --- then --- + require.Equal(t, sortStrSlice(tt.wantSymbols), sortStrSlice(m.GetAllSymbols())) + }) + } +} + +func sortStrSlice(s []string) []string { + sort.SliceStable(s, func(i, j int) bool { + return s[i] < s[j] + }) + return s +} diff --git a/contrib/alpacabkfeeder/symbols/manager_test.go b/contrib/alpacabkfeeder/symbols/manager_test.go index 7c4881d3..084e0284 100644 --- a/contrib/alpacabkfeeder/symbols/manager_test.go +++ b/contrib/alpacabkfeeder/symbols/manager_test.go @@ -4,10 +4,9 @@ import ( "reflect" "testing" + v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/configs" "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/internal" - - v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" ) type MockListAssetsAPIClient struct { @@ -54,6 +53,6 @@ func TestManagerImpl_UpdateSymbols(t *testing.T) { SUT.Symbols, expectedSymbols, ) { - t.Errorf("Symbols: want=%v, got=%v", expectedSymbols, SUT.Symbols) + t.Errorf("symbols: want=%v, got=%v", expectedSymbols, SUT.Symbols) } } diff --git a/contrib/alpacabkfeeder/writer/bar_writer.go b/contrib/alpacabkfeeder/writer/bar_writer.go index 1592a112..6396379f 100644 --- a/contrib/alpacabkfeeder/writer/bar_writer.go +++ b/contrib/alpacabkfeeder/writer/bar_writer.go @@ -6,10 +6,9 @@ import ( "github.com/pkg/errors" + v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" "github.com/alpacahq/marketstore/v4/utils/io" "github.com/alpacahq/marketstore/v4/utils/log" - - v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" ) // BarWriter is an interface to write chart data to the marketstore. diff --git a/contrib/alpacabkfeeder/writer/bar_writer_test.go b/contrib/alpacabkfeeder/writer/bar_writer_test.go index 42257b6a..51bfa03e 100644 --- a/contrib/alpacabkfeeder/writer/bar_writer_test.go +++ b/contrib/alpacabkfeeder/writer/bar_writer_test.go @@ -5,7 +5,6 @@ import ( "time" v1 "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api/v1" - "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/internal" "github.com/alpacahq/marketstore/v4/utils/io" ) diff --git a/contrib/alpacabkfeeder/writer/snapshot_writer.go b/contrib/alpacabkfeeder/writer/snapshot_writer.go index 10481638..02528c0d 100644 --- a/contrib/alpacabkfeeder/writer/snapshot_writer.go +++ b/contrib/alpacabkfeeder/writer/snapshot_writer.go @@ -4,10 +4,9 @@ import ( "fmt" "time" + "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api" "github.com/alpacahq/marketstore/v4/utils/io" "github.com/alpacahq/marketstore/v4/utils/log" - - "github.com/alpacahq/marketstore/v4/contrib/alpacabkfeeder/api" ) // SnapshotWriter is an interface to write the realtime stock data to the marketstore. diff --git a/contrib/xignitefeeder/api/client.go b/contrib/xignitefeeder/api/client.go index 35e76d6f..f546ff6c 100644 --- a/contrib/xignitefeeder/api/client.go +++ b/contrib/xignitefeeder/api/client.go @@ -22,10 +22,10 @@ const ( // GetQuotesURL is the URL of Get Quotes endpoint // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/GetQuotes) GetQuotesURL = XigniteBaseURL + "/QUICKEquityRealTime.json/GetQuotes" - // ListSymbolsURL is the URL of List Symbols endpoint + // ListSymbolsURL is the URL of List symbols endpoint // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/ListSymbols) ListSymbolsURL = XigniteBaseURL + "/QUICKEquityRealTime.json/ListSymbols" - // ListIndexSymbolsURL is the URL of List Symbols endpoint + // ListIndexSymbolsURL is the URL of List symbols endpoint // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexHistorical/Overview/ListSymbols) // /QUICKEquityRealTime.json/ListSymbols : list symbols for a exchange // /QUICKIndexHistorical.json/ListSymbols : list index symbols for an index group (ex. TOPIX). diff --git a/contrib/xignitefeeder/symbols/manager.go b/contrib/xignitefeeder/symbols/manager.go index e3e16e0e..ef47366c 100644 --- a/contrib/xignitefeeder/symbols/manager.go +++ b/contrib/xignitefeeder/symbols/manager.go @@ -69,7 +69,7 @@ func (m *ManagerImpl) UpdateSymbols(ctx context.Context) { // if ListSymbols API returns an error, don't update the target symbols if err != nil || resp.Outcome != "Success" { - log.Error(fmt.Sprintf("err=%v, List Symbols API response=%v", err, resp)) + log.Error(fmt.Sprintf("err=%v, List symbols API response=%v", err, resp)) return } diff --git a/contrib/xignitefeeder/xignitefeeder.go b/contrib/xignitefeeder/xignitefeeder.go index 50d5060e..972b3048 100644 --- a/contrib/xignitefeeder/xignitefeeder.go +++ b/contrib/xignitefeeder/xignitefeeder.go @@ -63,7 +63,7 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) { } ctx := context.Background() - // init Symbols Manager to... + // init symbols Manager to... // 1. update symbols in the target exchanges // 2. update index symbols in the target index groups // every day