From 007586568aa92a1c83189b3c06843dc27a23e5b3 Mon Sep 17 00:00:00 2001 From: dakimura <34202807+dakimura@users.noreply.github.com> Date: Mon, 7 Nov 2022 10:51:10 +0900 Subject: [PATCH] feat(xignitefeeder): migrate API endpoints from QUICK to MINKABU (#622) * feat(xignitefeeder): migrate API endpoints from QUICK to MINKABU --- contrib/xignitefeeder/README.md | 13 +++- contrib/xignitefeeder/api/client.go | 68 ++++++++----------- contrib/xignitefeeder/configs/config.go | 42 ++++++++++++ contrib/xignitefeeder/configs/config_test.go | 22 ++++++ contrib/xignitefeeder/symbols/manager.go | 6 ++ contrib/xignitefeeder/symbols/manager_test.go | 1 + contrib/xignitefeeder/xignitefeeder.go | 10 ++- 7 files changed, 121 insertions(+), 41 deletions(-) diff --git a/contrib/xignitefeeder/README.md b/contrib/xignitefeeder/README.md index ff408e48..406d9478 100644 --- a/contrib/xignitefeeder/README.md +++ b/contrib/xignitefeeder/README.md @@ -17,11 +17,9 @@ bgworkers: # exchange list exchanges: - XTKS # Tokyo Stock Exchange - - XJAS # Jasdaq #- XNGO # Nagoya Stock Exchange #- XSAP # Sapporo Stock Exchange #- XFKA # Fukuoka Stock Exchange - #- XTAM # Tokyo PRO Market # Xignite feeder also retrieves data of Index Symbols (ex. ＴＯＰＩＸ（東証１部株価指数）) every day. # To get target indices, index groups that the indices belong are necessary. # (cf. https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexHistorical/Overview/ListSymbols ) @@ -42,6 +40,17 @@ bgworkers: update_time: "22:00:00" # (UTC). = every day at 07:00:00 (JST) # XigniteFeeder writes data to "{identifier}/{timeframe}/TICK" TimeBucketKey timeframe: "1Sec" + # Base URL of the API + base_url: "https://stg-api.mk-smapi.jp/" + # Endpoint of each Xigntie API + endpoint: + equity_realtime_get_quotes: "MINKABUEquityRealTime.json/GetQuotes" + equity_realtime_list_symbols: "MINKABUEquityRealTime.json/ListSymbols" + equity_realtime_get_bars: "MINKABUEquityRealTime.json/GetBars" + equity_historical_get_quotes_range: "MINKABUEquityHistorical.json/GetQuotesRange" + index_realtime_get_bars: "MINKABUIndexRealTime.json/GetBars" + index_historical_list_symbols: "MINKABUIndexHistorical.json/ListSymbols" + index_historical_get_quotes_range: "MINKABUIndexHistorical.json/GetQuotesRange" # Auth token for Xignite API # This config can be manually overridden by "XIGNITE_FEEDER_API_TOKEN" environmental variable. token: "D***0" diff --git a/contrib/xignitefeeder/api/client.go b/contrib/xignitefeeder/api/client.go index c2571483..58d59ca3 100644 --- a/contrib/xignitefeeder/api/client.go +++ b/contrib/xignitefeeder/api/client.go @@ -15,36 +15,17 @@ import ( "github.com/alpacahq/marketstore/v4/utils/log" ) -const ( - // XigniteBaseURL is a Base URL for Quick Xignite API - // (https://www.marketdata-cloud.quick-co.jp/Products/) - XigniteBaseURL = "https://api.marketdata-cloud.quick-co.jp" - // GetQuotesURL is the URL of Get Quotes endpoint - // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/GetQuotes) - GetQuotesURL = XigniteBaseURL + "/QUICKEquityRealTime.json/GetQuotes" - // ListSymbolsURL is the URL of List symbols endpoint - // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/ListSymbols) - ListSymbolsURL = XigniteBaseURL + "/QUICKEquityRealTime.json/ListSymbols" - // ListIndexSymbolsURL is the URL of List symbols endpoint - // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexHistorical/Overview/ListSymbols) - // /QUICKEquityRealTime.json/ListSymbols : list symbols for a exchange - // /QUICKIndexHistorical.json/ListSymbols : list index symbols for an index group (ex. TOPIX). - ListIndexSymbolsURL = XigniteBaseURL + "/QUICKIndexHistorical.json/ListSymbols" - // GetBarsURL is the URL of Get Bars endpoint - // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/GetBars) - GetBarsURL = XigniteBaseURL + "/QUICKEquityRealTime.json/GetBars" - // GetIndexBarsURL is the URL of QuickIndexRealTime/GetBars endpoint - // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexRealTime/Overview/GetBars) - GetIndexBarsURL = XigniteBaseURL + "/QUICKIndexRealTime.json/GetBars" - // GetQuotesRangeURL is the URL of Get Quotes Range endpoint - // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityHistorical/Overview/GetQuotesRange) - GetQuotesRangeURL = XigniteBaseURL + "/QUICKEquityHistorical.json/GetQuotesRange" - // GetIndexQuotesRangeURL is the URL of Get Index Quotes Range endpoint - // (https://www.marketdata-cloud.quick-co.jp/Products/QUICKIndexHistorical/Overview/GetQuotesRange) - GetIndexQuotesRangeURL = XigniteBaseURL + "/QUICKIndexHistorical.json/GetQuotesRange" - - SuccessOutcome = "Success" -) +const SuccessOutcome = "Success" + +type Endpoints struct { + EquityRealTimeGetQuotes string + EquityRealTimeListSymbols string + EquityRealTimeGetBars string + EquityHistoricalGetQuotesRange string + IndexRealTimeGetBars string + IndexHistoricalListSymbols string + IndexHistoricalGetQuotesRange string +} // Client calls an endpoint and returns the parsed response. type Client interface { @@ -62,10 +43,12 @@ type Client interface { } // NewDefaultAPIClient initializes Xignite API client with the specified API token and HTTP timeout[sec]. -func NewDefaultAPIClient(token string, timeoutSec int) *DefaultClient { +func NewDefaultAPIClient(token string, timeoutSec int, baseURL string, endpoints Endpoints) *DefaultClient { return &DefaultClient{ httpClient: &http.Client{Timeout: time.Duration(timeoutSec) * time.Second}, token: token, + baseURL: baseURL, + endpoints: endpoints, } } @@ -73,6 +56,8 @@ func NewDefaultAPIClient(token string, timeoutSec int) *DefaultClient { type DefaultClient struct { httpClient *http.Client token string + baseURL string + endpoints Endpoints } // GetRealTimeQuotes calls GetQuotes endpoint of Xignite API with specified identifiers @@ -86,7 +71,8 @@ func (c *DefaultClient) GetRealTimeQuotes(ctx context.Context, identifiers []str "Identifiers": {strings.Join(identifiers, ",")}, } req, err := http.NewRequestWithContext(ctx, - "POST", GetQuotesURL, strings.NewReader(form.Encode())) + "POST", fmt.Sprintf("%s%s", c.baseURL, c.endpoints.EquityRealTimeGetQuotes), + strings.NewReader(form.Encode())) if err != nil { return response, errors.Wrap(err, "failed to create an http request") } @@ -121,7 +107,8 @@ func (c *DefaultClient) GetRealTimeQuotes(ctx context.Context, identifiers []str // https://www.marketdata-cloud.quick-co.jp/Products/QUICKEquityRealTime/Overview/ListSymbols // exchange: XTKS, XNGO, XSAP, XFKA, XJAS, XTAM func (c *DefaultClient) ListSymbols(ctx context.Context, exchange string) (response ListSymbolsResponse, err error) { - apiURL := ListSymbolsURL + fmt.Sprintf("?_token=%s&Exchange=%s", c.token, exchange) + apiURL := fmt.Sprintf("%s%s?_token=%s&Exchange=%s", + c.baseURL, c.endpoints.EquityRealTimeListSymbols, c.token, exchange) req, err := http.NewRequestWithContext(ctx, "GET", apiURL, http.NoBody) if err != nil { return response, errors.Wrap(err, "failed to create an http request") @@ -145,7 +132,8 @@ func (c *DefaultClient) ListSymbols(ctx context.Context, exchange string) (respo // indexGroup: INDXJPX, IND_NIKKEI. func (c *DefaultClient) ListIndexSymbols(ctx context.Context, indexGroup string, ) (response ListIndexSymbolsResponse, err error) { - apiURL := ListIndexSymbolsURL + fmt.Sprintf("?_token=%s&GroupName=%s", c.token, indexGroup) + apiURL := fmt.Sprintf("%s%s?_token=%s&GroupName=%s", + c.baseURL, c.endpoints.IndexHistoricalListSymbols, c.token, indexGroup) req, err := http.NewRequestWithContext(ctx, "GET", apiURL, http.NoBody) if err != nil { return response, errors.Wrap(err, "failed to create an http request") @@ -182,7 +170,8 @@ func getBarsURLValues(token, identifier string, start, end time.Time) url.Values func (c *DefaultClient) GetRealTimeBars(ctx context.Context, identifier string, start, end time.Time, ) (response GetBarsResponse, err error) { form := getBarsURLValues(c.token, identifier, start, end) - req, err := getBarsRequest(ctx, GetBarsURL, form.Encode()) + req, err := getBarsRequest(ctx, fmt.Sprintf("%s%s", c.baseURL, c.endpoints.EquityRealTimeGetBars), + form.Encode()) if err != nil { return response, err } @@ -202,7 +191,8 @@ func (c *DefaultClient) GetRealTimeBars(ctx context.Context, identifier string, func (c *DefaultClient) GetIndexBars(ctx context.Context, identifier string, start, end time.Time, ) (response GetIndexBarsResponse, err error) { form := getBarsURLValues(c.token, identifier, start, end) - req, err := getBarsRequest(ctx, GetIndexBarsURL, form.Encode()) + req, err := getBarsRequest(ctx, fmt.Sprintf("%s%s", c.baseURL, c.endpoints.IndexRealTimeGetBars), + form.Encode()) if err != nil { return response, err } @@ -233,7 +223,8 @@ func getBarsRequest(ctx context.Context, url, body string) (*http.Request, error func (c *DefaultClient) GetQuotesRange(ctx context.Context, identifier string, startDate, endDate time.Time, ) (response GetQuotesRangeResponse, err error) { formValues := quotesRangeFormValues(c.token, identifier, startDate, endDate) - req, err := quotesRangeReq(ctx, GetQuotesRangeURL, formValues.Encode()) + req, err := quotesRangeReq(ctx, fmt.Sprintf("%s%s", c.baseURL, c.endpoints.EquityHistoricalGetQuotesRange), + formValues.Encode()) if err != nil { return response, err } @@ -257,7 +248,8 @@ func (c *DefaultClient) GetQuotesRange(ctx context.Context, identifier string, s func (c *DefaultClient) GetIndexQuotesRange(ctx context.Context, identifier string, startDate, endDate time.Time, ) (response GetIndexQuotesRangeResponse, err error) { formValues := quotesRangeFormValues(c.token, identifier, startDate, endDate) - req, err := quotesRangeReq(ctx, GetIndexQuotesRangeURL, formValues.Encode()) + req, err := quotesRangeReq(ctx, fmt.Sprintf("%s%s", c.baseURL, c.endpoints.IndexHistoricalGetQuotesRange), + formValues.Encode()) if err != nil { return response, err } diff --git a/contrib/xignitefeeder/configs/config.go b/contrib/xignitefeeder/configs/config.go index ab826756..52687f92 100644 --- a/contrib/xignitefeeder/configs/config.go +++ b/contrib/xignitefeeder/configs/config.go @@ -30,6 +30,8 @@ type DefaultConfig struct { Timeframe string `json:"timeframe"` APIToken string `json:"token"` Timeout int `json:"timeout"` + BaseURL string `json:"base_url"` + Endpoint Endpoint `json:"endpoint"` OpenTime time.Time CloseTime time.Time ClosedDaysOfTheWeek []time.Weekday @@ -54,6 +56,16 @@ type DefaultConfig struct { } `json:"recentBackfill"` } +type Endpoint struct { + EquityRealTimeGetQuotes string `json:"equity_realtime_get_quotes"` + EquityRealTimeListSymbols string `json:"equity_realtime_list_symbols"` + EquityRealTimeGetBars string `json:"equity_realtime_get_bars"` + EquityHistoricalGetQuotesRange string `json:"equity_historical_get_quotes_range"` + IndexRealTimeGetBars string `json:"index_realtime_get_bars"` + IndexHistoricalListSymbols string `json:"index_historical_list_symbols"` + IndexHistoricalGetQuotesRange string `json:"index_historical_get_quotes_range"` +} + // NewConfig casts a map object to Config struct and returns it through json marshal->unmarshal. func NewConfig(config map[string]interface{}) (*DefaultConfig, error) { data, err := json.Marshal(config) @@ -74,10 +86,40 @@ func NewConfig(config map[string]interface{}) (*DefaultConfig, error) { if err := validate(ret); err != nil { return nil, fmt.Errorf("config validation error: %w", err) } + ret.BaseURL, ret.Endpoint = endpointWithDefault(ret.BaseURL, ret.Endpoint) return ret, nil } +func endpointWithDefault(baseURL string, endpoint Endpoint) (string, Endpoint) { + if baseURL == "" { + baseURL = "https://api.marketdata-cloud.quick-co.jp/" + } + if endpoint.EquityRealTimeGetQuotes == "" { + endpoint.EquityRealTimeGetQuotes = "QUICKEquityRealTime.json/GetQuotes" + } + if endpoint.EquityRealTimeListSymbols == "" { + endpoint.EquityRealTimeListSymbols = "QUICKEquityRealTime.json/ListSymbols" + } + if endpoint.EquityRealTimeGetBars == "" { + endpoint.EquityRealTimeGetBars = "QUICKEquityRealTime.json/GetBars" + } + if endpoint.EquityHistoricalGetQuotesRange == "" { + endpoint.EquityHistoricalGetQuotesRange = "QUICKEquityHistorical.json/GetQuotesRange" + } + if endpoint.IndexRealTimeGetBars == "" { + endpoint.IndexRealTimeGetBars = "QUICKIndexRealTime.json/GetBars" + } + if endpoint.IndexHistoricalListSymbols == "" { + endpoint.IndexHistoricalListSymbols = "QUICKIndexHistorical.json/ListSymbols" + } + if endpoint.IndexHistoricalGetQuotesRange == "" { + endpoint.IndexHistoricalGetQuotesRange = "QUICKIndexHistorical.json/GetQuotesRange" + } + + return baseURL, endpoint +} + func validate(cfg *DefaultConfig) error { if len(cfg.Exchanges) < 1 && len(cfg.IndexGroups) < 1 { return errors.New("must have 1 or more stock exchanges or index group in the config file") diff --git a/contrib/xignitefeeder/configs/config_test.go b/contrib/xignitefeeder/configs/config_test.go index 67924a69..6a9dc7eb 100644 --- a/contrib/xignitefeeder/configs/config_test.go +++ b/contrib/xignitefeeder/configs/config_test.go @@ -41,6 +41,17 @@ func TestNewConfig(t *testing.T) { ClosedDays: []time.Time{}, UpdateTime: time.Date(0, 1, 1, 20, 0, 0, 0, time.UTC), APIToken: "ABCDEFGHIJKLMNOPQRSTUVWXYZ789012", + // default urls + BaseURL: "https://api.marketdata-cloud.quick-co.jp/", + Endpoint: configs.Endpoint{ + EquityRealTimeGetQuotes: "QUICKEquityRealTime.json/GetQuotes", + EquityRealTimeListSymbols: "QUICKEquityRealTime.json/ListSymbols", + EquityRealTimeGetBars: "QUICKEquityRealTime.json/GetBars", + EquityHistoricalGetQuotesRange: "QUICKEquityHistorical.json/GetQuotesRange", + IndexRealTimeGetBars: "QUICKIndexRealTime.json/GetBars", + IndexHistoricalListSymbols: "QUICKIndexHistorical.json/ListSymbols", + IndexHistoricalGetQuotesRange: "QUICKIndexHistorical.json/GetQuotesRange", + }, }, wantErr: false, }, @@ -63,6 +74,17 @@ func TestNewConfig(t *testing.T) { ClosedDays: []time.Time{}, UpdateTime: time.Date(0, 1, 1, 12, 34, 56, 0, time.UTC), APIToken: "hellohellohellohellohellohello12", + // default urls + BaseURL: "https://api.marketdata-cloud.quick-co.jp/", + Endpoint: configs.Endpoint{ + EquityRealTimeGetQuotes: "QUICKEquityRealTime.json/GetQuotes", + EquityRealTimeListSymbols: "QUICKEquityRealTime.json/ListSymbols", + EquityRealTimeGetBars: "QUICKEquityRealTime.json/GetBars", + EquityHistoricalGetQuotesRange: "QUICKEquityHistorical.json/GetQuotesRange", + IndexRealTimeGetBars: "QUICKIndexRealTime.json/GetBars", + IndexHistoricalListSymbols: "QUICKIndexHistorical.json/ListSymbols", + IndexHistoricalGetQuotesRange: "QUICKIndexHistorical.json/GetQuotesRange", + }, }, wantErr: false, }, diff --git a/contrib/xignitefeeder/symbols/manager.go b/contrib/xignitefeeder/symbols/manager.go index 7577d95b..238d412b 100644 --- a/contrib/xignitefeeder/symbols/manager.go +++ b/contrib/xignitefeeder/symbols/manager.go @@ -87,6 +87,12 @@ func (m *ManagerImpl) UpdateSymbols(ctx context.Context) { var identifiers []string for _, securityDescription := range resp.ArrayOfSecurityDescription { symbol := securityDescription.Symbol + if len(symbol) >= 5 { + // ignore 5-digit stock code + log.Info(fmt.Sprintf("Ignore 5-digit stock code: %s", symbol)) + continue + } + if _, found := m.NotQuoteStockList[symbol]; found { // ignore symbols in not_quote_stock_list continue diff --git a/contrib/xignitefeeder/symbols/manager_test.go b/contrib/xignitefeeder/symbols/manager_test.go index 908aa7c6..acc7e373 100644 --- a/contrib/xignitefeeder/symbols/manager_test.go +++ b/contrib/xignitefeeder/symbols/manager_test.go @@ -21,6 +21,7 @@ func (mac *MockListSymbolsAPIClient) ListSymbols(_ context.Context, exchange str ArrayOfSecurityDescription: []api.SecurityDescription{ {Symbol: "1234"}, {Symbol: "5678"}, + {Symbol: "90123"}, // 5-digit stock code should be ignored }, }, nil } diff --git a/contrib/xignitefeeder/xignitefeeder.go b/contrib/xignitefeeder/xignitefeeder.go index acd703b9..de5ea30b 100644 --- a/contrib/xignitefeeder/xignitefeeder.go +++ b/contrib/xignitefeeder/xignitefeeder.go @@ -29,7 +29,15 @@ func NewBgWorker(conf map[string]interface{}) (bgworker.BgWorker, error) { log.Info("loaded Xignite Feeder config...") // init Xignite API client - apiClient := api.NewDefaultAPIClient(config.APIToken, config.Timeout) + apiClient := api.NewDefaultAPIClient(config.APIToken, config.Timeout, config.BaseURL, api.Endpoints{ + EquityRealTimeGetQuotes: config.Endpoint.EquityRealTimeGetQuotes, + EquityRealTimeListSymbols: config.Endpoint.EquityRealTimeListSymbols, + EquityRealTimeGetBars: config.Endpoint.EquityRealTimeGetBars, + EquityHistoricalGetQuotesRange: config.Endpoint.EquityHistoricalGetQuotesRange, + IndexRealTimeGetBars: config.Endpoint.IndexRealTimeGetBars, + IndexHistoricalListSymbols: config.Endpoint.IndexHistoricalListSymbols, + IndexHistoricalGetQuotesRange: config.Endpoint.IndexHistoricalGetQuotesRange, + }) // init Market Time Checker var timeChecker feed.MarketTimeChecker