From 244c6f4c85898ffafff10d98ef28fedba9954711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Comb=C3=BCchen?= Date: Thu, 21 Nov 2024 15:22:12 +0100 Subject: [PATCH] fix: handle Snyk API rate limit Closes #83. --- go.mod | 6 ++-- go.sum | 14 +++++++-- lib/snyk/enrich_cyclonedx.go | 2 +- lib/snyk/enrich_spdx.go | 2 +- lib/snyk/package.go | 39 +++++++++++++++++++++-- lib/snyk/package_test.go | 60 ++++++++++++++++++++++++++++++++++++ lib/snyk/service.go | 2 +- 7 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 lib/snyk/package_test.go diff --git a/go.mod b/go.mod index d723db5..8e6dac1 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/deepmap/oapi-codegen v1.12.4 github.com/edoardottt/depsdev v0.0.3 github.com/google/uuid v1.3.0 + github.com/hashicorp/go-retryablehttp v0.7.7 github.com/jarcoal/httpmock v1.3.0 github.com/package-url/packageurl-go v0.1.2 github.com/remeh/sizedwaitgroup v1.0.0 @@ -23,11 +24,12 @@ require ( github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -36,7 +38,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect - golang.org/x/sys v0.3.0 // indirect + golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.5.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 7bbbf85..c11c6eb 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -140,6 +141,11 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -168,8 +174,9 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -371,8 +378,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/lib/snyk/enrich_cyclonedx.go b/lib/snyk/enrich_cyclonedx.go index faefd64..e46af56 100644 --- a/lib/snyk/enrich_cyclonedx.go +++ b/lib/snyk/enrich_cyclonedx.go @@ -107,7 +107,7 @@ func enrichCycloneDX(cfg *Config, bom *cdx.BOM, logger *zerolog.Logger) *cdx.BOM for _, enrichFunc := range cdxEnrichers { enrichFunc(cfg, component, &purl) } - resp, err := GetPackageVulnerabilities(cfg, &purl, auth, orgID) + resp, err := GetPackageVulnerabilities(cfg, &purl, auth, orgID, logger) if err != nil { l.Err(err). Str("purl", purl.ToString()). diff --git a/lib/snyk/enrich_spdx.go b/lib/snyk/enrich_spdx.go index e295eeb..aa92814 100644 --- a/lib/snyk/enrich_spdx.go +++ b/lib/snyk/enrich_spdx.go @@ -112,7 +112,7 @@ func enrichSPDX(cfg *Config, bom *spdx.Document, logger *zerolog.Logger) *spdx.D for _, enrichFn := range spdxEnrichers { enrichFn(cfg, pkg, purl) } - resp, err := GetPackageVulnerabilities(cfg, purl, auth, orgID) + resp, err := GetPackageVulnerabilities(cfg, purl, auth, orgID, logger) if err != nil { l.Err(err). Str("purl", purl.ToString()). diff --git a/lib/snyk/package.go b/lib/snyk/package.go index f762e44..dd7ea82 100644 --- a/lib/snyk/package.go +++ b/lib/snyk/package.go @@ -20,10 +20,14 @@ import ( "context" "fmt" "net/http" + "strconv" + "time" "github.com/deepmap/oapi-codegen/pkg/securityprovider" "github.com/google/uuid" + "github.com/hashicorp/go-retryablehttp" "github.com/package-url/packageurl-go" + "github.com/rs/zerolog" "github.com/snyk/parlay/snyk/issues" ) @@ -82,8 +86,11 @@ func SnykVulnURL(cfg *Config, purl *packageurl.PackageURL) string { return url } -func GetPackageVulnerabilities(cfg *Config, purl *packageurl.PackageURL, auth *securityprovider.SecurityProviderApiKey, orgID *uuid.UUID) (*issues.FetchIssuesPerPurlResponse, error) { - client, err := issues.NewClientWithResponses(cfg.SnykAPIURL, issues.WithRequestEditorFn(auth.Intercept)) +func GetPackageVulnerabilities(cfg *Config, purl *packageurl.PackageURL, auth *securityprovider.SecurityProviderApiKey, orgID *uuid.UUID, logger *zerolog.Logger) (*issues.FetchIssuesPerPurlResponse, error) { + client, err := issues.NewClientWithResponses( + cfg.SnykAPIURL, + issues.WithRequestEditorFn(auth.Intercept), + issues.WithHTTPClient(getRetryClient(logger))) if err != nil { return nil, err } @@ -100,3 +107,31 @@ func GetPackageVulnerabilities(cfg *Config, purl *packageurl.PackageURL, auth *s return resp, nil } + +func getRetryClient(logger *zerolog.Logger) *http.Client { + rc := retryablehttp.NewClient() + rc.Logger = nil + rc.Backoff = func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { + if sleep, ok := parseRateLimitHeader(resp.Header.Get("X-RateLimit-Reset")); ok { + logger.Warn(). + Dur("Retry-After", sleep). + Msg("Getting rate-limited, waiting...") + return sleep + } + return retryablehttp.DefaultBackoff(min, max, attemptNum, resp) + } + + return rc.StandardClient() +} + +func parseRateLimitHeader(v string) (time.Duration, bool) { + if v == "" { + return 0, false + } + + if sec, err := strconv.ParseInt(v, 10, 64); err == nil { + return time.Duration(sec) * time.Second, true + } + + return 0, false +} diff --git a/lib/snyk/package_test.go b/lib/snyk/package_test.go new file mode 100644 index 0000000..3a7e0e9 --- /dev/null +++ b/lib/snyk/package_test.go @@ -0,0 +1,60 @@ +/* + * © 2023 Snyk Limited All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package snyk + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/package-url/packageurl-go" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetPackageVulnerabilities_RetryRateLimited(t *testing.T) { + logger := zerolog.Nop() + var numRequests int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + numRequests++ + if numRequests == 1 { + w.Header().Set("X-RateLimit-Reset", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.Header().Set("Content-Type", "application/vnd.json+api") + _, err := w.Write([]byte(`{"data":[{"type":"issues","id":"VULN-ID"}]}`)) + require.NoError(t, err) + })) + cfg := DefaultConfig() + cfg.SnykAPIURL = srv.URL + + auth, err := AuthFromToken("asdf") + require.NoError(t, err) + + purl, err := packageurl.FromString("pkg:golang/github.com/snyk/parlay") + require.NoError(t, err) + + orgID := uuid.New() + issues, err := GetPackageVulnerabilities(cfg, &purl, auth, &orgID, &logger) + require.NoError(t, err) + + assert.NotZero(t, numRequests, "retries failed requests") + assert.NotNil(t, issues, "should retrieve issues") +} diff --git a/lib/snyk/service.go b/lib/snyk/service.go index f50ca3b..1bee8be 100644 --- a/lib/snyk/service.go +++ b/lib/snyk/service.go @@ -41,7 +41,7 @@ func (svc *serviceImpl) GetPackageVulnerabilities(purl *packageurl.PackageURL) ( return nil, err } - return GetPackageVulnerabilities(svc.cfg, purl, auth, orgID) + return GetPackageVulnerabilities(svc.cfg, purl, auth, orgID, svc.logger) } func (svc *serviceImpl) getAuth() (*securityprovider.SecurityProviderApiKey, error) {