Skip to content

Commit

Permalink
fix: handle Snyk API rate limit
Browse files Browse the repository at this point in the history
Closes #83.
  • Loading branch information
mcombuechen committed Nov 28, 2024
1 parent 6f99e54 commit 1be757c
Show file tree
Hide file tree
Showing 7 changed files with 115 additions and 10 deletions.
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/deepmap/oapi-codegen v1.12.4
github.com/edoardottt/depsdev v0.0.3
github.com/google/uuid v1.3.0
github.com/hashicorp/go-retryablehttp v0.7.7
github.com/jarcoal/httpmock v1.3.0
github.com/package-url/packageurl-go v0.1.2
github.com/remeh/sizedwaitgroup v1.0.0
Expand All @@ -23,11 +24,12 @@ require (
github.com/avast/retry-go v3.0.0+incompatible // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand All @@ -36,7 +38,7 @@ require (
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
golang.org/x/sys v0.3.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.5.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
14 changes: 11 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
Expand Down Expand Up @@ -140,6 +141,11 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
Expand Down Expand Up @@ -168,8 +174,9 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
Expand Down Expand Up @@ -371,8 +378,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
2 changes: 1 addition & 1 deletion lib/snyk/enrich_cyclonedx.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func enrichCycloneDX(cfg *Config, bom *cdx.BOM, logger *zerolog.Logger) *cdx.BOM
for _, enrichFunc := range cdxEnrichers {
enrichFunc(cfg, component, &purl)
}
resp, err := GetPackageVulnerabilities(cfg, &purl, auth, orgID)
resp, err := GetPackageVulnerabilities(cfg, &purl, auth, orgID, logger)
if err != nil {
l.Err(err).
Str("purl", purl.ToString()).
Expand Down
2 changes: 1 addition & 1 deletion lib/snyk/enrich_spdx.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func enrichSPDX(cfg *Config, bom *spdx.Document, logger *zerolog.Logger) *spdx.D
for _, enrichFn := range spdxEnrichers {
enrichFn(cfg, pkg, purl)
}
resp, err := GetPackageVulnerabilities(cfg, purl, auth, orgID)
resp, err := GetPackageVulnerabilities(cfg, purl, auth, orgID, logger)
if err != nil {
l.Err(err).
Str("purl", purl.ToString()).
Expand Down
39 changes: 37 additions & 2 deletions lib/snyk/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,14 @@ import (
"context"
"fmt"
"net/http"
"strconv"
"time"

"github.com/deepmap/oapi-codegen/pkg/securityprovider"
"github.com/google/uuid"
"github.com/hashicorp/go-retryablehttp"
"github.com/package-url/packageurl-go"
"github.com/rs/zerolog"

"github.com/snyk/parlay/snyk/issues"
)
Expand Down Expand Up @@ -82,8 +86,11 @@ func SnykVulnURL(cfg *Config, purl *packageurl.PackageURL) string {
return url
}

func GetPackageVulnerabilities(cfg *Config, purl *packageurl.PackageURL, auth *securityprovider.SecurityProviderApiKey, orgID *uuid.UUID) (*issues.FetchIssuesPerPurlResponse, error) {
client, err := issues.NewClientWithResponses(cfg.SnykAPIURL, issues.WithRequestEditorFn(auth.Intercept))
func GetPackageVulnerabilities(cfg *Config, purl *packageurl.PackageURL, auth *securityprovider.SecurityProviderApiKey, orgID *uuid.UUID, logger *zerolog.Logger) (*issues.FetchIssuesPerPurlResponse, error) {
client, err := issues.NewClientWithResponses(
cfg.SnykAPIURL,
issues.WithRequestEditorFn(auth.Intercept),
issues.WithHTTPClient(getRetryClient(logger)))
if err != nil {
return nil, err
}
Expand All @@ -100,3 +107,31 @@ func GetPackageVulnerabilities(cfg *Config, purl *packageurl.PackageURL, auth *s

return resp, nil
}

func getRetryClient(logger *zerolog.Logger) *http.Client {
rc := retryablehttp.NewClient()
rc.Logger = nil
rc.Backoff = func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
if sleep, ok := parseRateLimitHeader(resp.Header.Get("X-RateLimit-Reset")); ok {
logger.Warn().
Dur("Retry-After", sleep).
Msg("Getting rate-limited, waiting...")
return sleep
}
return retryablehttp.DefaultBackoff(min, max, attemptNum, resp)
}

return rc.StandardClient()
}

func parseRateLimitHeader(v string) (time.Duration, bool) {
if v == "" {
return 0, false
}

if sec, err := strconv.ParseInt(v, 10, 64); err == nil {
return time.Duration(sec) * time.Second, true
}

return 0, false
}
60 changes: 60 additions & 0 deletions lib/snyk/package_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* © 2023 Snyk Limited All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package snyk

import (
"net/http"
"net/http/httptest"
"testing"

"github.com/google/uuid"
"github.com/package-url/packageurl-go"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestGetPackageVulnerabilities_RetryRateLimited(t *testing.T) {
logger := zerolog.Nop()
var numRequests int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
numRequests++
if numRequests == 1 {
w.Header().Set("X-RateLimit-Reset", "1")
w.WriteHeader(http.StatusTooManyRequests)
return
}
w.Header().Set("Content-Type", "application/vnd.json+api")
_, err := w.Write([]byte(`{"data":[{"type":"issues","id":"VULN-ID"}]}`))
require.NoError(t, err)
}))
cfg := DefaultConfig()
cfg.SnykAPIURL = srv.URL

auth, err := AuthFromToken("asdf")
require.NoError(t, err)

purl, err := packageurl.FromString("pkg:golang/github.com/snyk/parlay")
require.NoError(t, err)

orgID := uuid.New()
issues, err := GetPackageVulnerabilities(cfg, &purl, auth, &orgID, &logger)
require.NoError(t, err)

assert.Equal(t, 2, numRequests, "retries failed requests")
assert.NotNil(t, issues, "should retrieve issues")
}
2 changes: 1 addition & 1 deletion lib/snyk/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func (svc *serviceImpl) GetPackageVulnerabilities(purl *packageurl.PackageURL) (
return nil, err
}

return GetPackageVulnerabilities(svc.cfg, purl, auth, orgID)
return GetPackageVulnerabilities(svc.cfg, purl, auth, orgID, svc.logger)
}

func (svc *serviceImpl) getAuth() (*securityprovider.SecurityProviderApiKey, error) {
Expand Down

0 comments on commit 1be757c

Please sign in to comment.