From c7c25f7b37dfe9050cf80c7f0370b52f97002961 Mon Sep 17 00:00:00 2001 From: Paul Rosca Date: Mon, 19 Feb 2024 17:31:21 +0200 Subject: [PATCH] feat: improve discoverComps readability and test coverage --- internal/utils/cdx.go | 29 +++++++++ internal/utils/cdx_test.go | 25 ++++++++ lib/ecosystems/enrich_cyclonedx.go | 22 ++----- lib/ecosystems/enrich_cyclonedx_test.go | 49 ++++++++++++++- lib/scorecard/enrich_cyclonedx.go | 25 ++++---- lib/scorecard/enrich_test.go | 40 ++++++++++++- lib/snyk/enrich_cyclonedx.go | 19 +++--- lib/snyk/enrich_test.go | 36 +++++++++++ lib/snyk/testdata/pandas_issues.json | 80 +++++++++++++++++++++++++ 9 files changed, 279 insertions(+), 46 deletions(-) create mode 100644 internal/utils/cdx.go create mode 100644 internal/utils/cdx_test.go create mode 100644 lib/snyk/testdata/pandas_issues.json diff --git a/internal/utils/cdx.go b/internal/utils/cdx.go new file mode 100644 index 0000000..a6f81e0 --- /dev/null +++ b/internal/utils/cdx.go @@ -0,0 +1,29 @@ +package utils + +import ( + cdx "github.com/CycloneDX/cyclonedx-go" +) + +func CDXDiscoverComps(comps *[]*cdx.Component, comp *cdx.Component) { + *comps = append(*comps, comp) + if comp.Components == nil { + return + } + for i := range *comp.Components { + CDXDiscoverComps(comps, &(*comp.Components)[i]) + } +} + +func CDXRetrieveComps(bom *cdx.BOM) []*cdx.Component { + comps := make([]*cdx.Component, 0) + if bom.Metadata != nil && bom.Metadata.Component != nil { + CDXDiscoverComps(&comps, bom.Metadata.Component) + } + + if bom.Components != nil { + for i := range *bom.Components { + CDXDiscoverComps(&comps, &(*bom.Components)[i]) + } + } + return comps +} diff --git a/internal/utils/cdx_test.go b/internal/utils/cdx_test.go new file mode 100644 index 0000000..c724f11 --- /dev/null +++ b/internal/utils/cdx_test.go @@ -0,0 +1,25 @@ +package utils_test + +import ( + "testing" + + cdx "github.com/CycloneDX/cyclonedx-go" + "github.com/stretchr/testify/assert" + + "github.com/snyk/parlay/internal/utils" +) + +func TestCDXDiscoverComps(t *testing.T) { + assert := assert.New(t) + + component := cdx.Component{ + Name: "Parent", + Components: &[]cdx.Component{ + {Name: "Child"}, + }, + } + result := make([]*cdx.Component, 0) + utils.CDXDiscoverComps(&result, &component) + + assert.Equal(len(result), 2) +} diff --git a/lib/ecosystems/enrich_cyclonedx.go b/lib/ecosystems/enrich_cyclonedx.go index f37dca1..ec80b12 100644 --- a/lib/ecosystems/enrich_cyclonedx.go +++ b/lib/ecosystems/enrich_cyclonedx.go @@ -24,6 +24,7 @@ import ( "github.com/remeh/sizedwaitgroup" "github.com/snyk/parlay/ecosystems/packages" + "github.com/snyk/parlay/internal/utils" ) type cdxEnricher = func(cdx.Component, packages.Package) cdx.Component @@ -191,25 +192,10 @@ func enrichCDXTopics(component cdx.Component, packageData packages.Package) cdx. return component } -func discoverComps(comps *[]*cdx.Component, childComps *[]cdx.Component) { - if childComps == nil { - return - } - for i := range *childComps { - *comps = append(*comps, &(*childComps)[i]) - discoverComps(comps, (*childComps)[i].Components) - } -} - func enrichCDX(bom *cdx.BOM) { - if bom.Components == nil { - return - } - + comps := utils.CDXRetrieveComps(bom) wg := sizedwaitgroup.New(20) - deepComps := make([]*cdx.Component, 0) - discoverComps(&deepComps, bom.Components) - for i := range deepComps { + for i := range comps { wg.Add() go func(component *cdx.Component) { defer wg.Done() @@ -226,7 +212,7 @@ func enrichCDX(bom *cdx.BOM) { } } } - }(deepComps[i]) + }(comps[i]) } wg.Wait() } diff --git a/lib/ecosystems/enrich_cyclonedx_test.go b/lib/ecosystems/enrich_cyclonedx_test.go index a297f29..0bee223 100644 --- a/lib/ecosystems/enrich_cyclonedx_test.go +++ b/lib/ecosystems/enrich_cyclonedx_test.go @@ -44,6 +44,15 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { }) bom := &cdx.BOM{ + Metadata: &cdx.Metadata{ + Component: &cdx.Component{ + BOMRef: "pkg:golang/github.com/ACME/Project@v1.0.0", + Type: cdx.ComponentTypeApplication, + Name: "Project", + Version: "v1.0.0", + PackageURL: "pkg:golang/github.com/ACME/Project@v1.0.0", + }, + }, Components: &[]cdx.Component{ { BOMRef: "pkg:golang/github.com/CycloneDX/cyclonedx-go@v0.3.0", @@ -69,7 +78,45 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { httpmock.GetTotalCallCount() calls := httpmock.GetCallCountInfo() - assert.Equal(t, len(components), calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) + assert.Equal(t, 2, calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) +} + +func TestEnrichSBOM_CycloneDX_NestedComps(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries`, + func(req *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(200, map[string]interface{}{}) + }) + + bom := &cdx.BOM{ + Components: &[]cdx.Component{ + { + BOMRef: "@emotion/babel-plugin@11.11.0", + Type: cdx.ComponentTypeLibrary, + Name: "babel-plugin", + Version: "v11.11.0", + PackageURL: "pkg:npm/%40emotion/babel-plugin@11.11.0", + Components: &[]cdx.Component{ + { + Type: cdx.ComponentTypeLibrary, + Name: "convert-source-map", + Version: "v1.9.0", + BOMRef: "@emotion/babel-plugin@11.11.0|convert-source-map@1.9.0", + PackageURL: "pkg:npm/convert-source-map@1.9.0", + }, + }, + }, + }, + } + doc := &sbom.SBOMDocument{BOM: bom} + + EnrichSBOM(doc) + + httpmock.GetTotalCallCount() + calls := httpmock.GetCallCountInfo() + assert.Equal(t, 2, calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) } func TestEnrichSBOMWithoutLicense(t *testing.T) { diff --git a/lib/scorecard/enrich_cyclonedx.go b/lib/scorecard/enrich_cyclonedx.go index 46deac8..95383cc 100644 --- a/lib/scorecard/enrich_cyclonedx.go +++ b/lib/scorecard/enrich_cyclonedx.go @@ -24,6 +24,7 @@ import ( "github.com/package-url/packageurl-go" "github.com/remeh/sizedwaitgroup" + "github.com/snyk/parlay/internal/utils" "github.com/snyk/parlay/lib/ecosystems" ) @@ -42,17 +43,16 @@ func cdxEnrichExternalReference(component cdx.Component, url string, comment str } func enrichCDX(bom *cdx.BOM) { - if bom.Components == nil { - return - } - + comps := utils.CDXRetrieveComps(bom) wg := sizedwaitgroup.New(20) - newComponents := make([]cdx.Component, len(*bom.Components)) - for i, component := range *bom.Components { + for i := range comps { wg.Add() - go func(component cdx.Component, i int) { - // TODO: return when there is no usable Purl on the component. - purl, _ := packageurl.FromString(component.PackageURL) //nolint:errcheck + go func(component *cdx.Component) { + defer wg.Done() + purl, err := packageurl.FromString(component.PackageURL) + if err != nil { + return + } resp, err := ecosystems.GetPackageData(purl) if err == nil && resp.JSON200 != nil && resp.JSON200.RepositoryUrl != nil { scorecardUrl := strings.ReplaceAll(*resp.JSON200.RepositoryUrl, "https://", "https://api.securityscorecards.dev/projects/") @@ -60,14 +60,11 @@ func enrichCDX(bom *cdx.BOM) { if err == nil { defer response.Body.Close() if response.StatusCode == http.StatusOK { - component = cdxEnrichExternalReference(component, scorecardUrl, "OpenSSF Scorecard", cdx.ERTypeOther) + *component = cdxEnrichExternalReference(*component, scorecardUrl, "OpenSSF Scorecard", cdx.ERTypeOther) } } } - newComponents[i] = component - wg.Done() - }(component, i) + }(comps[i]) } wg.Wait() - bom.Components = &newComponents } diff --git a/lib/scorecard/enrich_test.go b/lib/scorecard/enrich_test.go index 0261427..260660d 100644 --- a/lib/scorecard/enrich_test.go +++ b/lib/scorecard/enrich_test.go @@ -39,7 +39,7 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { bom := &cdx.BOM{ Components: &[]cdx.Component{ { - PackageURL: "pkg:/example", + PackageURL: "pkg:type/example", }, }, } @@ -63,6 +63,44 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { assert.Equal(t, 1, calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) } +func TestEnrichSBOM_CycloneDX_NestedComponents(t *testing.T) { + teardown := setupEcosystemsAPIMock(t) + defer teardown() + + bom := &cdx.BOM{ + Components: &[]cdx.Component{ + { + PackageURL: "pkg:type/example", + Components: &[]cdx.Component{ + { + PackageURL: "pkg:otherType/otherExample", + }, + }, + }, + }, + } + doc := &sbom.SBOMDocument{BOM: bom} + + EnrichSBOM(doc) + + assert.NotNil(t, bom.Components) + assert.Len(t, *bom.Components, 1) + + for i := range *bom.Components { + enrichedComponent := (*bom.Components)[i] + assert.NotNil(t, enrichedComponent.ExternalReferences) + assert.Len(t, *enrichedComponent.ExternalReferences, 1) + assert.Equal(t, scorecardURL, (*enrichedComponent.ExternalReferences)[0].URL) + assert.Equal(t, "OpenSSF Scorecard", (*enrichedComponent.ExternalReferences)[0].Comment) + assert.Equal(t, cdx.ERTypeOther, (*enrichedComponent.ExternalReferences)[0].Type) + } + + total := httpmock.GetTotalCallCount() + assert.Equal(t, 4, total) + calls := httpmock.GetCallCountInfo() + assert.Equal(t, 2, calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) +} + func TestEnrichSBOM_ErrorFetchingPackageData(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() diff --git a/lib/snyk/enrich_cyclonedx.go b/lib/snyk/enrich_cyclonedx.go index 813da9f..b974e9f 100644 --- a/lib/snyk/enrich_cyclonedx.go +++ b/lib/snyk/enrich_cyclonedx.go @@ -27,14 +27,11 @@ import ( "github.com/remeh/sizedwaitgroup" "github.com/rs/zerolog" + "github.com/snyk/parlay/internal/utils" "github.com/snyk/parlay/snyk/issues" ) func enrichCycloneDX(bom *cdx.BOM, logger zerolog.Logger) *cdx.BOM { - if bom.Components == nil { - return bom - } - auth, err := AuthFromToken(APIToken()) if err != nil { logger.Fatal().Err(err).Msg("Failed to authenticate.") @@ -47,15 +44,14 @@ func enrichCycloneDX(bom *cdx.BOM, logger zerolog.Logger) *cdx.BOM { return nil } - wg := sizedwaitgroup.New(20) var mutex = &sync.Mutex{} vulnerabilities := make(map[cdx.Component][]issues.CommonIssueModelVTwo) - - for i, component := range *bom.Components { + comps := utils.CDXRetrieveComps(bom) + wg := sizedwaitgroup.New(20) + for i := range comps { wg.Add() - go func(component cdx.Component, i int) { + go func(component *cdx.Component) { defer wg.Done() - purl, err := packageurl.FromString(component.PackageURL) if err != nil { logger.Debug(). @@ -84,12 +80,11 @@ func enrichCycloneDX(bom *cdx.BOM, logger zerolog.Logger) *cdx.BOM { if packageDoc.Data != nil { mutex.Lock() - vulnerabilities[component] = *packageDoc.Data + vulnerabilities[*component] = *packageDoc.Data mutex.Unlock() } - }(component, i) + }(comps[i]) } - wg.Wait() var vulns []cdx.Vulnerability diff --git a/lib/snyk/enrich_test.go b/lib/snyk/enrich_test.go index 11636b1..0ce70ed 100644 --- a/lib/snyk/enrich_test.go +++ b/lib/snyk/enrich_test.go @@ -39,6 +39,37 @@ func TestEnrichSBOM_CycloneDXWithVulnerabilities(t *testing.T) { assert.Equal(t, "SNYK-PYTHON-NUMPY-73513", vuln.ID) } +func TestEnrichSBOM_CycloneDXWithVulnerabilities_NestedComponents(t *testing.T) { + teardown := setupTestEnv(t) + defer teardown() + + bom := &cdx.BOM{ + Components: &[]cdx.Component{ + { + BOMRef: "pkg:pypi/pandas@0.15.0", + Name: "pandas", + Version: "0.15.0", + PackageURL: "pkg:pypi/pandas@0.15.0", + Components: &[]cdx.Component{ + { + BOMRef: "pkg:pypi/numpy@1.16.0", + Name: "numpy", + Version: "1.16.0", + PackageURL: "pkg:pypi/numpy@1.16.0", + }, + }, + }, + }, + } + doc := &sbom.SBOMDocument{BOM: bom} + logger := zerolog.Nop() + + EnrichSBOM(doc, logger) + + assert.NotNil(t, bom.Vulnerabilities) + assert.Len(t, *bom.Vulnerabilities, 2) +} + func TestEnrichSBOM_CycloneDXWithoutVulnerabilities(t *testing.T) { teardown := setupTestEnv(t) defer teardown() @@ -109,6 +140,11 @@ func setupTestEnv(t *testing.T) func() { `=~^https://api\.snyk\.io/rest/orgs/[a-z0-9-]+/packages/pkg%3Apypi%2Fnumpy%401.16.0/issues`, httpmock.NewJsonResponderOrPanic(200, httpmock.File("testdata/numpy_issues.json")), ) + httpmock.RegisterResponder( + "GET", + `=~^https://api\.snyk\.io/rest/orgs/[a-z0-9-]+/packages/pkg%3Apypi%2Fpandas%400.15.0/issues`, + httpmock.NewJsonResponderOrPanic(200, httpmock.File("testdata/pandas_issues.json")), + ) httpmock.RegisterResponder( "GET", `=~^https://api\.snyk\.io/rest/orgs/[a-z0-9-]+/packages/.*/issues`, diff --git a/lib/snyk/testdata/pandas_issues.json b/lib/snyk/testdata/pandas_issues.json new file mode 100644 index 0000000..2a7dad5 --- /dev/null +++ b/lib/snyk/testdata/pandas_issues.json @@ -0,0 +1,80 @@ +{ + "jsonapi": { + "version": "1.0" + }, + "data": [ + { + "id": "SNYK-PYTHON-PANDAS-5879012", + "type": "issue", + "attributes": { + "key": "SNYK-PYTHON-PANDAS-5879012", + "title": "SQL Injection", + "type": "package_vulnerability", + "created_at": "2023-09-01T12:57:12.082697Z", + "updated_at": "2023-09-03T09:33:34.592326Z", + "description": "## Overview\n[pandas](https://pypi.org/project/pandas/) is a Python package providing data structures designed to make working with structured (tabular, multidimensional, potentially heterogeneous) and time series data both easy and intuitive.\n\nAffected versions of this package are vulnerable to SQL Injection via `sql.py`, due to improper input sanitization.\n## Remediation\nUpgrade `pandas` to version 0.16.0 or higher.\n## References\n- [GitHub Commit](https://github.com/pandas-dev/pandas/commit/a774ee84485412459f7205cccd87b639022afd07)\n- [GitHub PR](https://github.com/pandas-dev/pandas/pull/8986)\n", + "problems": [ + { + "id": "CWE-89", + "source": "CWE" + }, + { + "id": "PVE-2023-99975", + "source": "PVE" + } + ], + "coordinates": [ + { + "remedies": [ + { + "type": "indeterminate", + "description": "Upgrade the package version to 0.16.0 to fix this vulnerability", + "details": { + "upgrade_package": "0.16.0" + } + } + ], + "representation": [ + "[,0.16.0)" + ] + } + ], + "severities": [ + { + "source": "Snyk", + "level": "high", + "score": 7.3, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L" + } + ], + "effective_severity_level": "high", + "slots": { + "disclosure_time": "2023-09-01T06:35:07Z", + "exploit": "Not Defined", + "publication_time": "2023-09-03T09:33:34.590125Z", + "references": [ + { + "url": "https://github.com/pandas-dev/pandas/commit/a774ee84485412459f7205cccd87b639022afd07", + "title": "GitHub Commit" + }, + { + "url": "https://github.com/pandas-dev/pandas/pull/8986", + "title": "GitHub PR" + } + ] + } + } + } + ], + "links": { + "self": "/orgs/c8d8dc11-1d72-4051-bd73-9eb8694ceaa0/packages/pkg%3Apypi%2Fpandas%400.15.0/issues?version=2023-11-27&limit=1000&offset=0" + }, + "meta": { + "package": { + "name": "pandas", + "type": "pypi", + "url": "pkg:pypi/pandas@0.15.0", + "version": "0.15.0" + } + } +}