From 566d3bc0c8fa833a53204f1d7347692932776028 Mon Sep 17 00:00:00 2001 From: Paul Rosca Date: Mon, 19 Feb 2024 17:31:21 +0200 Subject: [PATCH] feat: improve discoverComps readability and test coverage --- internal/utils/cdx.go | 29 +++++++++ internal/utils/cdx_test.go | 33 ++++++++++ lib/ecosystems/enrich_cyclonedx.go | 22 ++----- lib/ecosystems/enrich_cyclonedx_test.go | 49 ++++++++++++++- lib/scorecard/enrich_cyclonedx.go | 25 ++++---- lib/scorecard/enrich_test.go | 40 ++++++++++++- lib/snyk/enrich_cyclonedx.go | 19 +++--- lib/snyk/enrich_test.go | 36 +++++++++++ lib/snyk/testdata/pandas_issues.json | 80 +++++++++++++++++++++++++ 9 files changed, 287 insertions(+), 46 deletions(-) create mode 100644 internal/utils/cdx.go create mode 100644 internal/utils/cdx_test.go create mode 100644 lib/snyk/testdata/pandas_issues.json diff --git a/internal/utils/cdx.go b/internal/utils/cdx.go new file mode 100644 index 0000000..3bc4421 --- /dev/null +++ b/internal/utils/cdx.go @@ -0,0 +1,29 @@ +package utils + +import ( + cdx "github.com/CycloneDX/cyclonedx-go" +) + +func traverseComponent(comps *[]*cdx.Component, comp *cdx.Component) { + *comps = append(*comps, comp) + if comp.Components == nil { + return + } + for i := range *comp.Components { + traverseComponent(comps, &(*comp.Components)[i]) + } +} + +func DiscoverCDXComponents(bom *cdx.BOM) []*cdx.Component { + comps := make([]*cdx.Component, 0) + if bom.Metadata != nil && bom.Metadata.Component != nil { + traverseComponent(&comps, bom.Metadata.Component) + } + + if bom.Components != nil { + for i := range *bom.Components { + traverseComponent(&comps, &(*bom.Components)[i]) + } + } + return comps +} diff --git a/internal/utils/cdx_test.go b/internal/utils/cdx_test.go new file mode 100644 index 0000000..cede7ab --- /dev/null +++ b/internal/utils/cdx_test.go @@ -0,0 +1,33 @@ +package utils_test + +import ( + "testing" + + "github.com/snyk/parlay/internal/utils" + + cdx "github.com/CycloneDX/cyclonedx-go" + "github.com/stretchr/testify/assert" +) + +func TestDiscoverCDXComponents(t *testing.T) { + assert := assert.New(t) + + bom := &cdx.BOM{ + Metadata: &cdx.Metadata{ + Component: &cdx.Component{ + Name: "MetaComp", + }, + }, + Components: &[]cdx.Component{ + { + Name: "Parent", + Components: &[]cdx.Component{ + {Name: "Child"}, + }, + }, + }, + } + result := utils.DiscoverCDXComponents(bom) + + assert.Equal(len(result), 3) +} diff --git a/lib/ecosystems/enrich_cyclonedx.go b/lib/ecosystems/enrich_cyclonedx.go index f37dca1..9870909 100644 --- a/lib/ecosystems/enrich_cyclonedx.go +++ b/lib/ecosystems/enrich_cyclonedx.go @@ -24,6 +24,7 @@ import ( "github.com/remeh/sizedwaitgroup" "github.com/snyk/parlay/ecosystems/packages" + "github.com/snyk/parlay/internal/utils" ) type cdxEnricher = func(cdx.Component, packages.Package) cdx.Component @@ -191,25 +192,10 @@ func enrichCDXTopics(component cdx.Component, packageData packages.Package) cdx. return component } -func discoverComps(comps *[]*cdx.Component, childComps *[]cdx.Component) { - if childComps == nil { - return - } - for i := range *childComps { - *comps = append(*comps, &(*childComps)[i]) - discoverComps(comps, (*childComps)[i].Components) - } -} - func enrichCDX(bom *cdx.BOM) { - if bom.Components == nil { - return - } - + comps := utils.DiscoverCDXComponents(bom) wg := sizedwaitgroup.New(20) - deepComps := make([]*cdx.Component, 0) - discoverComps(&deepComps, bom.Components) - for i := range deepComps { + for i := range comps { wg.Add() go func(component *cdx.Component) { defer wg.Done() @@ -226,7 +212,7 @@ func enrichCDX(bom *cdx.BOM) { } } } - }(deepComps[i]) + }(comps[i]) } wg.Wait() } diff --git a/lib/ecosystems/enrich_cyclonedx_test.go b/lib/ecosystems/enrich_cyclonedx_test.go index a297f29..0bee223 100644 --- a/lib/ecosystems/enrich_cyclonedx_test.go +++ b/lib/ecosystems/enrich_cyclonedx_test.go @@ -44,6 +44,15 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { }) bom := &cdx.BOM{ + Metadata: &cdx.Metadata{ + Component: &cdx.Component{ + BOMRef: "pkg:golang/github.com/ACME/Project@v1.0.0", + Type: cdx.ComponentTypeApplication, + Name: "Project", + Version: "v1.0.0", + PackageURL: "pkg:golang/github.com/ACME/Project@v1.0.0", + }, + }, Components: &[]cdx.Component{ { BOMRef: "pkg:golang/github.com/CycloneDX/cyclonedx-go@v0.3.0", @@ -69,7 +78,45 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { httpmock.GetTotalCallCount() calls := httpmock.GetCallCountInfo() - assert.Equal(t, len(components), calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) + assert.Equal(t, 2, calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) +} + +func TestEnrichSBOM_CycloneDX_NestedComps(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries`, + func(req *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(200, map[string]interface{}{}) + }) + + bom := &cdx.BOM{ + Components: &[]cdx.Component{ + { + BOMRef: "@emotion/babel-plugin@11.11.0", + Type: cdx.ComponentTypeLibrary, + Name: "babel-plugin", + Version: "v11.11.0", + PackageURL: "pkg:npm/%40emotion/babel-plugin@11.11.0", + Components: &[]cdx.Component{ + { + Type: cdx.ComponentTypeLibrary, + Name: "convert-source-map", + Version: "v1.9.0", + BOMRef: "@emotion/babel-plugin@11.11.0|convert-source-map@1.9.0", + PackageURL: "pkg:npm/convert-source-map@1.9.0", + }, + }, + }, + }, + } + doc := &sbom.SBOMDocument{BOM: bom} + + EnrichSBOM(doc) + + httpmock.GetTotalCallCount() + calls := httpmock.GetCallCountInfo() + assert.Equal(t, 2, calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) } func TestEnrichSBOMWithoutLicense(t *testing.T) { diff --git a/lib/scorecard/enrich_cyclonedx.go b/lib/scorecard/enrich_cyclonedx.go index 46deac8..1599c9e 100644 --- a/lib/scorecard/enrich_cyclonedx.go +++ b/lib/scorecard/enrich_cyclonedx.go @@ -24,6 +24,7 @@ import ( "github.com/package-url/packageurl-go" "github.com/remeh/sizedwaitgroup" + "github.com/snyk/parlay/internal/utils" "github.com/snyk/parlay/lib/ecosystems" ) @@ -42,17 +43,16 @@ func cdxEnrichExternalReference(component cdx.Component, url string, comment str } func enrichCDX(bom *cdx.BOM) { - if bom.Components == nil { - return - } - + comps := utils.DiscoverCDXComponents(bom) wg := sizedwaitgroup.New(20) - newComponents := make([]cdx.Component, len(*bom.Components)) - for i, component := range *bom.Components { + for i := range comps { wg.Add() - go func(component cdx.Component, i int) { - // TODO: return when there is no usable Purl on the component. - purl, _ := packageurl.FromString(component.PackageURL) //nolint:errcheck + go func(component *cdx.Component) { + defer wg.Done() + purl, err := packageurl.FromString(component.PackageURL) + if err != nil { + return + } resp, err := ecosystems.GetPackageData(purl) if err == nil && resp.JSON200 != nil && resp.JSON200.RepositoryUrl != nil { scorecardUrl := strings.ReplaceAll(*resp.JSON200.RepositoryUrl, "https://", "https://api.securityscorecards.dev/projects/") @@ -60,14 +60,11 @@ func enrichCDX(bom *cdx.BOM) { if err == nil { defer response.Body.Close() if response.StatusCode == http.StatusOK { - component = cdxEnrichExternalReference(component, scorecardUrl, "OpenSSF Scorecard", cdx.ERTypeOther) + *component = cdxEnrichExternalReference(*component, scorecardUrl, "OpenSSF Scorecard", cdx.ERTypeOther) } } } - newComponents[i] = component - wg.Done() - }(component, i) + }(comps[i]) } wg.Wait() - bom.Components = &newComponents } diff --git a/lib/scorecard/enrich_test.go b/lib/scorecard/enrich_test.go index 0261427..260660d 100644 --- a/lib/scorecard/enrich_test.go +++ b/lib/scorecard/enrich_test.go @@ -39,7 +39,7 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { bom := &cdx.BOM{ Components: &[]cdx.Component{ { - PackageURL: "pkg:/example", + PackageURL: "pkg:type/example", }, }, } @@ -63,6 +63,44 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { assert.Equal(t, 1, calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) } +func TestEnrichSBOM_CycloneDX_NestedComponents(t *testing.T) { + teardown := setupEcosystemsAPIMock(t) + defer teardown() + + bom := &cdx.BOM{ + Components: &[]cdx.Component{ + { + PackageURL: "pkg:type/example", + Components: &[]cdx.Component{ + { + PackageURL: "pkg:otherType/otherExample", + }, + }, + }, + }, + } + doc := &sbom.SBOMDocument{BOM: bom} + + EnrichSBOM(doc) + + assert.NotNil(t, bom.Components) + assert.Len(t, *bom.Components, 1) + + for i := range *bom.Components { + enrichedComponent := (*bom.Components)[i] + assert.NotNil(t, enrichedComponent.ExternalReferences) + assert.Len(t, *enrichedComponent.ExternalReferences, 1) + assert.Equal(t, scorecardURL, (*enrichedComponent.ExternalReferences)[0].URL) + assert.Equal(t, "OpenSSF Scorecard", (*enrichedComponent.ExternalReferences)[0].Comment) + assert.Equal(t, cdx.ERTypeOther, (*enrichedComponent.ExternalReferences)[0].Type) + } + + total := httpmock.GetTotalCallCount() + assert.Equal(t, 4, total) + calls := httpmock.GetCallCountInfo() + assert.Equal(t, 2, calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) +} + func TestEnrichSBOM_ErrorFetchingPackageData(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() diff --git a/lib/snyk/enrich_cyclonedx.go b/lib/snyk/enrich_cyclonedx.go index 813da9f..99ac218 100644 --- a/lib/snyk/enrich_cyclonedx.go +++ b/lib/snyk/enrich_cyclonedx.go @@ -27,14 +27,11 @@ import ( "github.com/remeh/sizedwaitgroup" "github.com/rs/zerolog" + "github.com/snyk/parlay/internal/utils" "github.com/snyk/parlay/snyk/issues" ) func enrichCycloneDX(bom *cdx.BOM, logger zerolog.Logger) *cdx.BOM { - if bom.Components == nil { - return bom - } - auth, err := AuthFromToken(APIToken()) if err != nil { logger.Fatal().Err(err).Msg("Failed to authenticate.") @@ -47,15 +44,14 @@ func enrichCycloneDX(bom *cdx.BOM, logger zerolog.Logger) *cdx.BOM { return nil } - wg := sizedwaitgroup.New(20) var mutex = &sync.Mutex{} vulnerabilities := make(map[cdx.Component][]issues.CommonIssueModelVTwo) - - for i, component := range *bom.Components { + comps := utils.DiscoverCDXComponents(bom) + wg := sizedwaitgroup.New(20) + for i := range comps { wg.Add() - go func(component cdx.Component, i int) { + go func(component *cdx.Component) { defer wg.Done() - purl, err := packageurl.FromString(component.PackageURL) if err != nil { logger.Debug(). @@ -84,12 +80,11 @@ func enrichCycloneDX(bom *cdx.BOM, logger zerolog.Logger) *cdx.BOM { if packageDoc.Data != nil { mutex.Lock() - vulnerabilities[component] = *packageDoc.Data + vulnerabilities[*component] = *packageDoc.Data mutex.Unlock() } - }(component, i) + }(comps[i]) } - wg.Wait() var vulns []cdx.Vulnerability diff --git a/lib/snyk/enrich_test.go b/lib/snyk/enrich_test.go index 11636b1..0ce70ed 100644 --- a/lib/snyk/enrich_test.go +++ b/lib/snyk/enrich_test.go @@ -39,6 +39,37 @@ func TestEnrichSBOM_CycloneDXWithVulnerabilities(t *testing.T) { assert.Equal(t, "SNYK-PYTHON-NUMPY-73513", vuln.ID) } +func TestEnrichSBOM_CycloneDXWithVulnerabilities_NestedComponents(t *testing.T) { + teardown := setupTestEnv(t) + defer teardown() + + bom := &cdx.BOM{ + Components: &[]cdx.Component{ + { + BOMRef: "pkg:pypi/pandas@0.15.0", + Name: "pandas", + Version: "0.15.0", + PackageURL: "pkg:pypi/pandas@0.15.0", + Components: &[]cdx.Component{ + { + BOMRef: "pkg:pypi/numpy@1.16.0", + Name: "numpy", + Version: "1.16.0", + PackageURL: "pkg:pypi/numpy@1.16.0", + }, + }, + }, + }, + } + doc := &sbom.SBOMDocument{BOM: bom} + logger := zerolog.Nop() + + EnrichSBOM(doc, logger) + + assert.NotNil(t, bom.Vulnerabilities) + assert.Len(t, *bom.Vulnerabilities, 2) +} + func TestEnrichSBOM_CycloneDXWithoutVulnerabilities(t *testing.T) { teardown := setupTestEnv(t) defer teardown() @@ -109,6 +140,11 @@ func setupTestEnv(t *testing.T) func() { `=~^https://api\.snyk\.io/rest/orgs/[a-z0-9-]+/packages/pkg%3Apypi%2Fnumpy%401.16.0/issues`, httpmock.NewJsonResponderOrPanic(200, httpmock.File("testdata/numpy_issues.json")), ) + httpmock.RegisterResponder( + "GET", + `=~^https://api\.snyk\.io/rest/orgs/[a-z0-9-]+/packages/pkg%3Apypi%2Fpandas%400.15.0/issues`, + httpmock.NewJsonResponderOrPanic(200, httpmock.File("testdata/pandas_issues.json")), + ) httpmock.RegisterResponder( "GET", `=~^https://api\.snyk\.io/rest/orgs/[a-z0-9-]+/packages/.*/issues`, diff --git a/lib/snyk/testdata/pandas_issues.json b/lib/snyk/testdata/pandas_issues.json new file mode 100644 index 0000000..2a7dad5 --- /dev/null +++ b/lib/snyk/testdata/pandas_issues.json @@ -0,0 +1,80 @@ +{ + "jsonapi": { + "version": "1.0" + }, + "data": [ + { + "id": "SNYK-PYTHON-PANDAS-5879012", + "type": "issue", + "attributes": { + "key": "SNYK-PYTHON-PANDAS-5879012", + "title": "SQL Injection", + "type": "package_vulnerability", + "created_at": "2023-09-01T12:57:12.082697Z", + "updated_at": "2023-09-03T09:33:34.592326Z", + "description": "## Overview\n[pandas](https://pypi.org/project/pandas/) is a Python package providing data structures designed to make working with structured (tabular, multidimensional, potentially heterogeneous) and time series data both easy and intuitive.\n\nAffected versions of this package are vulnerable to SQL Injection via `sql.py`, due to improper input sanitization.\n## Remediation\nUpgrade `pandas` to version 0.16.0 or higher.\n## References\n- [GitHub Commit](https://github.com/pandas-dev/pandas/commit/a774ee84485412459f7205cccd87b639022afd07)\n- [GitHub PR](https://github.com/pandas-dev/pandas/pull/8986)\n", + "problems": [ + { + "id": "CWE-89", + "source": "CWE" + }, + { + "id": "PVE-2023-99975", + "source": "PVE" + } + ], + "coordinates": [ + { + "remedies": [ + { + "type": "indeterminate", + "description": "Upgrade the package version to 0.16.0 to fix this vulnerability", + "details": { + "upgrade_package": "0.16.0" + } + } + ], + "representation": [ + "[,0.16.0)" + ] + } + ], + "severities": [ + { + "source": "Snyk", + "level": "high", + "score": 7.3, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L" + } + ], + "effective_severity_level": "high", + "slots": { + "disclosure_time": "2023-09-01T06:35:07Z", + "exploit": "Not Defined", + "publication_time": "2023-09-03T09:33:34.590125Z", + "references": [ + { + "url": "https://github.com/pandas-dev/pandas/commit/a774ee84485412459f7205cccd87b639022afd07", + "title": "GitHub Commit" + }, + { + "url": "https://github.com/pandas-dev/pandas/pull/8986", + "title": "GitHub PR" + } + ] + } + } + } + ], + "links": { + "self": "/orgs/c8d8dc11-1d72-4051-bd73-9eb8694ceaa0/packages/pkg%3Apypi%2Fpandas%400.15.0/issues?version=2023-11-27&limit=1000&offset=0" + }, + "meta": { + "package": { + "name": "pandas", + "type": "pypi", + "url": "pkg:pypi/pandas@0.15.0", + "version": "0.15.0" + } + } +}