From 8506331d1a6440d3c44f5e4ee18cb37e73f21503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Comb=C3=BCchen?= Date: Mon, 4 Mar 2024 14:07:52 +0100 Subject: [PATCH] refactor: enrich ecosyste.ms data through mutation --- lib/ecosystems/enrich_cyclonedx.go | 126 +++++++++----------- lib/ecosystems/enrich_cyclonedx_test.go | 147 ++++++++++++------------ lib/ecosystems/enrich_spdx_test.go | 2 + 3 files changed, 132 insertions(+), 143 deletions(-) diff --git a/lib/ecosystems/enrich_cyclonedx.go b/lib/ecosystems/enrich_cyclonedx.go index c06e0bd..9edc08b 100644 --- a/lib/ecosystems/enrich_cyclonedx.go +++ b/lib/ecosystems/enrich_cyclonedx.go @@ -28,7 +28,7 @@ import ( "github.com/snyk/parlay/internal/utils" ) -type cdxEnricher = func(cdx.Component, packages.Package) cdx.Component +type cdxEnricher = func(*cdx.Component, *packages.Package) var cdxEnrichers = []cdxEnricher{ enrichCDXDescription, @@ -46,122 +46,114 @@ var cdxEnrichers = []cdxEnricher{ enrichCDXSupplier, } -func enrichCDXDescription(component cdx.Component, packageData packages.Package) cdx.Component { - if packageData.Description != nil { - component.Description = *packageData.Description +func enrichCDXDescription(comp *cdx.Component, data *packages.Package) { + if data.Description != nil { + comp.Description = *data.Description } - return component } -func enrichCDXLicense(component cdx.Component, packageData packages.Package) cdx.Component { - if packageData.NormalizedLicenses != nil { - if len(packageData.NormalizedLicenses) > 0 { - expression := packageData.NormalizedLicenses[0] +func enrichCDXLicense(comp *cdx.Component, data *packages.Package) { + if data.NormalizedLicenses != nil { + if len(data.NormalizedLicenses) > 0 { + expression := data.NormalizedLicenses[0] licenses := cdx.LicenseChoice{Expression: expression} - component.Licenses = &cdx.Licenses{licenses} + comp.Licenses = &cdx.Licenses{licenses} } } - return component } -func enrichExternalReference(component cdx.Component, _ packages.Package, url *string, refType cdx.ExternalReferenceType) cdx.Component { +func enrichExternalReference(comp *cdx.Component, url *string, refType cdx.ExternalReferenceType) { if url == nil { - return component + return } ext := cdx.ExternalReference{ URL: *url, Type: refType, } - if component.ExternalReferences == nil { - component.ExternalReferences = &[]cdx.ExternalReference{ext} + if comp.ExternalReferences == nil { + comp.ExternalReferences = &[]cdx.ExternalReference{ext} } else { - *component.ExternalReferences = append(*component.ExternalReferences, ext) + *comp.ExternalReferences = append(*comp.ExternalReferences, ext) } - return component } -func enrichProperty(component cdx.Component, name string, value string) cdx.Component { +func enrichProperty(comp *cdx.Component, name string, value string) { prop := cdx.Property{ Name: name, Value: value, } - if component.Properties == nil { - component.Properties = &[]cdx.Property{prop} + if comp.Properties == nil { + comp.Properties = &[]cdx.Property{prop} } else { - *component.Properties = append(*component.Properties, prop) + *comp.Properties = append(*comp.Properties, prop) } - return component } -func enrichCDXHomepage(component cdx.Component, packageData packages.Package) cdx.Component { - return enrichExternalReference(component, packageData, packageData.Homepage, cdx.ERTypeWebsite) +func enrichCDXHomepage(comp *cdx.Component, data *packages.Package) { + enrichExternalReference(comp, data.Homepage, cdx.ERTypeWebsite) } -func enrichCDXRegistryURL(component cdx.Component, packageData packages.Package) cdx.Component { - return enrichExternalReference(component, packageData, packageData.RegistryUrl, cdx.ERTypeDistribution) +func enrichCDXRegistryURL(comp *cdx.Component, data *packages.Package) { + enrichExternalReference(comp, data.RegistryUrl, cdx.ERTypeDistribution) } -func enrichCDXRepositoryURL(component cdx.Component, packageData packages.Package) cdx.Component { - return enrichExternalReference(component, packageData, packageData.RepositoryUrl, cdx.ERTypeVCS) +func enrichCDXRepositoryURL(comp *cdx.Component, data *packages.Package) { + enrichExternalReference(comp, data.RepositoryUrl, cdx.ERTypeVCS) } -func enrichCDXDocumentationURL(component cdx.Component, packageData packages.Package) cdx.Component { - return enrichExternalReference(component, packageData, packageData.DocumentationUrl, cdx.ERTypeDocumentation) +func enrichCDXDocumentationURL(comp *cdx.Component, data *packages.Package) { + enrichExternalReference(comp, data.DocumentationUrl, cdx.ERTypeDocumentation) } -func enrichCDXFirstReleasePublishedAt(component cdx.Component, packageData packages.Package) cdx.Component { - if packageData.FirstReleasePublishedAt == nil { - return component +func enrichCDXFirstReleasePublishedAt(comp *cdx.Component, data *packages.Package) { + if data.FirstReleasePublishedAt == nil { + return } - timestamp := packageData.FirstReleasePublishedAt.UTC().Format(time.RFC3339) - return enrichProperty(component, "ecosystems:first_release_published_at", timestamp) + timestamp := data.FirstReleasePublishedAt.UTC().Format(time.RFC3339) + enrichProperty(comp, "ecosystems:first_release_published_at", timestamp) } -func enrichCDXLatestReleasePublishedAt(component cdx.Component, packageData packages.Package) cdx.Component { - if packageData.LatestReleasePublishedAt == nil { - return component +func enrichCDXLatestReleasePublishedAt(comp *cdx.Component, data *packages.Package) { + if data.LatestReleasePublishedAt == nil { + return } - timestamp := packageData.LatestReleasePublishedAt.UTC().Format(time.RFC3339) - return enrichProperty(component, "ecosystems:latest_release_published_at", timestamp) + timestamp := data.LatestReleasePublishedAt.UTC().Format(time.RFC3339) + enrichProperty(comp, "ecosystems:latest_release_published_at", timestamp) } -func enrichCDXRepoArchived(component cdx.Component, packageData packages.Package) cdx.Component { - if packageData.RepoMetadata != nil { - if archived, ok := (*packageData.RepoMetadata)["archived"].(bool); ok && archived { - return enrichProperty(component, "ecosystems:repository_archived", "true") +func enrichCDXRepoArchived(comp *cdx.Component, data *packages.Package) { + if data.RepoMetadata != nil { + if archived, ok := (*data.RepoMetadata)["archived"].(bool); ok && archived { + enrichProperty(comp, "ecosystems:repository_archived", "true") } } - return component } -func enrichCDXLocation(component cdx.Component, packageData packages.Package) cdx.Component { - if packageData.RepoMetadata != nil { - meta := *packageData.RepoMetadata +func enrichCDXLocation(comp *cdx.Component, data *packages.Package) { + if data.RepoMetadata != nil { + meta := *data.RepoMetadata if ownerRecord, ok := meta["owner_record"].(map[string]interface{}); ok { if location, ok := ownerRecord["location"].(string); ok { - return enrichProperty(component, "ecosystems:owner_location", location) + enrichProperty(comp, "ecosystems:owner_location", location) } } } - return component } -func enrichCDXAuthor(component cdx.Component, packageData packages.Package) cdx.Component { - if packageData.RepoMetadata != nil { - meta := *packageData.RepoMetadata +func enrichCDXAuthor(comp *cdx.Component, data *packages.Package) { + if data.RepoMetadata != nil { + meta := *data.RepoMetadata if ownerRecord, ok := meta["owner_record"].(map[string]interface{}); ok { if name, ok := ownerRecord["name"].(string); ok { - component.Author = name - return component + comp.Author = name } } } - return component } -func enrichCDXSupplier(component cdx.Component, packageData packages.Package) cdx.Component { - if packageData.RepoMetadata != nil { - meta := *packageData.RepoMetadata +func enrichCDXSupplier(comp *cdx.Component, data *packages.Package) { + if data.RepoMetadata != nil { + meta := *data.RepoMetadata if ownerRecord, ok := meta["owner_record"].(map[string]interface{}); ok { if name, ok := ownerRecord["name"].(string); ok { supplier := cdx.OrganizationalEntity{ @@ -171,26 +163,22 @@ func enrichCDXSupplier(component cdx.Component, packageData packages.Package) cd websites := []string{website} supplier.URL = &websites } - component.Supplier = &supplier - return component + comp.Supplier = &supplier } } } - return component } -func enrichCDXTopics(component cdx.Component, packageData packages.Package) cdx.Component { - if packageData.RepoMetadata != nil { - meta := *packageData.RepoMetadata +func enrichCDXTopics(comp *cdx.Component, data *packages.Package) { + if data.RepoMetadata != nil { + meta := *data.RepoMetadata if topics, ok := meta["topics"].([]interface{}); ok { for _, topic := range topics { - component = enrichProperty(component, "ecosystems:topic", topic.(string)) + enrichProperty(comp, "ecosystems:topic", topic.(string)) } } - return component } - return component } func enrichCDX(bom *cdx.BOM, logger *zerolog.Logger) { @@ -229,7 +217,7 @@ func enrichCDX(bom *cdx.BOM, logger *zerolog.Logger) { } for _, enrichFunc := range cdxEnrichers { - *comp = enrichFunc(*comp, *resp.JSON200) + enrichFunc(comp, resp.JSON200) } }(comps[i]) } diff --git a/lib/ecosystems/enrich_cyclonedx_test.go b/lib/ecosystems/enrich_cyclonedx_test.go index 000a07e..0027f7b 100644 --- a/lib/ecosystems/enrich_cyclonedx_test.go +++ b/lib/ecosystems/enrich_cyclonedx_test.go @@ -30,8 +30,6 @@ import ( "github.com/snyk/parlay/lib/sbom" ) -var logger = zerolog.Nop() - func TestEnrichSBOM_CycloneDX(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -67,6 +65,7 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} + logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -114,6 +113,7 @@ func TestEnrichSBOM_CycloneDX_NestedComps(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} + logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -146,6 +146,7 @@ func TestEnrichSBOMWithoutLicense(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} + logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -159,31 +160,34 @@ func TestEnrichSBOMWithoutLicense(t *testing.T) { } func TestEnrichDescription(t *testing.T) { - component := cdx.Component{ + component := &cdx.Component{ Type: cdx.ComponentTypeLibrary, Name: "cyclonedx-go", Version: "v0.3.0", } desc := "description" - pack := packages.Package{ + pack := &packages.Package{ Description: &desc, } - component = enrichCDXDescription(component, pack) + + enrichCDXDescription(component, pack) + assert.Equal(t, "description", component.Description) } func TestEnrichLicense(t *testing.T) { - component := cdx.Component{ + component := &cdx.Component{ Type: cdx.ComponentTypeLibrary, Name: "cyclonedx-go", Version: "v0.3.0", } - pack := packages.Package{ + pack := &packages.Package{ NormalizedLicenses: []string{"BSD-3-Clause"}, } - component = enrichCDXLicense(component, pack) - licenses := *component.Licenses + enrichCDXLicense(component, pack) + + licenses := *component.Licenses comp := cdx.LicenseChoice(cdx.LicenseChoice{Expression: "BSD-3-Clause"}) assert.Equal(t, comp, licenses[0]) } @@ -191,6 +195,7 @@ func TestEnrichLicense(t *testing.T) { func TestEnrichBlankSBOM(t *testing.T) { bom := new(cdx.BOM) doc := &sbom.SBOMDocument{BOM: bom} + logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -198,93 +203,91 @@ func TestEnrichBlankSBOM(t *testing.T) { } func TestEnrichExternalReferenceWithNilURL(t *testing.T) { - component := cdx.Component{} - packageData := packages.Package{Homepage: nil} + component := &cdx.Component{} + packageData := &packages.Package{Homepage: nil} - result := enrichExternalReference(component, packageData, packageData.Homepage, cdx.ERTypeWebsite) + enrichExternalReference(component, packageData.Homepage, cdx.ERTypeWebsite) - assert.Equal(t, component, result) + assert.Nil(t, component.ExternalReferences) } func TestEnrichExternalReferenceWithNonNullURL(t *testing.T) { - component := cdx.Component{} - packageData := packages.Package{Homepage: pointerToString("https://example.com")} + component := &cdx.Component{} + packageData := packages.Package{Homepage: pointerToString(t, "https://example.com")} - result := enrichExternalReference(component, packageData, packageData.Homepage, cdx.ERTypeWebsite) + enrichExternalReference(component, packageData.Homepage, cdx.ERTypeWebsite) - expected := cdx.Component{ - ExternalReferences: &[]cdx.ExternalReference{ - {URL: "https://example.com", Type: cdx.ERTypeWebsite}, - }, + expected := &[]cdx.ExternalReference{ + {URL: "https://example.com", Type: cdx.ERTypeWebsite}, } - assert.Equal(t, expected, result) + assert.Equal(t, expected, component.ExternalReferences) } func TestEnrichHomepageWithNilHomepage(t *testing.T) { - component := cdx.Component{} - packageData := packages.Package{Homepage: nil} + component := &cdx.Component{} + packageData := &packages.Package{Homepage: nil} - result := enrichCDXHomepage(component, packageData) + enrichCDXHomepage(component, packageData) - assert.Equal(t, component, result) + assert.Nil(t, component.ExternalReferences) } func TestEnrichHomepageWithNonNullHomepage(t *testing.T) { - component := cdx.Component{} - packageData := packages.Package{Homepage: pointerToString("https://example.com")} + component := &cdx.Component{} + packageData := &packages.Package{Homepage: pointerToString(t, "https://example.com")} - result := enrichCDXHomepage(component, packageData) + enrichCDXHomepage(component, packageData) - expected := cdx.Component{ - ExternalReferences: &[]cdx.ExternalReference{ - {URL: "https://example.com", Type: cdx.ERTypeWebsite}, - }, + expected := &[]cdx.ExternalReference{ + {URL: "https://example.com", Type: cdx.ERTypeWebsite}, } - assert.Equal(t, expected, result) + assert.Equal(t, expected, component.ExternalReferences) } func TestEnrichRegistryURLWithNilRegistryURL(t *testing.T) { - component := cdx.Component{} - packageData := packages.Package{RegistryUrl: nil} + component := &cdx.Component{} + packageData := &packages.Package{RegistryUrl: nil} - result := enrichCDXRegistryURL(component, packageData) + enrichCDXRegistryURL(component, packageData) - assert.Equal(t, component, result) + assert.Nil(t, component.ExternalReferences) } func TestEnrichRegistryURLWithNonNullRegistryURL(t *testing.T) { - component := cdx.Component{} - packageData := packages.Package{RegistryUrl: pointerToString("https://example.com")} + component := &cdx.Component{} + packageData := &packages.Package{RegistryUrl: pointerToString(t, "https://example.com")} - result := enrichCDXRegistryURL(component, packageData) + enrichCDXRegistryURL(component, packageData) - expected := cdx.Component{ - ExternalReferences: &[]cdx.ExternalReference{ - {URL: "https://example.com", Type: cdx.ERTypeDistribution}, - }, + expected := &[]cdx.ExternalReference{ + {URL: "https://example.com", Type: cdx.ERTypeDistribution}, } - assert.Equal(t, expected, result) + assert.Equal(t, expected, component.ExternalReferences) } -func pointerToString(s string) *string { +func pointerToString(t *testing.T, s string) *string { + t.Helper() return &s } func TestEnrichLatestReleasePublishedAt(t *testing.T) { - component := cdx.Component{} - packageData := packages.Package{ + component := &cdx.Component{} + packageData := &packages.Package{ LatestReleasePublishedAt: nil, } - result := enrichCDXLatestReleasePublishedAt(component, packageData) - assert.Equal(t, component, result) + enrichCDXLatestReleasePublishedAt(component, packageData) + + assert.Nil(t, component.Properties) latestReleasePublishedAt := time.Date(2023, time.May, 1, 0, 0, 0, 0, time.UTC) packageData.LatestReleasePublishedAt = &latestReleasePublishedAt expectedTimestamp := latestReleasePublishedAt.UTC().Format(time.RFC3339) - result = enrichCDXLatestReleasePublishedAt(component, packageData) - prop := (*result.Properties)[0] + enrichCDXLatestReleasePublishedAt(component, packageData) + + assert.Len(t, *component.Properties, 1) + prop := (*component.Properties)[0] assert.Equal(t, "ecosystems:latest_release_published_at", prop.Name) assert.Equal(t, expectedTimestamp, prop.Value) } @@ -293,42 +296,38 @@ func TestEnrichLocation(t *testing.T) { assert := assert.New(t) // Test case 1: packageData.RepoMetadata is nil - component := cdx.Component{Name: "test"} - packageData := packages.Package{} - result := enrichCDXLocation(component, packageData) - assert.Equal(component, result) + component := &cdx.Component{Name: "test"} + packageData := &packages.Package{} + enrichCDXLocation(component, packageData) + assert.Nil(component.Properties) // Test case 2: packageData.RepoMetadata is not nil, but "owner_record" is missing - component = cdx.Component{Name: "test"} - packageData = packages.Package{RepoMetadata: &map[string]interface{}{ + component = &cdx.Component{Name: "test"} + packageData = &packages.Package{RepoMetadata: &map[string]interface{}{ "not_owner_record": map[string]interface{}{}, }} - result = enrichCDXLocation(component, packageData) - assert.Equal(component, result) + enrichCDXLocation(component, packageData) + assert.Nil(component.Properties) // Test case 3: "location" field is missing in "owner_record" - component = cdx.Component{Name: "test"} - packageData = packages.Package{RepoMetadata: &map[string]interface{}{ + component = &cdx.Component{Name: "test"} + packageData = &packages.Package{RepoMetadata: &map[string]interface{}{ "owner_record": map[string]interface{}{ "not_location": "test", }, }} - result = enrichCDXLocation(component, packageData) - assert.Equal(component, result) + enrichCDXLocation(component, packageData) + assert.Nil(component.Properties) // Test case 4: "location" field is present in "owner_record" - component = cdx.Component{Name: "test"} - packageData = packages.Package{RepoMetadata: &map[string]interface{}{ + component = &cdx.Component{Name: "test"} + packageData = &packages.Package{RepoMetadata: &map[string]interface{}{ "owner_record": map[string]interface{}{ "location": "test_location", }, }} - expectedComponent := cdx.Component{ - Name: "test", - Properties: &[]cdx.Property{ - {Name: "ecosystems:owner_location", Value: "test_location"}, - }, - } - result = enrichCDXLocation(component, packageData) - assert.Equal(expectedComponent, result) + enrichCDXLocation(component, packageData) + assert.Equal(&[]cdx.Property{ + {Name: "ecosystems:owner_location", Value: "test_location"}, + }, component.Properties) } diff --git a/lib/ecosystems/enrich_spdx_test.go b/lib/ecosystems/enrich_spdx_test.go index 114d4fc..76f7497 100644 --- a/lib/ecosystems/enrich_spdx_test.go +++ b/lib/ecosystems/enrich_spdx_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/jarcoal/httpmock" + "github.com/rs/zerolog" "github.com/spdx/tools-golang/spdx/v2/common" "github.com/spdx/tools-golang/spdx/v2/v2_3" "github.com/stretchr/testify/assert" @@ -60,6 +61,7 @@ func TestEnrichSBOM_SPDX(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} + logger := zerolog.Nop() EnrichSBOM(doc, &logger)