From 27689fea9700ddafec4792dfb4744cf096741351 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 11 Jun 2024 13:22:07 -0400 Subject: [PATCH] add and fix tests from rebase Signed-off-by: Alex Goodman --- cmd/grype/cli/commands/root.go | 9 +- go.mod | 8 +- grype/deprecated.go | 4 +- grype/vex/openvex/implementation.go | 263 ++++++++++++++++++----- grype/vex/openvex/implementation_test.go | 154 +++++++++++++ grype/vex/processor.go | 59 ++--- grype/vex/processor_test.go | 6 +- grype/vulnerability_matcher.go | 23 +- grype/vulnerability_matcher_test.go | 3 +- 9 files changed, 430 insertions(+), 99 deletions(-) create mode 100644 grype/vex/openvex/implementation_test.go diff --git a/cmd/grype/cli/commands/root.go b/cmd/grype/cli/commands/root.go index fc07e43e027..bc4f9ea799c 100644 --- a/cmd/grype/cli/commands/root.go +++ b/cmd/grype/cli/commands/root.go @@ -1,6 +1,7 @@ package commands import ( + "context" "errors" "fmt" "strings" @@ -73,12 +74,12 @@ You can also pipe in Syft JSON directly: Args: validateRootArgs, SilenceUsage: true, SilenceErrors: true, - RunE: func(_ *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { userInput := "" if len(args) > 0 { userInput = args[0] } - return runGrype(app, opts, userInput) + return runGrype(cmd.Context(), app, opts, userInput) }, ValidArgsFunction: dockerImageValidArgsFunction, }, opts) @@ -106,7 +107,7 @@ var ignoreLinuxKernelHeaders = []match.IgnoreRule{ } //nolint:funlen -func runGrype(app clio.Application, opts *options.Grype, userInput string) (errs error) { +func runGrype(ctx context.Context, app clio.Application, opts *options.Grype, userInput string) (errs error) { writer, err := format.MakeScanResultWriter(opts.Outputs, opts.File, format.PresentationConfig{ TemplateFilePath: opts.OutputTemplateFile, ShowSuppressed: opts.ShowSuppressed, @@ -194,7 +195,7 @@ func runGrype(app clio.Application, opts *options.Grype, userInput string) (errs }), } - remainingMatches, ignoredMatches, err := vulnMatcher.FindMatches(packages, pkgContext) + remainingMatches, ignoredMatches, err := vulnMatcher.FindMatches(ctx, packages, pkgContext) if err != nil { if !errors.Is(err, grypeerr.ErrAboveSeverityThreshold) { return err diff --git a/go.mod b/go.mod index 0b16d666950..d515a9a9fb4 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/glebarez/sqlite v1.11.0 github.com/go-test/deep v1.1.0 github.com/google/go-cmp v0.6.0 - github.com/google/go-containerregistry v0.19.1 + github.com/google/go-containerregistry v0.19.1 // indirect github.com/google/uuid v1.6.0 github.com/gookit/color v1.5.4 github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b @@ -61,7 +61,10 @@ require ( gorm.io/gorm v1.25.10 ) -require github.com/openvex/discovery v0.1.0 +require ( + github.com/openvex/discovery v0.1.0 + golang.org/x/sync v0.7.0 +) require ( cloud.google.com/go v0.110.10 // indirect @@ -285,7 +288,6 @@ require ( golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.19.0 // indirect - golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect diff --git a/grype/deprecated.go b/grype/deprecated.go index e9e4d4eb701..ec5ccf27c5e 100644 --- a/grype/deprecated.go +++ b/grype/deprecated.go @@ -1,6 +1,8 @@ package grype import ( + "context" + "github.com/anchore/grype/grype/match" "github.com/anchore/grype/grype/matcher" "github.com/anchore/grype/grype/pkg" @@ -40,7 +42,7 @@ func FindVulnerabilitiesForPackage(store store.Store, d *linux.Release, matchers NormalizeByCVE: false, } - actualResults, _, err := runner.FindMatches(packages, pkg.Context{ + actualResults, _, err := runner.FindMatches(context.Background(), packages, pkg.Context{ Distro: d, }) if err != nil || actualResults == nil { diff --git a/grype/vex/openvex/implementation.go b/grype/vex/openvex/implementation.go index bf9037e4d52..dd8f7b62946 100644 --- a/grype/vex/openvex/implementation.go +++ b/grype/vex/openvex/implementation.go @@ -1,24 +1,34 @@ package openvex import ( + "context" "errors" "fmt" - "github.com/anchore/grype/grype/event" - "github.com/anchore/grype/internal/bus" + "runtime" + "sort" + "strings" + "sync/atomic" + "github.com/openvex/discovery/pkg/discovery" "github.com/openvex/discovery/pkg/oci" openvex "github.com/openvex/go-vex/pkg/vex" "github.com/scylladb/go-set/strset" "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" - "strings" + "golang.org/x/sync/errgroup" + "github.com/anchore/grype/grype/event" "github.com/anchore/grype/grype/match" "github.com/anchore/grype/grype/pkg" + "github.com/anchore/grype/internal/bus" + "github.com/anchore/grype/internal/log" "github.com/anchore/syft/syft/source" ) -type Processor struct{} +type Processor struct { + // we always merge all discovered / read vex documents into a single document + documents *openvex.VEX +} func New() *Processor { return &Processor{} @@ -49,43 +59,67 @@ var ignoreStatuses = []openvex.Status{ } // ReadVexDocuments reads and merges VEX documents -func (ovm *Processor) ReadVexDocuments(docs []string) (interface{}, error) { - if len(docs) == 0 { - return &openvex.VEX{}, nil +func (ovm *Processor) ReadVexDocuments(docRefs []string) error { + if len(docRefs) == 0 { + return nil } - // Combine all VEX documents into a single VEX document - vexdata, err := openvex.MergeFiles(docs) + // combine all VEX documents into a single VEX document + vexdata, err := openvex.MergeFiles(docRefs) if err != nil { - return nil, fmt.Errorf("merging vex documents: %w", err) + return fmt.Errorf("merging vex documents: %w", err) } - return vexdata, nil + ovm.documents = vexdata + + return nil } // productIdentifiersFromContext reads the package context and returns software // identifiers identifying the scanned image. -func productIdentifiersFromContext(pkgContext *pkg.Context) ([]string, error) { +func productIdentifiersFromContext(pkgContext pkg.Context) ([]string, error) { + if pkgContext.Source == nil || pkgContext.Source.Metadata == nil { + return nil, nil + } + + var ret []string + switch v := pkgContext.Source.Metadata.(type) { case source.ImageMetadata: - // Call the OpenVEX OCI module to generate the identifiers from the + // call the OpenVEX OCI module to generate the identifiers from the // image reference specified by the user. - bundle, err := oci.GenerateReferenceIdentifiers(v.UserInput, v.OS, v.Architecture) - if err != nil { - return nil, fmt.Errorf("generating identifiers from image reference: %w", err) + refs := []string{v.UserInput} + refs = append(refs, v.RepoDigests...) + + set := strset.New() + for _, ref := range refs { + bundle, err := oci.GenerateReferenceIdentifiers(ref, v.OS, v.Architecture) + if err != nil { + log.WithFields("error", err).Trace("unable to generate OCI identifiers from image reference") + continue + } + set.Add(bundle.ToStringSlice()...) } - return bundle.ToStringSlice(), nil + ret = set.List() + default: // Fail as we only support VEXing container images for now return nil, errors.New("source type not supported for VEX") } + + sort.Strings(ret) + return ret, nil } // subcomponentIdentifiersFromMatch returns the list of identifiers from the // package where grype did the match. func subcomponentIdentifiersFromMatch(m *match.Match) []string { - ret := []string{} + if m == nil { + return nil + } + + var ret []string if m.Package.PURL != "" { ret = append(ret, m.Package.PURL) } @@ -102,18 +136,19 @@ func subcomponentIdentifiersFromMatch(m *match.Match) []string { // FilterMatches takes a set of scanning results and moves any results marked in // the VEX data as fixed or not_affected to the ignored list. func (ovm *Processor) FilterMatches( - docRaw interface{}, ignoreRules []match.IgnoreRule, pkgContext *pkg.Context, matches *match.Matches, ignoredMatches []match.IgnoredMatch, + ignoreRules []match.IgnoreRule, pkgContext pkg.Context, matches *match.Matches, ignoredMatches []match.IgnoredMatch, ) (*match.Matches, []match.IgnoredMatch, error) { - doc, ok := docRaw.(*openvex.VEX) - if !ok { - return nil, nil, errors.New("unable to cast vex document as openvex") + if ovm.documents == nil { + return matches, ignoredMatches, nil } + doc := ovm.documents + remainingMatches := match.NewMatches() products, err := productIdentifiersFromContext(pkgContext) if err != nil { - return nil, nil, fmt.Errorf("reading product identifiers from context: %w", err) + return nil, nil, err } // TODO(alex): should we apply the vex ignore rules to the already ignored matches? @@ -220,13 +255,14 @@ func matchingRule(ignoreRules []match.IgnoreRule, m match.Match, statement *open // about an affected VEX product is found on loaded VEX documents. Matches // are moved from the ignore list or synthesized when no previous data is found. func (ovm *Processor) AugmentMatches( - docRaw interface{}, ignoreRules []match.IgnoreRule, pkgContext *pkg.Context, remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, + ignoreRules []match.IgnoreRule, pkgContext pkg.Context, remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, ) (*match.Matches, []match.IgnoredMatch, error) { - doc, ok := docRaw.(*openvex.VEX) - if !ok { - return nil, nil, errors.New("unable to cast vex document as openvex") + if ovm.documents == nil { + return remainingMatches, ignoredMatches, nil } + doc := ovm.documents + additionalIgnoredMatches := []match.IgnoredMatch{} products, err := productIdentifiersFromContext(pkgContext) @@ -290,63 +326,174 @@ func (ovm *Processor) AugmentMatches( // DiscoverVexDocuments uses the OpenVEX discovery module to look for vex data // associated to the scanned object. If any data is found, the data will be // added to the existing vex data -func (ovm *Processor) DiscoverVexDocuments(pkgContext *pkg.Context, rawVexData interface{}) (interface{}, error) { +func (ovm *Processor) DiscoverVexDocuments(ctx context.Context, pkgContext pkg.Context) error { // Extract the identifiers from the package context identifiers, err := productIdentifiersFromContext(pkgContext) if err != nil { - return nil, fmt.Errorf("extracting identifiers from context") + return fmt.Errorf("extracting identifiers from context") } - prog, stage := trackVexDiscovery(identifiers) - - allDocs := []*openvex.VEX{} + searchTargets := searchableIdentifiers(identifiers) + log.WithFields("identifiers", len(identifiers), "usable", searchTargets).Debug("searching remotely for vex documents") - // If we already have some vex data, add it - if _, ok := rawVexData.(*openvex.VEX); ok { - allDocs = []*openvex.VEX{rawVexData.(*openvex.VEX)} + discoveredDocs, err := findVexDocuments(ctx, searchTargets) + if err != nil { + return err } - agent := discovery.NewAgent() - ids := strset.New() + var allDocs []*openvex.VEX + for _, doc := range discoveredDocs { + allDocs = append(allDocs, doc) + } - for _, i := range identifiers { - if !strings.HasPrefix(i, "pkg:") { - continue - } + if len(allDocs) == 0 { + return nil + } - stage.Set(fmt.Sprintf("searching %s", i)) + vexdata, err := openvex.MergeDocuments(allDocs) + if err != nil { + return fmt.Errorf("unable to merge discovered vex documents: %w", err) + } - discoveredDocs, err := agent.ProbePurl(i) + if ovm.documents != nil { + vexdata, err := openvex.MergeDocuments([]*openvex.VEX{ovm.documents, vexdata}) if err != nil { - prog.SetError(err) - return nil, fmt.Errorf("probing package url or vex data: %w", err) + return fmt.Errorf("unable to merge existing vex documents with discovered documents: %w", err) } + ovm.documents = vexdata + } else { + ovm.documents = vexdata + } + + return nil +} + +func findVexDocuments(ctx context.Context, identifiers []string) (map[string]*openvex.VEX, error) { + allDiscoveredDocs := make(chan *openvex.VEX) + + prog, stage := trackVexDiscovery(identifiers) + defer prog.SetCompleted() - // prune any existing documents so they are not applied multiple times - for j, doc := range discoveredDocs { - if ids.Has(doc.ID) { - discoveredDocs = append(discoveredDocs[:j], discoveredDocs[j+1:]...) + agent := discovery.NewAgent() + + grp, ctx := errgroup.WithContext(ctx) + + identifierQueue := produceSearchableIdentifiers(ctx, grp, identifiers) + + workers := int32(maxParallelism()) + for workerNum := int32(0); workerNum < workers; workerNum++ { + grp.Go(func() error { + defer func() { + if atomic.AddInt32(&workers, -1) == 0 { + close(allDiscoveredDocs) + } + }() + + for i := range identifierQueue { + stage.Set(fmt.Sprintf("searching %s", i)) + log.WithFields("identifier", i).Trace("searching remotely for vex documents") + + discoveredDocs, err := agent.ProbePurl(i) + if err != nil { + prog.SetError(err) + return fmt.Errorf("probing package url or vex data: %w", err) + } + + prog.Add(1) + + if len(discoveredDocs) > 0 { + log.WithFields("documents", len(discoveredDocs), "identifier", i).Debug("discovered vex documents") + } + + for _, doc := range discoveredDocs { + select { + case <-ctx.Done(): + return ctx.Err() + case allDiscoveredDocs <- doc: + } + } } - ids.Add(doc.ID) + return nil + }) + } + + return reduceVexDocuments(grp, stage, allDiscoveredDocs) +} + +func reduceVexDocuments(grp *errgroup.Group, stage *progress.AtomicStage, allDiscoveredDocs <-chan *openvex.VEX) (map[string]*openvex.VEX, error) { + finalDiscoveredDocs := make(map[string]*openvex.VEX) + grp.Go(func() error { + for doc := range allDiscoveredDocs { + if _, ok := finalDiscoveredDocs[doc.ID]; ok { + continue + } + finalDiscoveredDocs[doc.ID] = doc } + return nil + }) - allDocs = append(allDocs, discoveredDocs...) + if err := grp.Wait(); err != nil { + return nil, fmt.Errorf("searching remotely for vex documents: %w", err) } - stage.Set(fmt.Sprintf("%d documents", len(allDocs))) - prog.SetCompleted() + if len(finalDiscoveredDocs) > 0 { + log.WithFields("documents", len(finalDiscoveredDocs)).Debug("total vex documents discovered remotely") + } else { + log.Debug("no vex documents discovered remotely") + } - vexdata, err := openvex.MergeDocuments(allDocs) - if err != nil { - return nil, fmt.Errorf("merging vex documents: %w", err) + stage.Set(fmt.Sprintf("%d documents discovered", len(finalDiscoveredDocs))) + + return finalDiscoveredDocs, nil +} + +func maxParallelism() int { + // from docs: "If n < 1, it does not change the current setting." + maxProcs := runtime.GOMAXPROCS(0) + numCPU := runtime.NumCPU() + if maxProcs < numCPU { + return maxProcs } + return numCPU +} - return vexdata, nil +func produceSearchableIdentifiers(ctx context.Context, g *errgroup.Group, identifiers []string) chan string { + ids := make(chan string) + + g.Go(func() error { + defer close(ids) + for _, i := range identifiers { + i := i + + if !strings.HasPrefix(i, "pkg:") { + continue + } + select { + case <-ctx.Done(): + return ctx.Err() + case ids <- i: + } + } + + return nil + }) + return ids +} + +func searchableIdentifiers(identifiers []string) []string { + var ids []string + for _, i := range identifiers { + if !strings.HasPrefix(i, "pkg:") { + continue + } + ids = append(ids, i) + } + return ids } func trackVexDiscovery(identifiers []string) (*progress.Manual, *progress.AtomicStage) { stage := progress.NewAtomicStage("") - prog := progress.NewManual(-1) + prog := progress.NewManual(int64(len(identifiers))) bus.Publish(partybus.Event{ Type: event.VexDocumentDiscoveryStarted, diff --git a/grype/vex/openvex/implementation_test.go b/grype/vex/openvex/implementation_test.go new file mode 100644 index 00000000000..549d7764b33 --- /dev/null +++ b/grype/vex/openvex/implementation_test.go @@ -0,0 +1,154 @@ +package openvex + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/grype/grype/match" + "github.com/anchore/grype/grype/pkg" + "github.com/anchore/syft/syft/source" +) + +func Test_productIdentifiersFromContext(t *testing.T) { + + tests := []struct { + name string + pkgContext pkg.Context + want []string + wantErr require.ErrorAssertionFunc + }{ + { + name: "no source", + pkgContext: pkg.Context{}, + }, + { + name: "source with no metadata", + pkgContext: pkg.Context{ + Source: &source.Description{}, + }, + }, + { + name: "source with empty image metadata", + pkgContext: pkg.Context{ + Source: &source.Description{ + Metadata: source.ImageMetadata{}, + }, + }, + }, + { + name: "source with unusable image input", + pkgContext: pkg.Context{ + Source: &source.Description{ + Metadata: source.ImageMetadata{ + UserInput: "some-image:tag", + RepoDigests: []string{ + "some-other-image:tag", // we shouldn't expect this, but should be resilient to it + }, + }, + }, + }, + }, + { + name: "source with usable image input", + pkgContext: pkg.Context{ + Source: &source.Description{ + Metadata: source.ImageMetadata{ + UserInput: "some-image:tag@sha256:124c7d2707904eea7431fffe91522a01e5a861a624ee31d03372cc1d138a3126", + RepoDigests: []string{ + "some-other-image@sha256:a01fe91372c2a3126624ee31d0c1de5a861ad2707904eea7431f523f138124c7", + }, + }, + }, + }, + want: []string{ + "124c7d2707904eea7431fffe91522a01e5a861a624ee31d03372cc1d138a3126", + "pkg:oci/some-image@sha256%3A124c7d2707904eea7431fffe91522a01e5a861a624ee31d03372cc1d138a3126?repository_url=index.docker.io%2Flibrary", + "a01fe91372c2a3126624ee31d0c1de5a861ad2707904eea7431f523f138124c7", + "pkg:oci/some-other-image@sha256%3Aa01fe91372c2a3126624ee31d0c1de5a861ad2707904eea7431f523f138124c7?repository_url=index.docker.io%2Flibrary", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + got, err := productIdentifiersFromContext(tt.pkgContext) + tt.wantErr(t, err) + assert.ElementsMatch(t, tt.want, got) + }) + } +} + +func Test_searchableIdentifiers(t *testing.T) { + type args struct { + } + tests := []struct { + name string + identifiers []string + want []string + }{ + { + name: "no identifiers", + }, + { + name: "only keep pacakge urls", + identifiers: []string{ + "pkg:deb/debian@buster", + "pkg:deb/debian@buster?repository_url=http://deb.debian.org/debian", + "pkg:oci/some-other-image@sha256%3Aa01fe91372c2a3126624ee31d0c1de5a861ad2707904eea7431f523f138124c7?repository_url=index.docker.io%2Flibrary", + "124c7d2707904eea7431fffe91522a01e5a861a624ee31d03372cc1d138a3126", + "somethingelse", + }, + want: []string{ + "pkg:deb/debian@buster", + "pkg:deb/debian@buster?repository_url=http://deb.debian.org/debian", + "pkg:oci/some-other-image@sha256%3Aa01fe91372c2a3126624ee31d0c1de5a861ad2707904eea7431f523f138124c7?repository_url=index.docker.io%2Flibrary", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, searchableIdentifiers(tt.identifiers)) + }) + } +} + +func Test_subcomponentIdentifiersFromMatch(t *testing.T) { + + tests := []struct { + name string + match *match.Match + want []string + }{ + { + name: "no match", + }, + { + name: "no purl", + match: &match.Match{ + Package: pkg.Package{ + PURL: "", + }, + }, + }, + { + name: "keep purl", + match: &match.Match{ + Package: pkg.Package{ + PURL: "pkg:deb/debian@buster?repository_url=http://deb.debian.org/debian", + }, + }, + want: []string{ + "pkg:deb/debian@buster?repository_url=http://deb.debian.org/debian", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, subcomponentIdentifiersFromMatch(tt.match)) + }) + } +} diff --git a/grype/vex/processor.go b/grype/vex/processor.go index 477d34fe9ce..981ddbac2d9 100644 --- a/grype/vex/processor.go +++ b/grype/vex/processor.go @@ -1,6 +1,7 @@ package vex import ( + "context" "fmt" gopenvex "github.com/openvex/go-vex/pkg/vex" @@ -28,22 +29,22 @@ type vexProcessorImplementation interface { // ReadVexDocuments takes a list of vex filenames and returns a single // value representing the VEX information in the underlying implementation's // format. Returns an error if the files cannot be processed. - ReadVexDocuments(docs []string) (interface{}, error) + ReadVexDocuments(docs []string) error // DiscoverVexDocuments calls asks vex driver to find documents associated // to the scanned object. Autodiscovered documents are added to any that // are specified in the command line - DiscoverVexDocuments(*pkg.Context, interface{}) (interface{}, error) + DiscoverVexDocuments(context.Context, pkg.Context) error // FilterMatches matches receives the underlying VEX implementation VEX data and // the scanning context and matching results and filters the fixed and // not_affected results,moving them to the list of ignored matches. - FilterMatches(interface{}, []match.IgnoreRule, *pkg.Context, *match.Matches, []match.IgnoredMatch) (*match.Matches, []match.IgnoredMatch, error) + FilterMatches([]match.IgnoreRule, pkg.Context, *match.Matches, []match.IgnoredMatch) (*match.Matches, []match.IgnoredMatch, error) // AugmentMatches reads known affected VEX products from loaded documents and // adds new results to the scanner results when the product is marked as // affected in the VEX data. - AugmentMatches(interface{}, []match.IgnoreRule, *pkg.Context, *match.Matches, []match.IgnoredMatch) (*match.Matches, []match.IgnoredMatch, error) + AugmentMatches([]match.IgnoreRule, pkg.Context, *match.Matches, []match.IgnoredMatch) (*match.Matches, []match.IgnoredMatch, error) } // getVexImplementation this function returns the vex processor implementation @@ -73,46 +74,56 @@ type ProcessorOptions struct { IgnoreRules []match.IgnoreRule } -// ApplyVEX receives the results from a scan run and applies any VEX information -// in the files specified in the grype invocation. Any filtered results will -// be moved to the ignored matches slice. -func (vm *Processor) ApplyVEX(pkgContext *pkg.Context, remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch) (*match.Matches, []match.IgnoredMatch, error) { - var err error - - // If no VEX documents are loaded, just pass through the matches, effectively NOOP - if len(vm.Options.Documents) == 0 && !vm.Options.Autodiscover { - return remainingMatches, ignoredMatches, nil +func (vm *Processor) LoadVEXDocuments(ctx context.Context, pkgContext pkg.Context) error { + if vm == nil { + return nil } - // Read VEX data from all passed documents - rawVexData, err := vm.impl.ReadVexDocuments(vm.Options.Documents) - if err != nil { - return nil, nil, fmt.Errorf("parsing vex document: %w", err) + // read VEX data from all passed documents + if len(vm.Options.Documents) > 0 { + err := vm.impl.ReadVexDocuments(vm.Options.Documents) + if err != nil { + return fmt.Errorf("parsing vex document: %w", err) + } } - // If VEX autodiscover is enabled, run call the implementation's discovery + // if VEX autodiscover is enabled, run call the implementation's discovery // function to augment the known VEX data if vm.Options.Autodiscover { - rawVexData, err = vm.impl.DiscoverVexDocuments(pkgContext, rawVexData) + err := vm.impl.DiscoverVexDocuments(ctx, pkgContext) if err != nil { - return nil, nil, fmt.Errorf("probing for VEX data: %w", err) + return fmt.Errorf("probing for VEX data: %w", err) } } + return nil +} + +// ApplyVEX receives the results from a scan run and applies any VEX information +// in the files specified in the grype invocation. Any filtered results will +// be moved to the ignored matches slice. +func (vm *Processor) ApplyVEX(pkgContext pkg.Context, remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch) (*match.Matches, []match.IgnoredMatch, error) { + var err error + + // If no VEX documents are loaded, just pass through the matches, effectively NOOP + if len(vm.Options.Documents) == 0 && !vm.Options.Autodiscover { + return remainingMatches, ignoredMatches, nil + } + vexRules := extractVexRules(vm.Options.IgnoreRules) remainingMatches, ignoredMatches, err = vm.impl.FilterMatches( - rawVexData, vexRules, pkgContext, remainingMatches, ignoredMatches, + vexRules, pkgContext, remainingMatches, ignoredMatches, ) if err != nil { - return nil, nil, fmt.Errorf("checking matches against VEX data: %w", err) + return nil, nil, fmt.Errorf("unable to filter matches against VEX data: %w", err) } remainingMatches, ignoredMatches, err = vm.impl.AugmentMatches( - rawVexData, vexRules, pkgContext, remainingMatches, ignoredMatches, + vexRules, pkgContext, remainingMatches, ignoredMatches, ) if err != nil { - return nil, nil, fmt.Errorf("checking matches to augment from VEX data: %w", err) + return nil, nil, fmt.Errorf("unable to augment matches with VEX data: %w", err) } return remainingMatches, ignoredMatches, nil diff --git a/grype/vex/processor_test.go b/grype/vex/processor_test.go index a759792f2fb..bff58c3696d 100644 --- a/grype/vex/processor_test.go +++ b/grype/vex/processor_test.go @@ -1,6 +1,7 @@ package vex import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -14,7 +15,7 @@ import ( ) func TestProcessor_ApplyVEX(t *testing.T) { - pkgContext := &pkg.Context{ + pkgContext := pkg.Context{ Source: &source.Description{ Name: "alpine", Version: "3.17", @@ -96,7 +97,7 @@ func TestProcessor_ApplyVEX(t *testing.T) { } type args struct { - pkgContext *pkg.Context + pkgContext pkg.Context matches *match.Matches ignoredMatches []match.IgnoredMatch } @@ -298,6 +299,7 @@ func TestProcessor_ApplyVEX(t *testing.T) { } p := NewProcessor(tt.options) + require.NoError(t, p.LoadVEXDocuments(context.TODO(), pkgContext)) actualMatches, actualIgnoredMatches, err := p.ApplyVEX(tt.args.pkgContext, tt.args.matches, tt.args.ignoredMatches) tt.wantErr(t, err) if err != nil { diff --git a/grype/vulnerability_matcher.go b/grype/vulnerability_matcher.go index 0ce19261973..540550a53b4 100644 --- a/grype/vulnerability_matcher.go +++ b/grype/vulnerability_matcher.go @@ -1,12 +1,14 @@ package grype import ( + "context" "fmt" "slices" "strings" "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" + "golang.org/x/sync/errgroup" grypeDb "github.com/anchore/grype/grype/db/v5" "github.com/anchore/grype/grype/distro" @@ -62,7 +64,7 @@ func (m *VulnerabilityMatcher) WithIgnoreRules(ignoreRules []match.IgnoreRule) * return m } -func (m *VulnerabilityMatcher) FindMatches(pkgs []pkg.Package, context pkg.Context) (remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, err error) { +func (m *VulnerabilityMatcher) FindMatches(ctx context.Context, pkgs []pkg.Package, pkgContext pkg.Context) (remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, err error) { progressMonitor := trackMatcher(len(pkgs)) defer func() { @@ -73,12 +75,21 @@ func (m *VulnerabilityMatcher) FindMatches(pkgs []pkg.Package, context pkg.Conte } }() - remainingMatches, ignoredMatches, err = m.findDBMatches(pkgs, context, progressMonitor) + vexGroup, ctx := errgroup.WithContext(ctx) + vexGroup.Go(func() error { + return m.VexProcessor.LoadVEXDocuments(ctx, pkgContext) + }) + + remainingMatches, ignoredMatches, err = m.findDBMatches(pkgs, pkgContext, progressMonitor) if err != nil { return remainingMatches, ignoredMatches, err } - remainingMatches, ignoredMatches, err = m.findVEXMatches(context, remainingMatches, ignoredMatches, progressMonitor) + if err := vexGroup.Wait(); err != nil { + return remainingMatches, ignoredMatches, fmt.Errorf("unable to load VEX documents: %w", err) + } + + remainingMatches, ignoredMatches, err = m.pairVEXWithMatches(pkgContext, remainingMatches, ignoredMatches, progressMonitor) if err != nil { err = fmt.Errorf("unable to find matches against VEX sources: %w", err) return remainingMatches, ignoredMatches, err @@ -292,16 +303,16 @@ func filterMatchesUsingDistroFalsePositives(ms []match.Match, falsePositivesByLo return result } -func (m *VulnerabilityMatcher) findVEXMatches(context pkg.Context, remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, progressMonitor *monitorWriter) (*match.Matches, []match.IgnoredMatch, error) { +func (m *VulnerabilityMatcher) pairVEXWithMatches(pkgContext pkg.Context, remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, progressMonitor *monitorWriter) (*match.Matches, []match.IgnoredMatch, error) { if m.VexProcessor == nil { log.Trace("no VEX documents provided, skipping VEX matching") return remainingMatches, ignoredMatches, nil } log.Trace("finding matches against available VEX documents") - matchesAfterVex, ignoredMatchesAfterVex, err := m.VexProcessor.ApplyVEX(&context, remainingMatches, ignoredMatches) + matchesAfterVex, ignoredMatchesAfterVex, err := m.VexProcessor.ApplyVEX(pkgContext, remainingMatches, ignoredMatches) if err != nil { - return nil, nil, fmt.Errorf("unable to find matches against VEX documents: %w", err) + return nil, nil, err } diffMatches := matchesAfterVex.Diff(*remainingMatches) diff --git a/grype/vulnerability_matcher_test.go b/grype/vulnerability_matcher_test.go index 9f1d7373647..16f41708491 100644 --- a/grype/vulnerability_matcher_test.go +++ b/grype/vulnerability_matcher_test.go @@ -1,6 +1,7 @@ package grype import ( + "context" "testing" "github.com/google/go-cmp/cmp" @@ -1065,7 +1066,7 @@ func TestVulnerabilityMatcher_FindMatches(t *testing.T) { bus.Set(listener) defer bus.Set(nil) - actualMatches, actualIgnoreMatches, err := m.FindMatches(tt.args.pkgs, tt.args.context) + actualMatches, actualIgnoreMatches, err := m.FindMatches(context.TODO(), tt.args.pkgs, tt.args.context) if tt.wantErr != nil { require.ErrorIs(t, err, tt.wantErr) return