From 3d3f0e7a309819b476660643a730f64fb8c10121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Fri, 6 Oct 2023 17:21:04 +0200 Subject: [PATCH] add fingerprinting to scan command --- internal/cmd/files/fingerprint/fingerprint.go | 7 ++- .../cmd/files/fingerprint/fingerprint_test.go | 2 +- internal/cmd/root/root_test.go | 2 +- internal/cmd/scan/scan.go | 5 +- internal/file/fingerprint.go | 4 ++ internal/scan/scanner.go | 22 ++++++-- internal/scan/scanner_test.go | 52 ++++++++++++++++--- internal/wire/cli_container.go | 1 + internal/wire/cli_container_test.go | 1 + 9 files changed, 78 insertions(+), 18 deletions(-) diff --git a/internal/cmd/files/fingerprint/fingerprint.go b/internal/cmd/files/fingerprint/fingerprint.go index 3e26fc90..253d66ea 100644 --- a/internal/cmd/files/fingerprint/fingerprint.go +++ b/internal/cmd/files/fingerprint/fingerprint.go @@ -12,14 +12,13 @@ import ( var exclusions = file.DefaultExclusionsFingerprint() const ( - ExclusionFlag = "exclusion-fingerprint" - OutputFileName = ".debricked.fingerprints.wfp" + ExclusionFlag = "exclusion-fingerprint" ) func NewFingerprintCmd(fingerprinter file.IFingerprint) *cobra.Command { cmd := &cobra.Command{ Use: "fingerprint [path]", - Short: "Fingerprint files for identification in a given path and writes it to " + OutputFileName, + Short: "Fingerprint files for identification in a given path and writes it to " + file.OutputFileNameFingerprints, Long: `Fingerprint files for identification in a given path. This hashes all files and matches them against the Debricked knowledge base.`, PreRun: func(cmd *cobra.Command, _ []string) { @@ -65,7 +64,7 @@ func RunE(f file.IFingerprint) func(_ *cobra.Command, args []string) error { return err } - output.ToFile(OutputFileName) + output.ToFile(file.OutputFileNameFingerprints) return nil } diff --git a/internal/cmd/files/fingerprint/fingerprint_test.go b/internal/cmd/files/fingerprint/fingerprint_test.go index d892a06d..f5844ae9 100644 --- a/internal/cmd/files/fingerprint/fingerprint_test.go +++ b/internal/cmd/files/fingerprint/fingerprint_test.go @@ -44,7 +44,7 @@ func TestNewFingerprintCmd(t *testing.T) { func TestRunE(t *testing.T) { defer func() { - os.Remove(OutputFileName) + os.Remove(file.OutputFileNameFingerprints) }() fingerprintMock := testdata.NewFingerprintMock() runE := RunE(fingerprintMock) diff --git a/internal/cmd/root/root_test.go b/internal/cmd/root/root_test.go index a69d742b..2d02f16a 100644 --- a/internal/cmd/root/root_test.go +++ b/internal/cmd/root/root_test.go @@ -31,7 +31,7 @@ func TestNewRootCmd(t *testing.T) { } } assert.Truef(t, match, "failed to assert that flag was present: "+AccessTokenFlag) - assert.Len(t, viperKeys, 13) + assert.Len(t, viperKeys, 14) } func TestPreRun(t *testing.T) { diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index cc5e7926..5db92ce8 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -20,6 +20,7 @@ var repositoryUrl string var integrationName string var exclusions = file.DefaultExclusions() var noResolve bool +var noFingerprint bool var passOnDowntime bool const ( @@ -31,6 +32,7 @@ const ( IntegrationFlag = "integration" ExclusionFlag = "exclusion" NoResolveFlag = "no-resolve" + NoFingerprintFlag = "no-fingerprint" PassOnTimeOut = "pass-on-timeout" ) @@ -82,7 +84,7 @@ $ debricked scan . `+exampleFlags) cmd.Flags().BoolVarP(&passOnDowntime, PassOnTimeOut, "p", false, "pass scan if there is a service access timeout") cmd.Flags().BoolVar(&noResolve, NoResolveFlag, false, `disables resolution of manifest files that lack lock files. Resolving manifest files enables more accurate dependency scanning since the whole dependency tree will be analysed. For example, if there is a "go.mod" in the target path, its dependencies are going to get resolved onto a lock file, and latter scanned.`) - + cmd.Flags().BoolVar(&noFingerprint, NoFingerprintFlag, false, "disables fingerprinting for undeclared component identification. Can be run as a standalone command with more granular options.") viper.MustBindEnv(RepositoryFlag) viper.MustBindEnv(CommitFlag) viper.MustBindEnv(BranchFlag) @@ -103,6 +105,7 @@ func RunE(s *scan.IScanner) func(_ *cobra.Command, args []string) error { options := scan.DebrickedOptions{ Path: path, Resolve: !viper.GetBool(NoResolveFlag), + Fingerprint: !viper.GetBool(NoFingerprintFlag), Exclusions: viper.GetStringSlice(ExclusionFlag), RepositoryName: viper.GetString(RepositoryFlag), CommitName: viper.GetString(CommitFlag), diff --git a/internal/file/fingerprint.go b/internal/file/fingerprint.go index ca8a815a..92affc2b 100644 --- a/internal/file/fingerprint.go +++ b/internal/file/fingerprint.go @@ -37,6 +37,10 @@ var ECLUDED_FILES = []string{ "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile", } +const ( + OutputFileNameFingerprints = ".debricked.fingerprints.wfp" +) + func isExcludedFile(filename string) bool { filenameLower := strings.ToLower(filename) diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index 48ac90d7..91f2894e 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -29,16 +29,18 @@ type IScanner interface { type IOptions interface{} type DebrickedScanner struct { - client *client.IDebClient - finder file.IFinder - uploader *upload.IUploader - ciService ci.IService - resolver resolution.IResolver + client *client.IDebClient + finder file.IFinder + uploader *upload.IUploader + ciService ci.IService + resolver resolution.IResolver + fingerprint file.IFingerprint } type DebrickedOptions struct { Path string Resolve bool + Fingerprint bool Exclusions []string RepositoryName string CommitName string @@ -55,6 +57,7 @@ func NewDebrickedScanner( uploader upload.IUploader, ciService ci.IService, resolver resolution.IResolver, + fingerprint file.IFingerprint, ) *DebrickedScanner { return &DebrickedScanner{ c, @@ -62,6 +65,7 @@ func NewDebrickedScanner( &uploader, ciService, resolver, + fingerprint, } } @@ -125,6 +129,14 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g } } + if options.Fingerprint { + fingerprints, err := dScanner.fingerprint.FingerprintFiles(options.Path, file.DefaultExclusionsFingerprint()) + if err != nil { + return nil, err + } + fingerprints.ToFile(file.OutputFileNameFingerprints) + } + fileGroups, err := dScanner.finder.GetGroups(options.Path, options.Exclusions, false, file.StrictAll) if err != nil { return nil, err diff --git a/internal/scan/scanner_test.go b/internal/scan/scanner_test.go index 4564e778..2db82a22 100644 --- a/internal/scan/scanner_test.go +++ b/internal/scan/scanner_test.go @@ -52,7 +52,8 @@ func TestNewDebrickedScanner(t *testing.T) { var finder file.IFinder var uploader upload.IUploader var resolver resolution.IResolver - s := NewDebrickedScanner(&debClient, finder, uploader, cis, resolver) + var fingerprint file.IFingerprint + s := NewDebrickedScanner(&debClient, finder, uploader, cis, resolver, fingerprint) assert.NotNil(t, s) } @@ -120,7 +121,7 @@ func TestScan(t *testing.T) { func TestScanFailingMetaObject(t *testing.T) { var debClient client.IDebClient = testdata.NewDebClientMock() - scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil) + scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil, nil) cwd, _ := os.Getwd() path := testdataNpm opts := DebrickedOptions{ @@ -167,7 +168,7 @@ func TestScanFailingNoFiles(t *testing.T) { func TestScanBadOpts(t *testing.T) { var c client.IDebClient - scanner := NewDebrickedScanner(&c, nil, nil, nil, nil) + scanner := NewDebrickedScanner(&c, nil, nil, nil, nil, nil) var opts IOptions err := scanner.Scan(opts) @@ -226,7 +227,7 @@ func TestScanEmptyResult(t *testing.T) { func TestScanInCiWithPathSet(t *testing.T) { var debClient client.IDebClient = testdata.NewDebClientMock() - scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil) + scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil, nil) cwd, _ := os.Getwd() defer resetWd(t, cwd) path := testdataNpm @@ -508,7 +509,7 @@ func TestScanServiceDowntime(t *testing.T) { var ciService ci.IService = ci.NewService(nil) - scanner := NewDebrickedScanner(&debClient, finder, nil, ciService, nil) + scanner := NewDebrickedScanner(&debClient, finder, nil, ciService, nil, nil) path := testdataNpm repositoryName := path @@ -606,7 +607,7 @@ func makeScanner(clientMock *testdata.DebClientMock, resolverMock *resolveTestda var cis ci.IService = ci.NewService(nil) - return NewDebrickedScanner(&debClient, finder, uploader, cis, resolverMock) + return NewDebrickedScanner(&debClient, finder, uploader, cis, resolverMock, nil) } func cleanUpResolution(t *testing.T, resolverMock resolveTestdata.ResolverMock) { @@ -615,3 +616,42 @@ func cleanUpResolution(t *testing.T, resolverMock resolveTestdata.ResolverMock) t.Error(err) } } + +func TestScanWithFingerprint(t *testing.T) { + clientMock := testdata.NewDebClientMock() + addMockedFormatsResponse(clientMock, "yarn\\.lock") + addMockedFileUploadResponse(clientMock) + addMockedFinishResponse(clientMock, http.StatusNoContent) + addMockedStatusResponse(clientMock, http.StatusOK, 100) + + resolverMock := resolveTestdata.ResolverMock{} + resolverMock.SetFiles([]string{"yarn.lock"}) + + scanner := makeScanner(clientMock, &resolverMock) + scanner.fingerprint = file.NewFingerprinter() + + cwd, _ := os.Getwd() + defer resetWd(t, cwd) + // Clean up resolution must be done before wd reset, otherwise files cannot be deleted + defer cleanUpResolution(t, resolverMock) + + path := testdataNpm + repositoryName := path + commitName := "testdata/npm-commit" + opts := DebrickedOptions{ + Path: path, + Resolve: true, + Fingerprint: true, + Exclusions: nil, + RepositoryName: repositoryName, + CommitName: commitName, + BranchName: "", + CommitAuthor: "", + RepositoryUrl: "", + IntegrationName: "", + } + err := scanner.Scan(opts) + assert.NoError(t, err) + cwd, _ = os.Getwd() + assert.Contains(t, cwd, path) +} diff --git a/internal/wire/cli_container.go b/internal/wire/cli_container.go index 642c2ba2..66f45d7e 100644 --- a/internal/wire/cli_container.go +++ b/internal/wire/cli_container.go @@ -75,6 +75,7 @@ func (cc *CliContainer) wire() error { cc.uploader, cc.ciService, cc.resolver, + cc.fingerprinter, ) cc.licenseReporter = licenseReport.Reporter{DebClient: cc.debClient} diff --git a/internal/wire/cli_container_test.go b/internal/wire/cli_container_test.go index 536716fa..dd54124c 100644 --- a/internal/wire/cli_container_test.go +++ b/internal/wire/cli_container_test.go @@ -38,4 +38,5 @@ func assertCliContainer(t *testing.T, cc *CliContainer) { assert.NotNil(t, cc.Resolver()) assert.NotNil(t, cc.LicenseReporter()) assert.NotNil(t, cc.VulnerabilityReporter()) + assert.NotNil(t, cc.Fingerprinter()) }