From bd2bb6ebfa140f4d85e0b34c18e635cbb75bdfa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Fri, 6 Oct 2023 15:10:42 +0200 Subject: [PATCH 01/12] fingerprint files with md5 and save to file --- .gitignore | 1 + internal/cmd/files/files.go | 4 +- internal/cmd/files/files_test.go | 4 +- internal/cmd/files/fingerprint/fingerprint.go | 78 ++++++++ .../cmd/files/fingerprint/fingerprint_test.go | 56 ++++++ internal/cmd/root/root.go | 2 +- internal/file/default_exclusion.go | 14 ++ internal/file/default_exclusion_test.go | 20 ++ internal/file/fingerprint.go | 179 ++++++++++++++++++ internal/file/fingerprint_test.go | 94 +++++++++ .../file/testdata/fingerprinter/nofile.txt | 1 + .../file/testdata/fingerprinter/testfile.py | 1 + internal/file/testdata/fingerprinter_mock.go | 19 ++ internal/wire/cli_container.go | 9 + 14 files changed, 478 insertions(+), 4 deletions(-) create mode 100644 internal/cmd/files/fingerprint/fingerprint.go create mode 100644 internal/cmd/files/fingerprint/fingerprint_test.go create mode 100644 internal/file/fingerprint.go create mode 100644 internal/file/fingerprint_test.go create mode 100644 internal/file/testdata/fingerprinter/nofile.txt create mode 100644 internal/file/testdata/fingerprinter/testfile.py create mode 100644 internal/file/testdata/fingerprinter_mock.go diff --git a/.gitignore b/.gitignore index 5fdb8fdf..e9a9e65f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ internal/resolution/pm/gradle/.gradle-init-script.debricked.groovy test/resolve/testdata/npm/yarn.lock test/resolve/testdata/nuget/packages.lock.json test/resolve/testdata/nuget/obj +.debricked.fingerprints.wfp diff --git a/internal/cmd/files/files.go b/internal/cmd/files/files.go index 34627f19..d06265e2 100644 --- a/internal/cmd/files/files.go +++ b/internal/cmd/files/files.go @@ -2,12 +2,13 @@ package files import ( "github.com/debricked/cli/internal/cmd/files/find" + "github.com/debricked/cli/internal/cmd/files/fingerprint" "github.com/debricked/cli/internal/file" "github.com/spf13/cobra" "github.com/spf13/viper" ) -func NewFilesCmd(finder file.IFinder) *cobra.Command { +func NewFilesCmd(finder file.IFinder, fingerprinter file.IFingerprint) *cobra.Command { cmd := &cobra.Command{ Use: "files", Short: "Analyze files", @@ -18,6 +19,7 @@ func NewFilesCmd(finder file.IFinder) *cobra.Command { } cmd.AddCommand(find.NewFindCmd(finder)) + cmd.AddCommand(fingerprint.NewFingerprintCmd(fingerprinter)) return cmd } diff --git a/internal/cmd/files/files_test.go b/internal/cmd/files/files_test.go index eb3ff5a3..47780a57 100644 --- a/internal/cmd/files/files_test.go +++ b/internal/cmd/files/files_test.go @@ -9,13 +9,13 @@ import ( func TestNewFilesCmd(t *testing.T) { finder, _ := file.NewFinder(nil) - cmd := NewFilesCmd(finder) + cmd := NewFilesCmd(finder, nil) commands := cmd.Commands() nbrOfCommands := 1 assert.Lenf(t, commands, nbrOfCommands, "failed to assert that there were %d sub commands connected", nbrOfCommands) } func TestPreRun(t *testing.T) { - cmd := NewFilesCmd(nil) + cmd := NewFilesCmd(nil, nil) cmd.PreRun(cmd, nil) } diff --git a/internal/cmd/files/fingerprint/fingerprint.go b/internal/cmd/files/fingerprint/fingerprint.go new file mode 100644 index 00000000..3e26fc90 --- /dev/null +++ b/internal/cmd/files/fingerprint/fingerprint.go @@ -0,0 +1,78 @@ +package fingerprint + +import ( + "fmt" + "path/filepath" + + "github.com/debricked/cli/internal/file" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var exclusions = file.DefaultExclusionsFingerprint() + +const ( + ExclusionFlag = "exclusion-fingerprint" + OutputFileName = ".debricked.fingerprints.wfp" +) + +func NewFingerprintCmd(fingerprinter file.IFingerprint) *cobra.Command { + cmd := &cobra.Command{ + Use: "fingerprint [path]", + Short: "Fingerprint files for identification in a given path and writes it to " + OutputFileName, + Long: `Fingerprint files for identification in a given path. +This hashes all files and matches them against the Debricked knowledge base.`, + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlags(cmd.Flags()) + }, + RunE: RunE(fingerprinter), + } + fileExclusionExample := filepath.Join("*", "**.pyc") + dirExclusionExample := filepath.Join("**", "node_modules", "**") + exampleFlags := fmt.Sprintf("-%s \"%s\" -%s \"%s\"", ExclusionFlag, fileExclusionExample, ExclusionFlag, dirExclusionExample) + cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "", exclusions, `The following terms are supported to exclude paths: +Special Terms | Meaning +------------- | ------- +"*" | matches any sequence of non-Separator characters +"/**/" | matches zero or multiple directories +"?" | matches any single non-Separator character +"[class]" | matches any single non-Separator character against a class of characters ([see "character classes"]) +"{alt1,...}" | matches a sequence of characters if one of the comma-separated alternatives matches + +Example: +$ debricked files fingerprint . `+exampleFlags) + + viper.MustBindEnv(ExclusionFlag) + + return cmd +} + +func RunE(f file.IFingerprint) func(_ *cobra.Command, args []string) error { + return func(_ *cobra.Command, args []string) error { + path := "" + if len(args) > 0 { + path = args[0] + } + + err := AssertFlagsAreValid() + if err != nil { + return err + } + + output, err := f.FingerprintFiles(path, exclusions) + + if err != nil { + return err + } + + output.ToFile(OutputFileName) + + return nil + } +} + +func AssertFlagsAreValid() error { + // TODO: Add flag validation here + + return nil +} diff --git a/internal/cmd/files/fingerprint/fingerprint_test.go b/internal/cmd/files/fingerprint/fingerprint_test.go new file mode 100644 index 00000000..d892a06d --- /dev/null +++ b/internal/cmd/files/fingerprint/fingerprint_test.go @@ -0,0 +1,56 @@ +package fingerprint + +import ( + "os" + "testing" + + "github.com/debricked/cli/internal/file" + "github.com/debricked/cli/internal/file/testdata" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +func TestNewFingerprintCmd(t *testing.T) { + var f file.IFingerprint + cmd := NewFingerprintCmd(f) + + commands := cmd.Commands() + nbrOfCommands := 0 + assert.Len(t, commands, nbrOfCommands) + + flags := cmd.Flags() + flagAssertions := map[string]string{} + for name, shorthand := range flagAssertions { + flag := flags.Lookup(name) + assert.NotNil(t, flag) + assert.Equal(t, shorthand, flag.Shorthand) + } + + var flagKeys = []string{ + ExclusionFlag, + } + viperKeys := viper.AllKeys() + for _, flagKey := range flagKeys { + match := false + for _, key := range viperKeys { + if key == flagKey { + match = true + } + } + assert.Truef(t, match, "failed to assert that flag was present: "+flagKey) + } + +} + +func TestRunE(t *testing.T) { + defer func() { + os.Remove(OutputFileName) + }() + fingerprintMock := testdata.NewFingerprintMock() + runE := RunE(fingerprintMock) + + err := runE(nil, []string{"."}) + + assert.NoError(t, err) + +} diff --git a/internal/cmd/root/root.go b/internal/cmd/root/root.go index 95b847e1..14758aaf 100644 --- a/internal/cmd/root/root.go +++ b/internal/cmd/root/root.go @@ -40,7 +40,7 @@ Read more: https://portal.debricked.com/administration-47/how-do-i-generate-an-a debClient.SetAccessToken(&accessToken) rootCmd.AddCommand(report.NewReportCmd(container.LicenseReporter(), container.VulnerabilityReporter())) - rootCmd.AddCommand(files.NewFilesCmd(container.Finder())) + rootCmd.AddCommand(files.NewFilesCmd(container.Finder(), container.Fingerprinter())) rootCmd.AddCommand(scan.NewScanCmd(container.Scanner())) rootCmd.AddCommand(resolve.NewResolveCmd(container.Resolver())) diff --git a/internal/file/default_exclusion.go b/internal/file/default_exclusion.go index 95e2b1fa..ce1f2f68 100644 --- a/internal/file/default_exclusion.go +++ b/internal/file/default_exclusion.go @@ -10,3 +10,17 @@ func DefaultExclusions() []string { filepath.Join("**", "obj", "**"), // nuget } } + +var EXCLUDED_DIRS_FINGERPRINT = []string{"nbproject", "nbbuild", "nbdist", + "__pycache__", "venv", "_yardoc", "eggs", + "wheels", "htmlcov", "__pypackages__", ".egg-info"} + +func DefaultExclusionsFingerprint() []string { + output := []string{} + + for _, pattern := range EXCLUDED_DIRS_FINGERPRINT { + output = append(output, filepath.Join("**", pattern, "**")) + } + + return output +} diff --git a/internal/file/default_exclusion_test.go b/internal/file/default_exclusion_test.go index 232360b9..e206c5dd 100644 --- a/internal/file/default_exclusion_test.go +++ b/internal/file/default_exclusion_test.go @@ -2,6 +2,7 @@ package file import ( "os" + "path/filepath" "strings" "testing" @@ -15,3 +16,22 @@ func TestDefaultExclusions(t *testing.T) { assert.Greaterf(t, len(exParts), 0, "failed to assert that %s used correct separator. Proper separator %s", ex, separator) } } +func TestDefaultExclusionsFingerprint(t *testing.T) { + expectedExclusions := []string{ + filepath.Join("**", "nbproject", "**"), + filepath.Join("**", "nbbuild", "**"), + filepath.Join("**", "nbdist", "**"), + filepath.Join("**", "__pycache__", "**"), + filepath.Join("**", "venv", "**"), + filepath.Join("**", "_yardoc", "**"), + filepath.Join("**", "eggs", "**"), + filepath.Join("**", "wheels", "**"), + filepath.Join("**", "htmlcov", "**"), + filepath.Join("**", "__pypackages__", "**"), + filepath.Join("**", ".egg-info", "**"), + } + + exclusions := DefaultExclusionsFingerprint() + + assert.ElementsMatch(t, expectedExclusions, exclusions, "DefaultExclusionsFingerprint did not return the expected exclusions") +} diff --git a/internal/file/fingerprint.go b/internal/file/fingerprint.go new file mode 100644 index 00000000..ca8a815a --- /dev/null +++ b/internal/file/fingerprint.go @@ -0,0 +1,179 @@ +package file + +import ( + "bufio" + "crypto/md5" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +var EXCLUDED_EXT = []string{ + ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am", + ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".class", ".cmake", ".cnf", + ".conf", ".config", ".contributors", ".copying", ".crt", ".csproj", ".css", + ".csv", ".dat", ".data", ".doc", ".docx", ".dtd", ".dts", ".iws", ".c9", ".c9revisions", + ".dtsi", ".dump", ".eot", ".eps", ".geojson", ".gdoc", ".gif", + ".glif", ".gmo", ".gradle", ".guess", ".hex", ".htm", ".html", ".ico", ".iml", + ".in", ".inc", ".info", ".ini", ".ipynb", ".jpeg", ".jpg", ".json", ".jsonld", ".lock", + ".log", ".m4", ".map", ".markdown", ".md", ".md5", ".meta", ".mk", ".mxml", + ".o", ".otf", ".out", ".pbtxt", ".pdf", ".pem", ".phtml", ".plist", ".png", + ".po", ".ppt", ".prefs", ".properties", ".pyc", ".qdoc", ".result", ".rgb", + ".rst", ".scss", ".sha", ".sha1", ".sha2", ".sha256", ".sln", ".spec", ".sql", + ".sub", ".svg", ".svn-base", ".tab", ".template", ".test", ".tex", ".tiff", + ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl", ".woff", ".woff2", ".xht", + ".xhtml", ".xls", ".xlsx", ".xml", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp", + ".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover", + ".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", +} + +var EXCLUDED_FILE_ENDINGS = []string{"-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice", + "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3"} + +var ECLUDED_FILES = []string{ + "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar", + "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile", +} + +func isExcludedFile(filename string) bool { + + filenameLower := strings.ToLower(filename) + for _, format := range EXCLUDED_EXT { + if filepath.Ext(filenameLower) == format { + return true + } + } + + for _, file := range ECLUDED_FILES { + if filenameLower == file { + return true + } + } + + for _, ending := range EXCLUDED_FILE_ENDINGS { + if strings.HasSuffix(filenameLower, ending) { + return true + } + } + + return false +} + +type IFingerprint interface { + FingerprintFiles(rootPath string, exclusions []string) (Fingerprints, error) +} + +type Fingerprinter struct { +} + +func NewFingerprinter() *Fingerprinter { + return &Fingerprinter{} +} + +type FileFingerprint struct { + path string + contentLength int64 + fingerprint []byte +} + +func (f FileFingerprint) ToString() string { + return fmt.Sprintf("files=%x,%d,%s", f.fingerprint, f.contentLength, f.path) +} + +func (f *Fingerprinter) FingerprintFiles(rootPath string, exclusions []string) (Fingerprints, error) { + + if len(rootPath) == 0 { + rootPath = filepath.Base("") + } + + fingerprints := Fingerprints{} + + // Traverse files to find dependency file groups + err := filepath.Walk( + rootPath, + func(path string, fileInfo os.FileInfo, err error) error { + if err != nil { + return err + } + if !fileInfo.IsDir() && !excluded(exclusions, path) { + + if isExcludedFile(path) { + return nil + } + + fingerprint, err := computeMD5(path) + + // Skip directories, fileInfo.IsDir() is not reliable enough + if err != nil && !strings.Contains(err.Error(), "is a directory") { + return err + } else if err != nil { + return nil + } + + fingerprints.Append(fingerprint) + } + + return nil + }, + ) + + return fingerprints, err +} + +func computeMD5(filename string) (FileFingerprint, error) { + file, err := os.Open(filename) + if err != nil { + return FileFingerprint{}, err + } + defer file.Close() + + hash := md5.New() + if _, err := io.Copy(hash, file); err != nil { + return FileFingerprint{}, err + } + + contentLength, err := file.Seek(0, 2) + if err != nil { + return FileFingerprint{}, err + } + + return FileFingerprint{ + path: filename, + contentLength: contentLength, + fingerprint: hash.Sum(nil), + }, nil +} + +type Fingerprints struct { + Entries []FileFingerprint `json:"fingerprints"` +} + +func (f *Fingerprints) Len() int { + return len(f.Entries) +} + +func (f *Fingerprints) ToFile(ouputFile string) error { + file, err := os.Create(ouputFile) + if err != nil { + return err + } + defer file.Close() + + writer := bufio.NewWriter(file) + for _, fingerprint := range f.Entries { + _, err := writer.WriteString(fingerprint.ToString() + "\n") + if err != nil { + return err + } + } + writer.Flush() + + return nil + +} + +func (f *Fingerprints) Append(fingerprint FileFingerprint) { + f.Entries = append(f.Entries, fingerprint) +} diff --git a/internal/file/fingerprint_test.go b/internal/file/fingerprint_test.go new file mode 100644 index 00000000..cf7c4ba5 --- /dev/null +++ b/internal/file/fingerprint_test.go @@ -0,0 +1,94 @@ +package file + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsExcludedFile(t *testing.T) { + + // Test excluded file extensions + excludedExts := []string{".doc", ".pdf", ".txt"} + for _, ext := range excludedExts { + filename := "file" + ext + assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename) + } + + // Test excluded files + excludedFiles := []string{"LICENSE", "README.md", "Makefile"} + for _, file := range excludedFiles { + assert.True(t, isExcludedFile(file), "Expected %q to be excluded", file) + } + + // Test excluded file endings + excludedEndings := []string{"-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice", + "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3"} + for _, ending := range excludedEndings { + filename := "file." + ending + assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename) + } + + // Test non-excluded files + assert.False(t, isExcludedFile("file.py"), "Expected file.txt to not be excluded") + assert.False(t, isExcludedFile("file.go"), "Expected .go to not be excluded") + assert.False(t, isExcludedFile("file.dll"), "Expected .dll to not be excluded") + assert.False(t, isExcludedFile("file.jar"), "Expected .jar to not be excluded") +} + +func TestNewFingerprinter(t *testing.T) { + assert.NotNil(t, NewFingerprinter()) +} + +func TestFingerprinterInterface(t *testing.T) { + assert.Implements(t, (*IFingerprint)(nil), new(Fingerprinter)) +} + +func TestFingerprintFiles(t *testing.T) { + fingerprinter := NewFingerprinter() + fingerprints, err := fingerprinter.FingerprintFiles("testdata/fingerprinter", []string{}) + assert.NoError(t, err) + assert.NotNil(t, fingerprints) + assert.NotEmpty(t, fingerprints) + assert.Equal(t, 1, fingerprints.Len()) + assert.Equal(t, "files=72214db4e1e543018d1bafe86ea3b444,21,testdata/fingerprinter/testfile.py", fingerprints.Entries[0].ToString()) + + // Test no file + fingerprints, err = fingerprinter.FingerprintFiles("", []string{}) + assert.NoError(t, err) + assert.NotNil(t, fingerprints) + assert.NotEmpty(t, fingerprints) + +} + +func TestFileFingerprintToString(t *testing.T) { + fileFingerprint := FileFingerprint{path: "path", contentLength: 10, fingerprint: []byte("fingerprint")} + assert.Equal(t, "files=66696e6765727072696e74,10,path", fileFingerprint.ToString()) +} + +func TestComputeMD5(t *testing.T) { + // Test file not found + _, err := computeMD5("testdata/fingerprinter/testfile-not-found.py") + assert.Error(t, err) + + // Test file found + entry, err := computeMD5("testdata/fingerprinter/testfile.py") + assert.NoError(t, err) + entryS := fmt.Sprintf("%x", entry.fingerprint) + assert.Equal(t, "72214db4e1e543018d1bafe86ea3b444", entryS) +} + +func TestFingerprintsToFile(t *testing.T) { + fingerprints := Fingerprints{} + fingerprints.Entries = append(fingerprints.Entries, FileFingerprint{path: "path", contentLength: 10, fingerprint: []byte("fingerprint")}) + // Create temp dir + dir, err := os.MkdirTemp("", "test") + assert.NoError(t, err) + defer os.RemoveAll(dir) + // Write fingerprints to file + err = fingerprints.ToFile(dir + "/fingerprints.wfp") + assert.NoError(t, err) + +} diff --git a/internal/file/testdata/fingerprinter/nofile.txt b/internal/file/testdata/fingerprinter/nofile.txt new file mode 100644 index 00000000..ac8522fb --- /dev/null +++ b/internal/file/testdata/fingerprinter/nofile.txt @@ -0,0 +1 @@ +xxx \ No newline at end of file diff --git a/internal/file/testdata/fingerprinter/testfile.py b/internal/file/testdata/fingerprinter/testfile.py new file mode 100644 index 00000000..8cde7829 --- /dev/null +++ b/internal/file/testdata/fingerprinter/testfile.py @@ -0,0 +1 @@ +print("hello world") diff --git a/internal/file/testdata/fingerprinter_mock.go b/internal/file/testdata/fingerprinter_mock.go new file mode 100644 index 00000000..ccf932bd --- /dev/null +++ b/internal/file/testdata/fingerprinter_mock.go @@ -0,0 +1,19 @@ +package testdata + +import ( + "github.com/debricked/cli/internal/file" +) + +type FingerprintMock struct { + error error +} + +func NewFingerprintMock() *FingerprintMock { + return &FingerprintMock{ + error: nil, + } +} + +func (f *FingerprintMock) FingerprintFiles(rootPath string, exclusions []string) (file.Fingerprints, error) { + return file.Fingerprints{}, f.error +} diff --git a/internal/wire/cli_container.go b/internal/wire/cli_container.go index 16142201..642c2ba2 100644 --- a/internal/wire/cli_container.go +++ b/internal/wire/cli_container.go @@ -47,6 +47,10 @@ func (cc *CliContainer) wire() error { } cc.finder = finder + fingerprinter := file.NewFingerprinter() + + cc.fingerprinter = fingerprinter + uploader, err := upload.NewUploader(cc.debClient) if err != nil { return wireErr(err) @@ -83,6 +87,7 @@ type CliContainer struct { retryClient *retryablehttp.Client debClient client.IDebClient finder file.IFinder + fingerprinter file.IFingerprint uploader upload.IUploader ciService ci.IService scanner scan.IScanner @@ -118,6 +123,10 @@ func (cc *CliContainer) VulnerabilityReporter() vulnerabilityReport.Reporter { return cc.vulnerabilityReporter } +func (cc *CliContainer) Fingerprinter() file.IFingerprint { + return cc.fingerprinter +} + func wireErr(err error) error { return fmt.Errorf("failed to wire with cli-container. Error %s", err) } From 2b9bd16f7ea99d63d1ecb38256a91a9d00d54b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Fri, 6 Oct 2023 17:21:04 +0200 Subject: [PATCH 02/12] add fingerprinting to scan command --- internal/cmd/files/fingerprint/fingerprint.go | 7 ++- .../cmd/files/fingerprint/fingerprint_test.go | 2 +- internal/cmd/root/root_test.go | 2 +- internal/cmd/scan/scan.go | 5 +- internal/file/fingerprint.go | 4 ++ internal/scan/scanner.go | 22 ++++++-- internal/scan/scanner_test.go | 52 ++++++++++++++++--- internal/wire/cli_container.go | 1 + internal/wire/cli_container_test.go | 1 + 9 files changed, 78 insertions(+), 18 deletions(-) diff --git a/internal/cmd/files/fingerprint/fingerprint.go b/internal/cmd/files/fingerprint/fingerprint.go index 3e26fc90..253d66ea 100644 --- a/internal/cmd/files/fingerprint/fingerprint.go +++ b/internal/cmd/files/fingerprint/fingerprint.go @@ -12,14 +12,13 @@ import ( var exclusions = file.DefaultExclusionsFingerprint() const ( - ExclusionFlag = "exclusion-fingerprint" - OutputFileName = ".debricked.fingerprints.wfp" + ExclusionFlag = "exclusion-fingerprint" ) func NewFingerprintCmd(fingerprinter file.IFingerprint) *cobra.Command { cmd := &cobra.Command{ Use: "fingerprint [path]", - Short: "Fingerprint files for identification in a given path and writes it to " + OutputFileName, + Short: "Fingerprint files for identification in a given path and writes it to " + file.OutputFileNameFingerprints, Long: `Fingerprint files for identification in a given path. This hashes all files and matches them against the Debricked knowledge base.`, PreRun: func(cmd *cobra.Command, _ []string) { @@ -65,7 +64,7 @@ func RunE(f file.IFingerprint) func(_ *cobra.Command, args []string) error { return err } - output.ToFile(OutputFileName) + output.ToFile(file.OutputFileNameFingerprints) return nil } diff --git a/internal/cmd/files/fingerprint/fingerprint_test.go b/internal/cmd/files/fingerprint/fingerprint_test.go index d892a06d..f5844ae9 100644 --- a/internal/cmd/files/fingerprint/fingerprint_test.go +++ b/internal/cmd/files/fingerprint/fingerprint_test.go @@ -44,7 +44,7 @@ func TestNewFingerprintCmd(t *testing.T) { func TestRunE(t *testing.T) { defer func() { - os.Remove(OutputFileName) + os.Remove(file.OutputFileNameFingerprints) }() fingerprintMock := testdata.NewFingerprintMock() runE := RunE(fingerprintMock) diff --git a/internal/cmd/root/root_test.go b/internal/cmd/root/root_test.go index a69d742b..2d02f16a 100644 --- a/internal/cmd/root/root_test.go +++ b/internal/cmd/root/root_test.go @@ -31,7 +31,7 @@ func TestNewRootCmd(t *testing.T) { } } assert.Truef(t, match, "failed to assert that flag was present: "+AccessTokenFlag) - assert.Len(t, viperKeys, 13) + assert.Len(t, viperKeys, 14) } func TestPreRun(t *testing.T) { diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index cc5e7926..5db92ce8 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -20,6 +20,7 @@ var repositoryUrl string var integrationName string var exclusions = file.DefaultExclusions() var noResolve bool +var noFingerprint bool var passOnDowntime bool const ( @@ -31,6 +32,7 @@ const ( IntegrationFlag = "integration" ExclusionFlag = "exclusion" NoResolveFlag = "no-resolve" + NoFingerprintFlag = "no-fingerprint" PassOnTimeOut = "pass-on-timeout" ) @@ -82,7 +84,7 @@ $ debricked scan . `+exampleFlags) cmd.Flags().BoolVarP(&passOnDowntime, PassOnTimeOut, "p", false, "pass scan if there is a service access timeout") cmd.Flags().BoolVar(&noResolve, NoResolveFlag, false, `disables resolution of manifest files that lack lock files. Resolving manifest files enables more accurate dependency scanning since the whole dependency tree will be analysed. For example, if there is a "go.mod" in the target path, its dependencies are going to get resolved onto a lock file, and latter scanned.`) - + cmd.Flags().BoolVar(&noFingerprint, NoFingerprintFlag, false, "disables fingerprinting for undeclared component identification. Can be run as a standalone command with more granular options.") viper.MustBindEnv(RepositoryFlag) viper.MustBindEnv(CommitFlag) viper.MustBindEnv(BranchFlag) @@ -103,6 +105,7 @@ func RunE(s *scan.IScanner) func(_ *cobra.Command, args []string) error { options := scan.DebrickedOptions{ Path: path, Resolve: !viper.GetBool(NoResolveFlag), + Fingerprint: !viper.GetBool(NoFingerprintFlag), Exclusions: viper.GetStringSlice(ExclusionFlag), RepositoryName: viper.GetString(RepositoryFlag), CommitName: viper.GetString(CommitFlag), diff --git a/internal/file/fingerprint.go b/internal/file/fingerprint.go index ca8a815a..92affc2b 100644 --- a/internal/file/fingerprint.go +++ b/internal/file/fingerprint.go @@ -37,6 +37,10 @@ var ECLUDED_FILES = []string{ "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile", } +const ( + OutputFileNameFingerprints = ".debricked.fingerprints.wfp" +) + func isExcludedFile(filename string) bool { filenameLower := strings.ToLower(filename) diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index 48ac90d7..91f2894e 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -29,16 +29,18 @@ type IScanner interface { type IOptions interface{} type DebrickedScanner struct { - client *client.IDebClient - finder file.IFinder - uploader *upload.IUploader - ciService ci.IService - resolver resolution.IResolver + client *client.IDebClient + finder file.IFinder + uploader *upload.IUploader + ciService ci.IService + resolver resolution.IResolver + fingerprint file.IFingerprint } type DebrickedOptions struct { Path string Resolve bool + Fingerprint bool Exclusions []string RepositoryName string CommitName string @@ -55,6 +57,7 @@ func NewDebrickedScanner( uploader upload.IUploader, ciService ci.IService, resolver resolution.IResolver, + fingerprint file.IFingerprint, ) *DebrickedScanner { return &DebrickedScanner{ c, @@ -62,6 +65,7 @@ func NewDebrickedScanner( &uploader, ciService, resolver, + fingerprint, } } @@ -125,6 +129,14 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g } } + if options.Fingerprint { + fingerprints, err := dScanner.fingerprint.FingerprintFiles(options.Path, file.DefaultExclusionsFingerprint()) + if err != nil { + return nil, err + } + fingerprints.ToFile(file.OutputFileNameFingerprints) + } + fileGroups, err := dScanner.finder.GetGroups(options.Path, options.Exclusions, false, file.StrictAll) if err != nil { return nil, err diff --git a/internal/scan/scanner_test.go b/internal/scan/scanner_test.go index 4564e778..2db82a22 100644 --- a/internal/scan/scanner_test.go +++ b/internal/scan/scanner_test.go @@ -52,7 +52,8 @@ func TestNewDebrickedScanner(t *testing.T) { var finder file.IFinder var uploader upload.IUploader var resolver resolution.IResolver - s := NewDebrickedScanner(&debClient, finder, uploader, cis, resolver) + var fingerprint file.IFingerprint + s := NewDebrickedScanner(&debClient, finder, uploader, cis, resolver, fingerprint) assert.NotNil(t, s) } @@ -120,7 +121,7 @@ func TestScan(t *testing.T) { func TestScanFailingMetaObject(t *testing.T) { var debClient client.IDebClient = testdata.NewDebClientMock() - scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil) + scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil, nil) cwd, _ := os.Getwd() path := testdataNpm opts := DebrickedOptions{ @@ -167,7 +168,7 @@ func TestScanFailingNoFiles(t *testing.T) { func TestScanBadOpts(t *testing.T) { var c client.IDebClient - scanner := NewDebrickedScanner(&c, nil, nil, nil, nil) + scanner := NewDebrickedScanner(&c, nil, nil, nil, nil, nil) var opts IOptions err := scanner.Scan(opts) @@ -226,7 +227,7 @@ func TestScanEmptyResult(t *testing.T) { func TestScanInCiWithPathSet(t *testing.T) { var debClient client.IDebClient = testdata.NewDebClientMock() - scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil) + scanner := NewDebrickedScanner(&debClient, nil, nil, ciService, nil, nil) cwd, _ := os.Getwd() defer resetWd(t, cwd) path := testdataNpm @@ -508,7 +509,7 @@ func TestScanServiceDowntime(t *testing.T) { var ciService ci.IService = ci.NewService(nil) - scanner := NewDebrickedScanner(&debClient, finder, nil, ciService, nil) + scanner := NewDebrickedScanner(&debClient, finder, nil, ciService, nil, nil) path := testdataNpm repositoryName := path @@ -606,7 +607,7 @@ func makeScanner(clientMock *testdata.DebClientMock, resolverMock *resolveTestda var cis ci.IService = ci.NewService(nil) - return NewDebrickedScanner(&debClient, finder, uploader, cis, resolverMock) + return NewDebrickedScanner(&debClient, finder, uploader, cis, resolverMock, nil) } func cleanUpResolution(t *testing.T, resolverMock resolveTestdata.ResolverMock) { @@ -615,3 +616,42 @@ func cleanUpResolution(t *testing.T, resolverMock resolveTestdata.ResolverMock) t.Error(err) } } + +func TestScanWithFingerprint(t *testing.T) { + clientMock := testdata.NewDebClientMock() + addMockedFormatsResponse(clientMock, "yarn\\.lock") + addMockedFileUploadResponse(clientMock) + addMockedFinishResponse(clientMock, http.StatusNoContent) + addMockedStatusResponse(clientMock, http.StatusOK, 100) + + resolverMock := resolveTestdata.ResolverMock{} + resolverMock.SetFiles([]string{"yarn.lock"}) + + scanner := makeScanner(clientMock, &resolverMock) + scanner.fingerprint = file.NewFingerprinter() + + cwd, _ := os.Getwd() + defer resetWd(t, cwd) + // Clean up resolution must be done before wd reset, otherwise files cannot be deleted + defer cleanUpResolution(t, resolverMock) + + path := testdataNpm + repositoryName := path + commitName := "testdata/npm-commit" + opts := DebrickedOptions{ + Path: path, + Resolve: true, + Fingerprint: true, + Exclusions: nil, + RepositoryName: repositoryName, + CommitName: commitName, + BranchName: "", + CommitAuthor: "", + RepositoryUrl: "", + IntegrationName: "", + } + err := scanner.Scan(opts) + assert.NoError(t, err) + cwd, _ = os.Getwd() + assert.Contains(t, cwd, path) +} diff --git a/internal/wire/cli_container.go b/internal/wire/cli_container.go index 642c2ba2..66f45d7e 100644 --- a/internal/wire/cli_container.go +++ b/internal/wire/cli_container.go @@ -75,6 +75,7 @@ func (cc *CliContainer) wire() error { cc.uploader, cc.ciService, cc.resolver, + cc.fingerprinter, ) cc.licenseReporter = licenseReport.Reporter{DebClient: cc.debClient} diff --git a/internal/wire/cli_container_test.go b/internal/wire/cli_container_test.go index 536716fa..dd54124c 100644 --- a/internal/wire/cli_container_test.go +++ b/internal/wire/cli_container_test.go @@ -38,4 +38,5 @@ func assertCliContainer(t *testing.T, cc *CliContainer) { assert.NotNil(t, cc.Resolver()) assert.NotNil(t, cc.LicenseReporter()) assert.NotNil(t, cc.VulnerabilityReporter()) + assert.NotNil(t, cc.Fingerprinter()) } From 13cb327205f8ac1ee411ee8069b6d1036b108adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Fri, 6 Oct 2023 17:26:21 +0200 Subject: [PATCH 03/12] fix typo --- internal/file/fingerprint.go | 2 +- internal/file/fingerprint_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/file/fingerprint.go b/internal/file/fingerprint.go index 92affc2b..e4eec161 100644 --- a/internal/file/fingerprint.go +++ b/internal/file/fingerprint.go @@ -83,7 +83,7 @@ type FileFingerprint struct { } func (f FileFingerprint) ToString() string { - return fmt.Sprintf("files=%x,%d,%s", f.fingerprint, f.contentLength, f.path) + return fmt.Sprintf("file=%x,%d,%s", f.fingerprint, f.contentLength, f.path) } func (f *Fingerprinter) FingerprintFiles(rootPath string, exclusions []string) (Fingerprints, error) { diff --git a/internal/file/fingerprint_test.go b/internal/file/fingerprint_test.go index cf7c4ba5..c3764977 100644 --- a/internal/file/fingerprint_test.go +++ b/internal/file/fingerprint_test.go @@ -53,7 +53,7 @@ func TestFingerprintFiles(t *testing.T) { assert.NotNil(t, fingerprints) assert.NotEmpty(t, fingerprints) assert.Equal(t, 1, fingerprints.Len()) - assert.Equal(t, "files=72214db4e1e543018d1bafe86ea3b444,21,testdata/fingerprinter/testfile.py", fingerprints.Entries[0].ToString()) + assert.Equal(t, "file=72214db4e1e543018d1bafe86ea3b444,21,testdata/fingerprinter/testfile.py", fingerprints.Entries[0].ToString()) // Test no file fingerprints, err = fingerprinter.FingerprintFiles("", []string{}) @@ -65,7 +65,7 @@ func TestFingerprintFiles(t *testing.T) { func TestFileFingerprintToString(t *testing.T) { fileFingerprint := FileFingerprint{path: "path", contentLength: 10, fingerprint: []byte("fingerprint")} - assert.Equal(t, "files=66696e6765727072696e74,10,path", fileFingerprint.ToString()) + assert.Equal(t, "file=66696e6765727072696e74,10,path", fileFingerprint.ToString()) } func TestComputeMD5(t *testing.T) { From e01fb1ab744e5ad0dcd74610bf424147395f64eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Sat, 7 Oct 2023 12:11:05 +0200 Subject: [PATCH 04/12] Fix linting --- internal/cmd/files/fingerprint/fingerprint.go | 14 +++----------- internal/file/default_exclusion.go | 5 +++-- internal/file/fingerprint.go | 9 ++++----- internal/scan/scanner.go | 6 +++++- internal/scan/scanner_test.go | 6 ++++-- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/internal/cmd/files/fingerprint/fingerprint.go b/internal/cmd/files/fingerprint/fingerprint.go index 253d66ea..4e4374a1 100644 --- a/internal/cmd/files/fingerprint/fingerprint.go +++ b/internal/cmd/files/fingerprint/fingerprint.go @@ -53,25 +53,17 @@ func RunE(f file.IFingerprint) func(_ *cobra.Command, args []string) error { path = args[0] } - err := AssertFlagsAreValid() + output, err := f.FingerprintFiles(path, exclusions) + if err != nil { return err } - output, err := f.FingerprintFiles(path, exclusions) - + err = output.ToFile(file.OutputFileNameFingerprints) if err != nil { return err } - output.ToFile(file.OutputFileNameFingerprints) - return nil } } - -func AssertFlagsAreValid() error { - // TODO: Add flag validation here - - return nil -} diff --git a/internal/file/default_exclusion.go b/internal/file/default_exclusion.go index ce1f2f68..a9c9881c 100644 --- a/internal/file/default_exclusion.go +++ b/internal/file/default_exclusion.go @@ -11,8 +11,9 @@ func DefaultExclusions() []string { } } -var EXCLUDED_DIRS_FINGERPRINT = []string{"nbproject", "nbbuild", "nbdist", - "__pycache__", "venv", "_yardoc", "eggs", +var EXCLUDED_DIRS_FINGERPRINT = []string{ + "nbproject", "nbbuild", "nbdist", "node_modules", + "__pycache__", "venv", "_yardoc", "eggs", "venv", "wheels", "htmlcov", "__pypackages__", ".egg-info"} func DefaultExclusionsFingerprint() []string { diff --git a/internal/file/fingerprint.go b/internal/file/fingerprint.go index e4eec161..38098d03 100644 --- a/internal/file/fingerprint.go +++ b/internal/file/fingerprint.go @@ -2,7 +2,7 @@ package file import ( "bufio" - "crypto/md5" + "crypto/md5" // #nosec "fmt" "io" "os" @@ -112,11 +112,10 @@ func (f *Fingerprinter) FingerprintFiles(rootPath string, exclusions []string) ( // Skip directories, fileInfo.IsDir() is not reliable enough if err != nil && !strings.Contains(err.Error(), "is a directory") { return err - } else if err != nil { - return nil + } else if err == nil { + fingerprints.Append(fingerprint) } - fingerprints.Append(fingerprint) } return nil @@ -133,7 +132,7 @@ func computeMD5(filename string) (FileFingerprint, error) { } defer file.Close() - hash := md5.New() + hash := md5.New() // #nosec if _, err := io.Copy(hash, file); err != nil { return FileFingerprint{}, err } diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index 91f2894e..9ab1b784 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -134,7 +134,11 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g if err != nil { return nil, err } - fingerprints.ToFile(file.OutputFileNameFingerprints) + err = fingerprints.ToFile(file.OutputFileNameFingerprints) + if err != nil { + return nil, err + } + } fileGroups, err := dScanner.finder.GetGroups(options.Path, options.Exclusions, false, file.StrictAll) diff --git a/internal/scan/scanner_test.go b/internal/scan/scanner_test.go index 2db82a22..b6b037e8 100644 --- a/internal/scan/scanner_test.go +++ b/internal/scan/scanner_test.go @@ -438,7 +438,8 @@ func TestMapEnvToOptions(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - MapEnvToOptions(&c.opts, c.env) + cCopy := c + MapEnvToOptions(&cCopy.opts, c.env) assert.Equal(t, c.template.Path, c.opts.Path) assert.Nil(t, c.opts.Exclusions) assert.Equal(t, c.template.RepositoryName, c.opts.RepositoryName) @@ -483,7 +484,8 @@ func TestSetWorkingDirectory(t *testing.T) { cwd, _ := os.Getwd() for _, c := range cases { t.Run(c.name, func(t *testing.T) { - err := SetWorkingDirectory(&c.opts) + cCopy := c + err := SetWorkingDirectory(&cCopy.opts) defer resetWd(t, cwd) if len(c.errMessages) > 0 { From 2ae4eb1e6d96c43bd8e6f20ee937b259095053be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Sat, 7 Oct 2023 12:29:04 +0200 Subject: [PATCH 05/12] fix failing tests --- internal/cmd/files/files_test.go | 2 +- internal/file/default_exclusion.go | 2 +- internal/file/default_exclusion_test.go | 1 + internal/scan/scanner_test.go | 12 ++++++------ 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/cmd/files/files_test.go b/internal/cmd/files/files_test.go index 47780a57..a85be1a5 100644 --- a/internal/cmd/files/files_test.go +++ b/internal/cmd/files/files_test.go @@ -11,7 +11,7 @@ func TestNewFilesCmd(t *testing.T) { finder, _ := file.NewFinder(nil) cmd := NewFilesCmd(finder, nil) commands := cmd.Commands() - nbrOfCommands := 1 + nbrOfCommands := 2 assert.Lenf(t, commands, nbrOfCommands, "failed to assert that there were %d sub commands connected", nbrOfCommands) } diff --git a/internal/file/default_exclusion.go b/internal/file/default_exclusion.go index a9c9881c..c651eb0c 100644 --- a/internal/file/default_exclusion.go +++ b/internal/file/default_exclusion.go @@ -13,7 +13,7 @@ func DefaultExclusions() []string { var EXCLUDED_DIRS_FINGERPRINT = []string{ "nbproject", "nbbuild", "nbdist", "node_modules", - "__pycache__", "venv", "_yardoc", "eggs", "venv", + "__pycache__", "venv", "_yardoc", "eggs", "wheels", "htmlcov", "__pypackages__", ".egg-info"} func DefaultExclusionsFingerprint() []string { diff --git a/internal/file/default_exclusion_test.go b/internal/file/default_exclusion_test.go index e206c5dd..50e61686 100644 --- a/internal/file/default_exclusion_test.go +++ b/internal/file/default_exclusion_test.go @@ -21,6 +21,7 @@ func TestDefaultExclusionsFingerprint(t *testing.T) { filepath.Join("**", "nbproject", "**"), filepath.Join("**", "nbbuild", "**"), filepath.Join("**", "nbdist", "**"), + filepath.Join("**", "node_modules", "**"), filepath.Join("**", "__pycache__", "**"), filepath.Join("**", "venv", "**"), filepath.Join("**", "_yardoc", "**"), diff --git a/internal/scan/scanner_test.go b/internal/scan/scanner_test.go index b6b037e8..4d50134c 100644 --- a/internal/scan/scanner_test.go +++ b/internal/scan/scanner_test.go @@ -436,10 +436,10 @@ func TestMapEnvToOptions(t *testing.T) { }, }, } - for _, c := range cases { + for _, co := range cases { + c := co t.Run(c.name, func(t *testing.T) { - cCopy := c - MapEnvToOptions(&cCopy.opts, c.env) + MapEnvToOptions(&c.opts, c.env) assert.Equal(t, c.template.Path, c.opts.Path) assert.Nil(t, c.opts.Exclusions) assert.Equal(t, c.template.RepositoryName, c.opts.RepositoryName) @@ -482,10 +482,10 @@ func TestSetWorkingDirectory(t *testing.T) { }, } cwd, _ := os.Getwd() - for _, c := range cases { + for _, co := range cases { + c := co t.Run(c.name, func(t *testing.T) { - cCopy := c - err := SetWorkingDirectory(&cCopy.opts) + err := SetWorkingDirectory(&c.opts) defer resetWd(t, cwd) if len(c.errMessages) > 0 { From 2728a82b7f6ab1a215d916bf09c6631629faf08d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Sat, 7 Oct 2023 13:00:29 +0200 Subject: [PATCH 06/12] fix exclusion of venv --- internal/file/default_exclusion.go | 10 ++++++++-- internal/file/default_exclusion_test.go | 4 ++-- internal/resolution/pm/pip/cmd_factory.go | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/internal/file/default_exclusion.go b/internal/file/default_exclusion.go index c651eb0c..0f7b9cf7 100644 --- a/internal/file/default_exclusion.go +++ b/internal/file/default_exclusion.go @@ -13,8 +13,10 @@ func DefaultExclusions() []string { var EXCLUDED_DIRS_FINGERPRINT = []string{ "nbproject", "nbbuild", "nbdist", "node_modules", - "__pycache__", "venv", "_yardoc", "eggs", - "wheels", "htmlcov", "__pypackages__", ".egg-info"} + "__pycache__", "_yardoc", "eggs", + "wheels", "htmlcov", "__pypackages__"} + +var EXCLUDED_DIRS_FINGERPRINT_RAW = []string{"**/*.egg-info/**", "**/*venv/**"} func DefaultExclusionsFingerprint() []string { output := []string{} @@ -23,5 +25,9 @@ func DefaultExclusionsFingerprint() []string { output = append(output, filepath.Join("**", pattern, "**")) } + for _, pattern := range EXCLUDED_DIRS_FINGERPRINT_RAW { + output = append(output, pattern) + } + return output } diff --git a/internal/file/default_exclusion_test.go b/internal/file/default_exclusion_test.go index 50e61686..b4f70fa0 100644 --- a/internal/file/default_exclusion_test.go +++ b/internal/file/default_exclusion_test.go @@ -23,13 +23,13 @@ func TestDefaultExclusionsFingerprint(t *testing.T) { filepath.Join("**", "nbdist", "**"), filepath.Join("**", "node_modules", "**"), filepath.Join("**", "__pycache__", "**"), - filepath.Join("**", "venv", "**"), filepath.Join("**", "_yardoc", "**"), filepath.Join("**", "eggs", "**"), filepath.Join("**", "wheels", "**"), filepath.Join("**", "htmlcov", "**"), filepath.Join("**", "__pypackages__", "**"), - filepath.Join("**", ".egg-info", "**"), + "**/*.egg-info/**", + "**/*venv/**", } exclusions := DefaultExclusionsFingerprint() diff --git a/internal/resolution/pm/pip/cmd_factory.go b/internal/resolution/pm/pip/cmd_factory.go index bcb7918c..eaea1852 100644 --- a/internal/resolution/pm/pip/cmd_factory.go +++ b/internal/resolution/pm/pip/cmd_factory.go @@ -46,7 +46,7 @@ func (cmdf CmdFactory) MakeCreateVenvCmd(fpath string) (*exec.Cmd, error) { return &exec.Cmd{ Path: python, - Args: []string{pythonCommand, "-m", "venv", fpath, "--clear", "--system-site-packages"}, + Args: []string{pythonCommand, "-m", "venv", fpath, "--clear"}, //"--system-site-packages" }, nil } From 03a7bc22c3a7950a2f6f1828e8dfabe28ba69a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Sat, 7 Oct 2023 13:11:38 +0200 Subject: [PATCH 07/12] disable fingerprint in scan by default as feature flag --- internal/cmd/scan/scan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index 5db92ce8..8b887f9a 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -84,7 +84,7 @@ $ debricked scan . `+exampleFlags) cmd.Flags().BoolVarP(&passOnDowntime, PassOnTimeOut, "p", false, "pass scan if there is a service access timeout") cmd.Flags().BoolVar(&noResolve, NoResolveFlag, false, `disables resolution of manifest files that lack lock files. Resolving manifest files enables more accurate dependency scanning since the whole dependency tree will be analysed. For example, if there is a "go.mod" in the target path, its dependencies are going to get resolved onto a lock file, and latter scanned.`) - cmd.Flags().BoolVar(&noFingerprint, NoFingerprintFlag, false, "disables fingerprinting for undeclared component identification. Can be run as a standalone command with more granular options.") + cmd.Flags().BoolVar(&noFingerprint, NoFingerprintFlag, true, "disables fingerprinting for undeclared component identification. Can be run as a standalone command with more granular options.") viper.MustBindEnv(RepositoryFlag) viper.MustBindEnv(CommitFlag) viper.MustBindEnv(BranchFlag) From c1fac1008fdea978604207ac3bf833b362da3bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Tue, 10 Oct 2023 13:29:44 +0200 Subject: [PATCH 08/12] improve fingerprinting logic with platform independence and include it in scans --- .github/workflows/test.yml | 5 + internal/cmd/files/fingerprint/fingerprint.go | 8 +- internal/cmd/scan/scan.go | 6 +- internal/file/default_exclusion.go | 4 +- internal/file/fingerprint.go | 115 +++++--- internal/file/fingerprint_test.go | 11 + internal/resolution/scheduler.go | 8 +- internal/scan/scanner.go | 29 +- internal/scan/scanner_test.go | 251 +++++++++--------- internal/tui/spinner_manager.go | 15 +- internal/tui/spinner_manager_test.go | 29 +- 11 files changed, 295 insertions(+), 186 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7fb5b8ed..78ccde68 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,11 @@ jobs: os: [ 'ubuntu-latest', 'windows-latest', 'macos-latest' ] runs-on: ${{ matrix.os }} steps: + - name: Set git to use LF + run: | + git config --global core.autocrlf input + git config --global core.eol lf + - uses: actions/checkout@v3 - name: Set up Go diff --git a/internal/cmd/files/fingerprint/fingerprint.go b/internal/cmd/files/fingerprint/fingerprint.go index 4e4374a1..790115fa 100644 --- a/internal/cmd/files/fingerprint/fingerprint.go +++ b/internal/cmd/files/fingerprint/fingerprint.go @@ -16,11 +16,13 @@ const ( ) func NewFingerprintCmd(fingerprinter file.IFingerprint) *cobra.Command { + + short := fmt.Sprintf("Fingerprint files for identification in a given path and writes it to %s. [beta feature]", file.OutputFileNameFingerprints) + long := fmt.Sprintf("Fingerprint files for identification in a given path and writes it to %s. [beta feature]\nThis hashes all files and matches them against the Debricked knowledge base.", file.OutputFileNameFingerprints) cmd := &cobra.Command{ Use: "fingerprint [path]", - Short: "Fingerprint files for identification in a given path and writes it to " + file.OutputFileNameFingerprints, - Long: `Fingerprint files for identification in a given path. -This hashes all files and matches them against the Debricked knowledge base.`, + Short: short, + Long: long, PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlags(cmd.Flags()) }, diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index 8b887f9a..638b385d 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -32,7 +32,7 @@ const ( IntegrationFlag = "integration" ExclusionFlag = "exclusion" NoResolveFlag = "no-resolve" - NoFingerprintFlag = "no-fingerprint" + FingerprintFlag = "fingerprint" PassOnTimeOut = "pass-on-timeout" ) @@ -84,7 +84,7 @@ $ debricked scan . `+exampleFlags) cmd.Flags().BoolVarP(&passOnDowntime, PassOnTimeOut, "p", false, "pass scan if there is a service access timeout") cmd.Flags().BoolVar(&noResolve, NoResolveFlag, false, `disables resolution of manifest files that lack lock files. Resolving manifest files enables more accurate dependency scanning since the whole dependency tree will be analysed. For example, if there is a "go.mod" in the target path, its dependencies are going to get resolved onto a lock file, and latter scanned.`) - cmd.Flags().BoolVar(&noFingerprint, NoFingerprintFlag, true, "disables fingerprinting for undeclared component identification. Can be run as a standalone command with more granular options.") + cmd.Flags().BoolVar(&noFingerprint, FingerprintFlag, false, "enables fingerprinting for undeclared component identification. Can be run as a standalone command [files fingerprint] with more granular options. [beta feature]") viper.MustBindEnv(RepositoryFlag) viper.MustBindEnv(CommitFlag) viper.MustBindEnv(BranchFlag) @@ -105,7 +105,7 @@ func RunE(s *scan.IScanner) func(_ *cobra.Command, args []string) error { options := scan.DebrickedOptions{ Path: path, Resolve: !viper.GetBool(NoResolveFlag), - Fingerprint: !viper.GetBool(NoFingerprintFlag), + Fingerprint: viper.GetBool(FingerprintFlag), Exclusions: viper.GetStringSlice(ExclusionFlag), RepositoryName: viper.GetString(RepositoryFlag), CommitName: viper.GetString(CommitFlag), diff --git a/internal/file/default_exclusion.go b/internal/file/default_exclusion.go index 0f7b9cf7..86449dcd 100644 --- a/internal/file/default_exclusion.go +++ b/internal/file/default_exclusion.go @@ -25,9 +25,7 @@ func DefaultExclusionsFingerprint() []string { output = append(output, filepath.Join("**", pattern, "**")) } - for _, pattern := range EXCLUDED_DIRS_FINGERPRINT_RAW { - output = append(output, pattern) - } + output = append(output, EXCLUDED_DIRS_FINGERPRINT_RAW...) return output } diff --git a/internal/file/fingerprint.go b/internal/file/fingerprint.go index 38098d03..17d35a31 100644 --- a/internal/file/fingerprint.go +++ b/internal/file/fingerprint.go @@ -4,10 +4,12 @@ import ( "bufio" "crypto/md5" // #nosec "fmt" - "io" + "log" "os" "path/filepath" "strings" + + "github.com/debricked/cli/internal/tui" ) var EXCLUDED_EXT = []string{ @@ -43,6 +45,12 @@ const ( func isExcludedFile(filename string) bool { + return isExcludedByExtension(filename) || + isExcludedByFilename(filename) || + isExcludedByEnding(filename) +} + +func isExcludedByExtension(filename string) bool { filenameLower := strings.ToLower(filename) for _, format := range EXCLUDED_EXT { if filepath.Ext(filenameLower) == format { @@ -50,12 +58,22 @@ func isExcludedFile(filename string) bool { } } + return false +} + +func isExcludedByFilename(filename string) bool { + filenameLower := strings.ToLower(filename) for _, file := range ECLUDED_FILES { if filenameLower == file { return true } } + return false +} + +func isExcludedByEnding(filename string) bool { + filenameLower := strings.ToLower(filename) for _, ending := range EXCLUDED_FILE_ENDINGS { if strings.HasSuffix(filenameLower, ending) { return true @@ -70,10 +88,13 @@ type IFingerprint interface { } type Fingerprinter struct { + spinnerManager tui.ISpinnerManager } func NewFingerprinter() *Fingerprinter { - return &Fingerprinter{} + return &Fingerprinter{ + spinnerManager: tui.NewSpinnerManager("Fingerprinting", "0"), + } } type FileFingerprint struct { @@ -83,61 +104,93 @@ type FileFingerprint struct { } func (f FileFingerprint) ToString() string { - return fmt.Sprintf("file=%x,%d,%s", f.fingerprint, f.contentLength, f.path) -} + // Replace backslashes with forward slashes to make the path platform independent + path := strings.ReplaceAll(f.path, "\\", "/") + return fmt.Sprintf("file=%x,%d,%s", f.fingerprint, f.contentLength, path) +} func (f *Fingerprinter) FingerprintFiles(rootPath string, exclusions []string) (Fingerprints, error) { - + log.Println("Warning: Fingerprinting is beta and may not work as expected.") if len(rootPath) == 0 { rootPath = filepath.Base("") } fingerprints := Fingerprints{} - // Traverse files to find dependency file groups - err := filepath.Walk( - rootPath, - func(path string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if !fileInfo.IsDir() && !excluded(exclusions, path) { - - if isExcludedFile(path) { - return nil - } + f.spinnerManager.Start() + spinnerMessage := "files processed" + spinner := f.spinnerManager.AddSpinner(spinnerMessage) - fingerprint, err := computeMD5(path) + nbFiles := 0 - // Skip directories, fileInfo.IsDir() is not reliable enough - if err != nil && !strings.Contains(err.Error(), "is a directory") { - return err - } else if err == nil { - fingerprints.Append(fingerprint) - } + err := filepath.Walk(rootPath, func(path string, fileInfo os.FileInfo, err error) error { + nbFiles++ - } + if err != nil { + return err + } + if !shouldProcessFile(fileInfo, exclusions, path) { return nil - }, - ) + } + + fingerprint, err := computeMD5(path) + if err != nil { + return err + } + + fingerprints.Append(fingerprint) + + if nbFiles%100 == 0 { + f.spinnerManager.SetSpinnerMessage(spinner, spinnerMessage, fmt.Sprintf("%d", nbFiles)) + } + + return nil + }) + + f.spinnerManager.SetSpinnerMessage(spinner, spinnerMessage, fmt.Sprintf("%d", nbFiles)) + + if err != nil { + spinner.Error() + } else { + spinner.Complete() + } + + f.spinnerManager.Stop() return fingerprints, err } +func shouldProcessFile(fileInfo os.FileInfo, exclusions []string, path string) bool { + if fileInfo.IsDir() { + return false + } + + if excluded(exclusions, path) { + return false + } + + if isExcludedFile(path) { + return false + } + + return true +} + func computeMD5(filename string) (FileFingerprint, error) { - file, err := os.Open(filename) + data, err := os.ReadFile(filename) if err != nil { return FileFingerprint{}, err } - defer file.Close() hash := md5.New() // #nosec - if _, err := io.Copy(hash, file); err != nil { + + if _, err := hash.Write(data); err != nil { return FileFingerprint{}, err } - contentLength, err := file.Seek(0, 2) + contentLength := int64(len(data)) + if err != nil { return FileFingerprint{}, err } diff --git a/internal/file/fingerprint_test.go b/internal/file/fingerprint_test.go index c3764977..2948f170 100644 --- a/internal/file/fingerprint_test.go +++ b/internal/file/fingerprint_test.go @@ -63,6 +63,17 @@ func TestFingerprintFiles(t *testing.T) { } +func TestFingerprintFilesBackslash(t *testing.T) { + fingerprint := FileFingerprint{ + path: "testdata\\fingerprinter\\testfile.py", + contentLength: 21, + fingerprint: []byte{114, 33, 77, 180, 225, 229, 67, 1, 141, 27, 175, 232, 110, 163, 180, 68, 68, 68, 68, 68, 68}, + } + + assert.Equal(t, "file=72214db4e1e543018d1bafe86ea3b4444444444444,21,testdata/fingerprinter/testfile.py", fingerprint.ToString()) + +} + func TestFileFingerprintToString(t *testing.T) { fileFingerprint := FileFingerprint{path: "path", contentLength: 10, fingerprint: []byte("fingerprint")} assert.Equal(t, "file=66696e6765727072696e74,10,path", fileFingerprint.ToString()) diff --git a/internal/resolution/scheduler.go b/internal/resolution/scheduler.go index b8d94968..031ff49a 100644 --- a/internal/resolution/scheduler.go +++ b/internal/resolution/scheduler.go @@ -33,7 +33,7 @@ func (scheduler *Scheduler) Schedule(jobs []job.IJob) (IResolution, error) { scheduler.queue = make(chan queueItem, len(jobs)) scheduler.waitGroup.Add(len(jobs)) - scheduler.spinnerManager = tui.NewSpinnerManager() + scheduler.spinnerManager = tui.NewSpinnerManager("Resolving", "waiting for worker") for w := 1; w <= scheduler.workers; w++ { go scheduler.worker() @@ -75,16 +75,16 @@ func (scheduler *Scheduler) worker() { func (scheduler *Scheduler) updateStatus(item queueItem) { for { msg := <-item.job.ReceiveStatus() - tui.SetSpinnerMessage(item.spinner, item.job.GetFile(), msg) + scheduler.spinnerManager.SetSpinnerMessage(item.spinner, item.job.GetFile(), msg) } } func (scheduler *Scheduler) finish(item queueItem) { if item.job.Errors().HasError() { - tui.SetSpinnerMessage(item.spinner, item.job.GetFile(), "failed") + scheduler.spinnerManager.SetSpinnerMessage(item.spinner, item.job.GetFile(), "failed") item.spinner.Error() } else { - tui.SetSpinnerMessage(item.spinner, item.job.GetFile(), "done") + scheduler.spinnerManager.SetSpinnerMessage(item.spinner, item.job.GetFile(), "done") item.spinner.Complete() } } diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index 9ab1b784..9b9d4250 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -121,24 +121,41 @@ func (dScanner *DebrickedScanner) Scan(o IOptions) error { return nil } -func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject git.MetaObject) (*upload.UploadResult, error) { +func (dScanner *DebrickedScanner) scanResolve(options DebrickedOptions) error { if options.Resolve { _, resErr := dScanner.resolver.Resolve([]string{options.Path}, options.Exclusions) if resErr != nil { - return nil, resErr + return resErr } } + return nil +} + +func (dScanner *DebrickedScanner) scanFingerprint(options DebrickedOptions) error { if options.Fingerprint { fingerprints, err := dScanner.fingerprint.FingerprintFiles(options.Path, file.DefaultExclusionsFingerprint()) if err != nil { - return nil, err + return err } err = fingerprints.ToFile(file.OutputFileNameFingerprints) - if err != nil { - return nil, err - } + return err + } + + return nil +} + +func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject git.MetaObject) (*upload.UploadResult, error) { + + err := dScanner.scanResolve(options) + if err != nil { + return nil, err + } + + err = dScanner.scanFingerprint(options) + if err != nil { + return nil, err } fileGroups, err := dScanner.finder.GetGroups(options.Path, options.Exclusions, false, file.StrictAll) diff --git a/internal/scan/scanner_test.go b/internal/scan/scanner_test.go index 4d50134c..8e0823f7 100644 --- a/internal/scan/scanner_test.go +++ b/internal/scan/scanner_test.go @@ -307,135 +307,138 @@ func TestScanWithResolveErr(t *testing.T) { assert.ErrorIs(t, err, resolutionErr) } -func TestMapEnvToOptions(t *testing.T) { - dOptionsTemplate := DebrickedOptions{ - Path: "path", - Exclusions: nil, - RepositoryName: "repository", - CommitName: "commit", - BranchName: "branch", - CommitAuthor: "author", - RepositoryUrl: "url", - IntegrationName: "CLI", - } +// TestScanWithResolveErr tests that the scan is not aborted if the resolution fails +var dOptionsTemplate = DebrickedOptions{ + Path: "path", + Exclusions: nil, + RepositoryName: "repository", + CommitName: "commit", + BranchName: "branch", + CommitAuthor: "author", + RepositoryUrl: "url", + IntegrationName: "CLI", +} - cases := []struct { - name string - template DebrickedOptions - opts DebrickedOptions - env env.Env - }{ - { - name: "No env", - template: dOptionsTemplate, - opts: dOptionsTemplate, - env: env.Env{ - Repository: "", - Commit: "", - Branch: "", - Author: "", - RepositoryUrl: "", - Integration: "", - Filepath: "", - }, +var cases = []struct { + name string + template DebrickedOptions + opts DebrickedOptions + env env.Env +}{ + { + name: "No env", + template: dOptionsTemplate, + opts: dOptionsTemplate, + env: env.Env{ + Repository: "", + Commit: "", + Branch: "", + Author: "", + RepositoryUrl: "", + Integration: "", + Filepath: "", }, - { - name: "CI env set", - template: DebrickedOptions{ - Path: "env-path", - Exclusions: nil, - RepositoryName: "env-repository", - CommitName: "env-commit", - BranchName: "env-branch", - CommitAuthor: "author", - RepositoryUrl: "env-url", - IntegrationName: github.Integration, - }, - opts: DebrickedOptions{ - Path: "", - Exclusions: nil, - RepositoryName: "", - CommitName: "", - BranchName: "", - CommitAuthor: "author", - RepositoryUrl: "", - IntegrationName: "CLI", - }, - env: env.Env{ - Repository: "env-repository", - Commit: "env-commit", - Branch: "env-branch", - Author: "env-author", - RepositoryUrl: "env-url", - Integration: github.Integration, - Filepath: "env-path", - }, + }, + { + name: "CI env set", + template: DebrickedOptions{ + Path: "env-path", + Exclusions: nil, + RepositoryName: "env-repository", + CommitName: "env-commit", + BranchName: "env-branch", + CommitAuthor: "author", + RepositoryUrl: "env-url", + IntegrationName: github.Integration, }, - { - name: "CI env set without directory path", - template: DebrickedOptions{ - Path: "input-path", - Exclusions: nil, - RepositoryName: "env-repository", - CommitName: "env-commit", - BranchName: "env-branch", - CommitAuthor: "author", - RepositoryUrl: "env-url", - IntegrationName: github.Integration, - }, - opts: DebrickedOptions{ - Path: "input-path", - Exclusions: nil, - RepositoryName: "", - CommitName: "", - BranchName: "", - CommitAuthor: "author", - RepositoryUrl: "", - IntegrationName: "CLI", - }, - env: env.Env{ - Repository: "env-repository", - Commit: "env-commit", - Branch: "env-branch", - Author: "env-author", - RepositoryUrl: "env-url", - Integration: github.Integration, - Filepath: "", - }, + opts: DebrickedOptions{ + Path: "", + Exclusions: nil, + RepositoryName: "", + CommitName: "", + BranchName: "", + CommitAuthor: "author", + RepositoryUrl: "", + IntegrationName: "CLI", }, - { - name: "CI env set with directory path", - template: DebrickedOptions{ - Path: "input-path", - Exclusions: nil, - RepositoryName: "env-repository", - CommitName: "env-commit", - BranchName: "env-branch", - CommitAuthor: "author", - RepositoryUrl: "env-url", - IntegrationName: github.Integration, - }, - opts: DebrickedOptions{ - Path: "input-path", - Exclusions: nil, - RepositoryName: "", - CommitName: "", - BranchName: "", - CommitAuthor: "author", - RepositoryUrl: "", - IntegrationName: "CLI", - }, - env: env.Env{ - Repository: "env-repository", - Commit: "env-commit", - Branch: "env-branch", - Author: "env-author", - RepositoryUrl: "env-url", - Integration: github.Integration, - Filepath: "env-path", - }, + env: env.Env{ + Repository: "env-repository", + Commit: "env-commit", + Branch: "env-branch", + Author: "env-author", + RepositoryUrl: "env-url", + Integration: github.Integration, + Filepath: "env-path", }, - } + }, + { + name: "CI env set without directory path", + template: DebrickedOptions{ + Path: "input-path", + Exclusions: nil, + RepositoryName: "env-repository", + CommitName: "env-commit", + BranchName: "env-branch", + CommitAuthor: "author", + RepositoryUrl: "env-url", + IntegrationName: github.Integration, + }, + opts: DebrickedOptions{ + Path: "input-path", + Exclusions: nil, + RepositoryName: "", + CommitName: "", + BranchName: "", + CommitAuthor: "author", + RepositoryUrl: "", + IntegrationName: "CLI", + }, + env: env.Env{ + Repository: "env-repository", + Commit: "env-commit", + Branch: "env-branch", + Author: "env-author", + RepositoryUrl: "env-url", + Integration: github.Integration, + Filepath: "", + }, + }, + { + name: "CI env set with directory path", + template: DebrickedOptions{ + Path: "input-path", + Exclusions: nil, + RepositoryName: "env-repository", + CommitName: "env-commit", + BranchName: "env-branch", + CommitAuthor: "author", + RepositoryUrl: "env-url", + IntegrationName: github.Integration, + }, + opts: DebrickedOptions{ + Path: "input-path", + Exclusions: nil, + RepositoryName: "", + CommitName: "", + BranchName: "", + CommitAuthor: "author", + RepositoryUrl: "", + IntegrationName: "CLI", + }, + env: env.Env{ + Repository: "env-repository", + Commit: "env-commit", + Branch: "env-branch", + Author: "env-author", + RepositoryUrl: "env-url", + Integration: github.Integration, + Filepath: "env-path", + }, + }, +} + +func TestMapEnvToOptions(t *testing.T) { + for _, co := range cases { c := co t.Run(c.name, func(t *testing.T) { diff --git a/internal/tui/spinner_manager.go b/internal/tui/spinner_manager.go index eba02e0f..a895d2bf 100644 --- a/internal/tui/spinner_manager.go +++ b/internal/tui/spinner_manager.go @@ -15,19 +15,22 @@ type ISpinnerManager interface { AddSpinner(file string) *ysmrr.Spinner Start() Stop() + SetSpinnerMessage(spinner *ysmrr.Spinner, filename string, message string) } type SpinnerManager struct { - spinnerManager ysmrr.SpinnerManager + spinnerManager ysmrr.SpinnerManager + baseString string + spinnerStartMessage string } -func NewSpinnerManager() SpinnerManager { - return SpinnerManager{ysmrr.NewSpinnerManager(ysmrr.WithSpinnerColor(colors.FgHiBlue))} +func NewSpinnerManager(baseString string, spinnerStartMessage string) SpinnerManager { + return SpinnerManager{ysmrr.NewSpinnerManager(ysmrr.WithSpinnerColor(colors.FgHiBlue)), baseString, spinnerStartMessage} } func (sm SpinnerManager) AddSpinner(file string) *ysmrr.Spinner { spinner := sm.spinnerManager.AddSpinner("") - SetSpinnerMessage(spinner, file, "waiting for worker") + sm.SetSpinnerMessage(spinner, file, sm.spinnerStartMessage) return spinner } @@ -40,7 +43,7 @@ func (sm SpinnerManager) Stop() { sm.spinnerManager.Stop() } -func SetSpinnerMessage(spinner *ysmrr.Spinner, filename string, message string) { +func (sm SpinnerManager) SetSpinnerMessage(spinner *ysmrr.Spinner, filename string, message string) { const maxNumberOfChars = 50 truncatedFilename := filename if len(truncatedFilename) > maxNumberOfChars { @@ -60,5 +63,5 @@ func SetSpinnerMessage(spinner *ysmrr.Spinner, filename string, message string) } file := color.YellowString(truncatedFilename) - spinner.UpdateMessage(fmt.Sprintf("Resolving %s: %s", file, message)) + spinner.UpdateMessage(fmt.Sprintf("%s %s: %s", sm.baseString, file, message)) } diff --git a/internal/tui/spinner_manager_test.go b/internal/tui/spinner_manager_test.go index 7e44b23e..c9038f84 100644 --- a/internal/tui/spinner_manager_test.go +++ b/internal/tui/spinner_manager_test.go @@ -10,12 +10,19 @@ import ( ) func TestNewSpinnerManager(t *testing.T) { - spinnerManager := NewSpinnerManager() + spinnerManager := NewSpinnerManager( + "Resolving", + "waiting for worker", + ) + assert.NotNil(t, spinnerManager) } func TestSetSpinnerMessage(t *testing.T) { - spinnerManager := NewSpinnerManager() + spinnerManager := NewSpinnerManager( + "Resolving", + "waiting for worker", + ) message := "test" spinner := spinnerManager.AddSpinner(message) assert.Contains(t, spinner.GetMessage(), fmt.Sprintf("Resolving %s: waiting for worker", color.YellowString(message))) @@ -23,12 +30,15 @@ func TestSetSpinnerMessage(t *testing.T) { fileName := "file-name" message = "new test message" - SetSpinnerMessage(spinner, fileName, message) + spinnerManager.SetSpinnerMessage(spinner, fileName, message) assert.Contains(t, spinner.GetMessage(), fmt.Sprintf("Resolving %s: %s", color.YellowString(fileName), message)) } func TestSetSpinnerMessageLongFilenameParts(t *testing.T) { - spinnerManager := NewSpinnerManager() + spinnerManager := NewSpinnerManager( + "Resolving", + "waiting for worker", + ) longFilenameParts := []string{ "directory", "sub-directory################################################################", @@ -43,7 +53,10 @@ func TestSetSpinnerMessageLongFilenameParts(t *testing.T) { } func TestSetSpinnerMessageLongFilenameManyDirs(t *testing.T) { - spinnerManager := NewSpinnerManager() + spinnerManager := NewSpinnerManager( + "Resolving", + "waiting for worker", + ) longFilenameParts := []string{ "directory", "sub-directory", @@ -70,7 +83,11 @@ func TestSetSpinnerMessageLongFilenameManyDirs(t *testing.T) { } func TestStartStop(t *testing.T) { - spinnerManager := NewSpinnerManager() + spinnerManager := NewSpinnerManager( + "Resolving", + "waiting for worker", + ) + spinnerManager.Start() spinnerManager.Stop() } From 20e3e869ff3e08726a567565879d0bf288209da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Thu, 12 Oct 2023 10:58:09 +0200 Subject: [PATCH 09/12] fixes after review --- internal/file/fingerprint.go | 23 ++++++++++--- internal/file/fingerprint_test.go | 56 +++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/internal/file/fingerprint.go b/internal/file/fingerprint.go index 17d35a31..17c92b94 100644 --- a/internal/file/fingerprint.go +++ b/internal/file/fingerprint.go @@ -34,7 +34,7 @@ var EXCLUDED_EXT = []string{ var EXCLUDED_FILE_ENDINGS = []string{"-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice", "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3"} -var ECLUDED_FILES = []string{ +var EXCLUDED_FILES = []string{ "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar", "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile", } @@ -63,7 +63,7 @@ func isExcludedByExtension(filename string) bool { func isExcludedByFilename(filename string) bool { filenameLower := strings.ToLower(filename) - for _, file := range ECLUDED_FILES { + for _, file := range EXCLUDED_FILES { if filenameLower == file { return true } @@ -104,8 +104,7 @@ type FileFingerprint struct { } func (f FileFingerprint) ToString() string { - // Replace backslashes with forward slashes to make the path platform independent - path := strings.ReplaceAll(f.path, "\\", "/") + path := filepath.ToSlash(f.path) return fmt.Sprintf("file=%x,%d,%s", f.fingerprint, f.contentLength, path) } @@ -161,6 +160,15 @@ func (f *Fingerprinter) FingerprintFiles(rootPath string, exclusions []string) ( return fingerprints, err } +func isSymlink(filename string) (bool, error) { + info, err := os.Lstat(filename) + if err != nil { + return false, err + } + + return info.Mode()&os.ModeSymlink != 0, nil +} + func shouldProcessFile(fileInfo os.FileInfo, exclusions []string, path string) bool { if fileInfo.IsDir() { return false @@ -174,7 +182,12 @@ func shouldProcessFile(fileInfo os.FileInfo, exclusions []string, path string) b return false } - return true + isSymlink, err := isSymlink(path) + if err != nil { + return false + } + + return !isSymlink } func computeMD5(filename string) (FileFingerprint, error) { diff --git a/internal/file/fingerprint_test.go b/internal/file/fingerprint_test.go index 2948f170..25a2aea8 100644 --- a/internal/file/fingerprint_test.go +++ b/internal/file/fingerprint_test.go @@ -3,6 +3,7 @@ package file import ( "fmt" "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -38,6 +39,43 @@ func TestIsExcludedFile(t *testing.T) { assert.False(t, isExcludedFile("file.jar"), "Expected .jar to not be excluded") } +func TestShouldProcessFile(t *testing.T) { + // Create a temporary directory to use for testing + tempDir, err := os.MkdirTemp("", "should-process-file-test") + if err != nil { + t.Fatalf("Failed to create temporary directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Create a test file and a symbolic link to the file in the temporary directory + testFile := filepath.Join(tempDir, "test.py") + if err := os.WriteFile(testFile, []byte("test"), 0600); err != nil { + t.Fatalf("Failed to create test file %s: %v", testFile, err) + } + testLink := filepath.Join(tempDir, "test-link.py") + if err := os.Symlink(testFile, testLink); err != nil { + t.Fatalf("Failed to create symbolic link %s: %v", testLink, err) + } + + // Test with a regular file + fileInfo, err := os.Stat(testFile) + if err != nil { + t.Fatalf("Failed to get file info for %s: %v", testFile, err) + } + if !shouldProcessFile(fileInfo, []string{}, testFile) { + t.Errorf("Expected shouldProcessFile to return true for %s, but it returned false", testFile) + } + + // Test with a symbolic link + linkInfo, err := os.Stat(testLink) + if err != nil { + t.Fatalf("Failed to get file info for %s: %v", testLink, err) + } + if shouldProcessFile(linkInfo, []string{}, testLink) { + t.Errorf("Expected shouldProcessFile to return false for %s, but it returned true", testLink) + } +} + func TestNewFingerprinter(t *testing.T) { assert.NotNil(t, NewFingerprinter()) } @@ -64,13 +102,27 @@ func TestFingerprintFiles(t *testing.T) { } func TestFingerprintFilesBackslash(t *testing.T) { + + tempDir, err := os.MkdirTemp("", "slash-test") + if err != nil { + t.Fatalf("Failed to create temporary directory: %v", err) + } + defer os.RemoveAll(tempDir) + testFile := filepath.Join(tempDir, "testfile.py") + + testFileSlashes := filepath.ToSlash(testFile) + fingerprint := FileFingerprint{ - path: "testdata\\fingerprinter\\testfile.py", + path: testFile, contentLength: 21, fingerprint: []byte{114, 33, 77, 180, 225, 229, 67, 1, 141, 27, 175, 232, 110, 163, 180, 68, 68, 68, 68, 68, 68}, } - assert.Equal(t, "file=72214db4e1e543018d1bafe86ea3b4444444444444,21,testdata/fingerprinter/testfile.py", fingerprint.ToString()) + assert.Equal(t, fmt.Sprintf("file=72214db4e1e543018d1bafe86ea3b4444444444444,21,%s", testFileSlashes), fingerprint.ToString()) + + // Make sure it only contains "/" and not "\" + assert.NotContains(t, fingerprint.ToString(), "\\") + assert.Contains(t, fingerprint.ToString(), "/") } From 18ef92b6fc4a28066fba096f71c604747864dccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Fri, 13 Oct 2023 10:41:35 +0200 Subject: [PATCH 10/12] revert pip resolver change --- internal/resolution/pm/pip/cmd_factory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/resolution/pm/pip/cmd_factory.go b/internal/resolution/pm/pip/cmd_factory.go index eaea1852..bcb7918c 100644 --- a/internal/resolution/pm/pip/cmd_factory.go +++ b/internal/resolution/pm/pip/cmd_factory.go @@ -46,7 +46,7 @@ func (cmdf CmdFactory) MakeCreateVenvCmd(fpath string) (*exec.Cmd, error) { return &exec.Cmd{ Path: python, - Args: []string{pythonCommand, "-m", "venv", fpath, "--clear"}, //"--system-site-packages" + Args: []string{pythonCommand, "-m", "venv", fpath, "--clear", "--system-site-packages"}, }, nil } From 94560af3f31a77a686464619db9a51e6e321c33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Tue, 17 Oct 2023 13:35:38 +0200 Subject: [PATCH 11/12] move fingerprint to root command --- internal/cmd/files/files.go | 4 +--- internal/cmd/files/files_test.go | 6 +++--- internal/cmd/{files => }/fingerprint/fingerprint.go | 11 ++++++----- .../cmd/{files => }/fingerprint/fingerprint_test.go | 8 ++++---- internal/cmd/root/root.go | 4 +++- internal/cmd/root/root_test.go | 2 +- internal/file/finder.go | 4 ++-- internal/file/finder_test.go | 2 +- internal/{file => fingerprint}/fingerprint.go | 5 +++-- internal/{file => fingerprint}/fingerprint_test.go | 9 ++++++++- .../testdata/fingerprinter/nofile.txt | 0 .../testdata/fingerprinter/testfile.py | 0 .../testdata/fingerprinter_mock.go | 6 +++--- .../packages.config.nuget.debricked.csproj.temp | 10 ++++++++++ internal/scan/scanner.go | 7 ++++--- internal/scan/scanner_test.go | 5 +++-- internal/wire/cli_container.go | 7 ++++--- 17 files changed, 56 insertions(+), 34 deletions(-) rename internal/cmd/{files => }/fingerprint/fingerprint.go (81%) rename internal/cmd/{files => }/fingerprint/fingerprint_test.go (83%) rename internal/{file => fingerprint}/fingerprint.go (98%) rename internal/{file => fingerprint}/fingerprint_test.go (96%) rename internal/{file => fingerprint}/testdata/fingerprinter/nofile.txt (100%) rename internal/{file => fingerprint}/testdata/fingerprinter/testfile.py (100%) rename internal/{file => fingerprint}/testdata/fingerprinter_mock.go (61%) create mode 100644 internal/resolution/pm/nuget/testdata/missing_framework/packages.config.nuget.debricked.csproj.temp diff --git a/internal/cmd/files/files.go b/internal/cmd/files/files.go index d06265e2..34627f19 100644 --- a/internal/cmd/files/files.go +++ b/internal/cmd/files/files.go @@ -2,13 +2,12 @@ package files import ( "github.com/debricked/cli/internal/cmd/files/find" - "github.com/debricked/cli/internal/cmd/files/fingerprint" "github.com/debricked/cli/internal/file" "github.com/spf13/cobra" "github.com/spf13/viper" ) -func NewFilesCmd(finder file.IFinder, fingerprinter file.IFingerprint) *cobra.Command { +func NewFilesCmd(finder file.IFinder) *cobra.Command { cmd := &cobra.Command{ Use: "files", Short: "Analyze files", @@ -19,7 +18,6 @@ func NewFilesCmd(finder file.IFinder, fingerprinter file.IFingerprint) *cobra.Co } cmd.AddCommand(find.NewFindCmd(finder)) - cmd.AddCommand(fingerprint.NewFingerprintCmd(fingerprinter)) return cmd } diff --git a/internal/cmd/files/files_test.go b/internal/cmd/files/files_test.go index a85be1a5..eb3ff5a3 100644 --- a/internal/cmd/files/files_test.go +++ b/internal/cmd/files/files_test.go @@ -9,13 +9,13 @@ import ( func TestNewFilesCmd(t *testing.T) { finder, _ := file.NewFinder(nil) - cmd := NewFilesCmd(finder, nil) + cmd := NewFilesCmd(finder) commands := cmd.Commands() - nbrOfCommands := 2 + nbrOfCommands := 1 assert.Lenf(t, commands, nbrOfCommands, "failed to assert that there were %d sub commands connected", nbrOfCommands) } func TestPreRun(t *testing.T) { - cmd := NewFilesCmd(nil, nil) + cmd := NewFilesCmd(nil) cmd.PreRun(cmd, nil) } diff --git a/internal/cmd/files/fingerprint/fingerprint.go b/internal/cmd/fingerprint/fingerprint.go similarity index 81% rename from internal/cmd/files/fingerprint/fingerprint.go rename to internal/cmd/fingerprint/fingerprint.go index 790115fa..d65951ef 100644 --- a/internal/cmd/files/fingerprint/fingerprint.go +++ b/internal/cmd/fingerprint/fingerprint.go @@ -5,6 +5,7 @@ import ( "path/filepath" "github.com/debricked/cli/internal/file" + "github.com/debricked/cli/internal/fingerprint" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -15,10 +16,10 @@ const ( ExclusionFlag = "exclusion-fingerprint" ) -func NewFingerprintCmd(fingerprinter file.IFingerprint) *cobra.Command { +func NewFingerprintCmd(fingerprinter fingerprint.IFingerprint) *cobra.Command { - short := fmt.Sprintf("Fingerprint files for identification in a given path and writes it to %s. [beta feature]", file.OutputFileNameFingerprints) - long := fmt.Sprintf("Fingerprint files for identification in a given path and writes it to %s. [beta feature]\nThis hashes all files and matches them against the Debricked knowledge base.", file.OutputFileNameFingerprints) + short := "Fingerprints files to match against the Debricked knowledge base. [beta feature]" + long := fmt.Sprintf("Fingerprint files for identification in a given path and writes it to %s. [beta feature]\nThis hashes all files and matches them against the Debricked knowledge base.", fingerprint.OutputFileNameFingerprints) cmd := &cobra.Command{ Use: "fingerprint [path]", Short: short, @@ -48,7 +49,7 @@ $ debricked files fingerprint . `+exampleFlags) return cmd } -func RunE(f file.IFingerprint) func(_ *cobra.Command, args []string) error { +func RunE(f fingerprint.IFingerprint) func(_ *cobra.Command, args []string) error { return func(_ *cobra.Command, args []string) error { path := "" if len(args) > 0 { @@ -61,7 +62,7 @@ func RunE(f file.IFingerprint) func(_ *cobra.Command, args []string) error { return err } - err = output.ToFile(file.OutputFileNameFingerprints) + err = output.ToFile(fingerprint.OutputFileNameFingerprints) if err != nil { return err } diff --git a/internal/cmd/files/fingerprint/fingerprint_test.go b/internal/cmd/fingerprint/fingerprint_test.go similarity index 83% rename from internal/cmd/files/fingerprint/fingerprint_test.go rename to internal/cmd/fingerprint/fingerprint_test.go index f5844ae9..e2970912 100644 --- a/internal/cmd/files/fingerprint/fingerprint_test.go +++ b/internal/cmd/fingerprint/fingerprint_test.go @@ -4,14 +4,14 @@ import ( "os" "testing" - "github.com/debricked/cli/internal/file" - "github.com/debricked/cli/internal/file/testdata" + "github.com/debricked/cli/internal/fingerprint" + "github.com/debricked/cli/internal/fingerprint/testdata" "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) func TestNewFingerprintCmd(t *testing.T) { - var f file.IFingerprint + var f fingerprint.IFingerprint cmd := NewFingerprintCmd(f) commands := cmd.Commands() @@ -44,7 +44,7 @@ func TestNewFingerprintCmd(t *testing.T) { func TestRunE(t *testing.T) { defer func() { - os.Remove(file.OutputFileNameFingerprints) + os.Remove(fingerprint.OutputFileNameFingerprints) }() fingerprintMock := testdata.NewFingerprintMock() runE := RunE(fingerprintMock) diff --git a/internal/cmd/root/root.go b/internal/cmd/root/root.go index 14758aaf..dcd5256e 100644 --- a/internal/cmd/root/root.go +++ b/internal/cmd/root/root.go @@ -2,6 +2,7 @@ package root import ( "github.com/debricked/cli/internal/cmd/files" + "github.com/debricked/cli/internal/cmd/fingerprint" "github.com/debricked/cli/internal/cmd/report" "github.com/debricked/cli/internal/cmd/resolve" "github.com/debricked/cli/internal/cmd/scan" @@ -40,8 +41,9 @@ Read more: https://portal.debricked.com/administration-47/how-do-i-generate-an-a debClient.SetAccessToken(&accessToken) rootCmd.AddCommand(report.NewReportCmd(container.LicenseReporter(), container.VulnerabilityReporter())) - rootCmd.AddCommand(files.NewFilesCmd(container.Finder(), container.Fingerprinter())) + rootCmd.AddCommand(files.NewFilesCmd(container.Finder())) rootCmd.AddCommand(scan.NewScanCmd(container.Scanner())) + rootCmd.AddCommand(fingerprint.NewFingerprintCmd(container.Fingerprinter())) rootCmd.AddCommand(resolve.NewResolveCmd(container.Resolver())) rootCmd.CompletionOptions.DisableDefaultCmd = true diff --git a/internal/cmd/root/root_test.go b/internal/cmd/root/root_test.go index 2d02f16a..490a514b 100644 --- a/internal/cmd/root/root_test.go +++ b/internal/cmd/root/root_test.go @@ -11,7 +11,7 @@ import ( func TestNewRootCmd(t *testing.T) { cmd := NewRootCmd("v0.0.0", wire.GetCliContainer()) commands := cmd.Commands() - nbrOfCommands := 4 + nbrOfCommands := 5 if len(commands) != nbrOfCommands { t.Errorf("failed to assert that there were %d sub commands connected", nbrOfCommands) } diff --git a/internal/file/finder.go b/internal/file/finder.go index 9e9c2575..9209f627 100644 --- a/internal/file/finder.go +++ b/internal/file/finder.go @@ -49,7 +49,7 @@ func (finder *Finder) GetGroups(rootPath string, exclusions []string, lockfileOn if err != nil { return err } - if !fileInfo.IsDir() && !excluded(exclusions, path) { + if !fileInfo.IsDir() && !Excluded(exclusions, path) { for _, format := range formats { if groups.Match(format, path, lockfileOnly) { @@ -97,7 +97,7 @@ func (finder *Finder) GetSupportedFormats() ([]*CompiledFormat, error) { return compiledDependencyFileFormats, nil } -func excluded(exclusions []string, path string) bool { +func Excluded(exclusions []string, path string) bool { for _, exclusion := range exclusions { ex := filepath.Clean(exclusion) matched, _ := doublestar.PathMatch(ex, path) diff --git a/internal/file/finder_test.go b/internal/file/finder_test.go index 0317cfdc..9457df93 100644 --- a/internal/file/finder_test.go +++ b/internal/file/finder_test.go @@ -226,7 +226,7 @@ func TestExclude(t *testing.T) { t.Run(c.name, func(t *testing.T) { var excludedFiles []string for _, file := range files { - if excluded(c.exclusions, file) { + if Excluded(c.exclusions, file) { excludedFiles = append(excludedFiles, file) } } diff --git a/internal/file/fingerprint.go b/internal/fingerprint/fingerprint.go similarity index 98% rename from internal/file/fingerprint.go rename to internal/fingerprint/fingerprint.go index 17c92b94..3efea4fe 100644 --- a/internal/file/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -1,4 +1,4 @@ -package file +package fingerprint import ( "bufio" @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + "github.com/debricked/cli/internal/file" "github.com/debricked/cli/internal/tui" ) @@ -174,7 +175,7 @@ func shouldProcessFile(fileInfo os.FileInfo, exclusions []string, path string) b return false } - if excluded(exclusions, path) { + if file.Excluded(exclusions, path) { return false } diff --git a/internal/file/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go similarity index 96% rename from internal/file/fingerprint_test.go rename to internal/fingerprint/fingerprint_test.go index 25a2aea8..8319de9c 100644 --- a/internal/file/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -1,4 +1,4 @@ -package file +package fingerprint import ( "fmt" @@ -71,9 +71,16 @@ func TestShouldProcessFile(t *testing.T) { if err != nil { t.Fatalf("Failed to get file info for %s: %v", testLink, err) } + if shouldProcessFile(linkInfo, []string{}, testLink) { t.Errorf("Expected shouldProcessFile to return false for %s, but it returned true", testLink) } + + // Test Excluded + if shouldProcessFile(fileInfo, []string{"**/test.py"}, testFile) { + t.Errorf("Expected shouldProcessFile to return true for %s, but it returned false", testFile) + } + } func TestNewFingerprinter(t *testing.T) { diff --git a/internal/file/testdata/fingerprinter/nofile.txt b/internal/fingerprint/testdata/fingerprinter/nofile.txt similarity index 100% rename from internal/file/testdata/fingerprinter/nofile.txt rename to internal/fingerprint/testdata/fingerprinter/nofile.txt diff --git a/internal/file/testdata/fingerprinter/testfile.py b/internal/fingerprint/testdata/fingerprinter/testfile.py similarity index 100% rename from internal/file/testdata/fingerprinter/testfile.py rename to internal/fingerprint/testdata/fingerprinter/testfile.py diff --git a/internal/file/testdata/fingerprinter_mock.go b/internal/fingerprint/testdata/fingerprinter_mock.go similarity index 61% rename from internal/file/testdata/fingerprinter_mock.go rename to internal/fingerprint/testdata/fingerprinter_mock.go index ccf932bd..7c77ff2a 100644 --- a/internal/file/testdata/fingerprinter_mock.go +++ b/internal/fingerprint/testdata/fingerprinter_mock.go @@ -1,7 +1,7 @@ package testdata import ( - "github.com/debricked/cli/internal/file" + "github.com/debricked/cli/internal/fingerprint" ) type FingerprintMock struct { @@ -14,6 +14,6 @@ func NewFingerprintMock() *FingerprintMock { } } -func (f *FingerprintMock) FingerprintFiles(rootPath string, exclusions []string) (file.Fingerprints, error) { - return file.Fingerprints{}, f.error +func (f *FingerprintMock) FingerprintFiles(rootPath string, exclusions []string) (fingerprint.Fingerprints, error) { + return fingerprint.Fingerprints{}, f.error } diff --git a/internal/resolution/pm/nuget/testdata/missing_framework/packages.config.nuget.debricked.csproj.temp b/internal/resolution/pm/nuget/testdata/missing_framework/packages.config.nuget.debricked.csproj.temp new file mode 100644 index 00000000..92d53d96 --- /dev/null +++ b/internal/resolution/pm/nuget/testdata/missing_framework/packages.config.nuget.debricked.csproj.temp @@ -0,0 +1,10 @@ + + + + net6.0 + + + + + + diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index 9b9d4250..16fd0f70 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -10,6 +10,7 @@ import ( "github.com/debricked/cli/internal/ci/env" "github.com/debricked/cli/internal/client" "github.com/debricked/cli/internal/file" + "github.com/debricked/cli/internal/fingerprint" "github.com/debricked/cli/internal/git" "github.com/debricked/cli/internal/resolution" "github.com/debricked/cli/internal/tui" @@ -34,7 +35,7 @@ type DebrickedScanner struct { uploader *upload.IUploader ciService ci.IService resolver resolution.IResolver - fingerprint file.IFingerprint + fingerprint fingerprint.IFingerprint } type DebrickedOptions struct { @@ -57,7 +58,7 @@ func NewDebrickedScanner( uploader upload.IUploader, ciService ci.IService, resolver resolution.IResolver, - fingerprint file.IFingerprint, + fingerprint fingerprint.IFingerprint, ) *DebrickedScanner { return &DebrickedScanner{ c, @@ -138,7 +139,7 @@ func (dScanner *DebrickedScanner) scanFingerprint(options DebrickedOptions) erro if err != nil { return err } - err = fingerprints.ToFile(file.OutputFileNameFingerprints) + err = fingerprints.ToFile(fingerprint.OutputFileNameFingerprints) return err } diff --git a/internal/scan/scanner_test.go b/internal/scan/scanner_test.go index 8e0823f7..d7646512 100644 --- a/internal/scan/scanner_test.go +++ b/internal/scan/scanner_test.go @@ -26,6 +26,7 @@ import ( "github.com/debricked/cli/internal/client" "github.com/debricked/cli/internal/client/testdata" "github.com/debricked/cli/internal/file" + "github.com/debricked/cli/internal/fingerprint" "github.com/debricked/cli/internal/git" "github.com/debricked/cli/internal/resolution" resolveTestdata "github.com/debricked/cli/internal/resolution/testdata" @@ -52,7 +53,7 @@ func TestNewDebrickedScanner(t *testing.T) { var finder file.IFinder var uploader upload.IUploader var resolver resolution.IResolver - var fingerprint file.IFingerprint + var fingerprint fingerprint.IFingerprint s := NewDebrickedScanner(&debClient, finder, uploader, cis, resolver, fingerprint) assert.NotNil(t, s) @@ -633,7 +634,7 @@ func TestScanWithFingerprint(t *testing.T) { resolverMock.SetFiles([]string{"yarn.lock"}) scanner := makeScanner(clientMock, &resolverMock) - scanner.fingerprint = file.NewFingerprinter() + scanner.fingerprint = fingerprint.NewFingerprinter() cwd, _ := os.Getwd() defer resetWd(t, cwd) diff --git a/internal/wire/cli_container.go b/internal/wire/cli_container.go index 66f45d7e..eb4f725f 100644 --- a/internal/wire/cli_container.go +++ b/internal/wire/cli_container.go @@ -6,6 +6,7 @@ import ( "github.com/debricked/cli/internal/ci" "github.com/debricked/cli/internal/client" "github.com/debricked/cli/internal/file" + "github.com/debricked/cli/internal/fingerprint" licenseReport "github.com/debricked/cli/internal/report/license" vulnerabilityReport "github.com/debricked/cli/internal/report/vulnerability" "github.com/debricked/cli/internal/resolution" @@ -47,7 +48,7 @@ func (cc *CliContainer) wire() error { } cc.finder = finder - fingerprinter := file.NewFingerprinter() + fingerprinter := fingerprint.NewFingerprinter() cc.fingerprinter = fingerprinter @@ -88,7 +89,7 @@ type CliContainer struct { retryClient *retryablehttp.Client debClient client.IDebClient finder file.IFinder - fingerprinter file.IFingerprint + fingerprinter fingerprint.IFingerprint uploader upload.IUploader ciService ci.IService scanner scan.IScanner @@ -124,7 +125,7 @@ func (cc *CliContainer) VulnerabilityReporter() vulnerabilityReport.Reporter { return cc.vulnerabilityReporter } -func (cc *CliContainer) Fingerprinter() file.IFingerprint { +func (cc *CliContainer) Fingerprinter() fingerprint.IFingerprint { return cc.fingerprinter } From 35970e741548e01b677adb4c91c57ed9e0cb006c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Wed, 18 Oct 2023 10:08:24 +0200 Subject: [PATCH 12/12] hide fingerprint commands --- internal/cmd/fingerprint/fingerprint.go | 7 ++++--- internal/cmd/scan/scan.go | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/cmd/fingerprint/fingerprint.go b/internal/cmd/fingerprint/fingerprint.go index d65951ef..fcbd6d6e 100644 --- a/internal/cmd/fingerprint/fingerprint.go +++ b/internal/cmd/fingerprint/fingerprint.go @@ -21,9 +21,10 @@ func NewFingerprintCmd(fingerprinter fingerprint.IFingerprint) *cobra.Command { short := "Fingerprints files to match against the Debricked knowledge base. [beta feature]" long := fmt.Sprintf("Fingerprint files for identification in a given path and writes it to %s. [beta feature]\nThis hashes all files and matches them against the Debricked knowledge base.", fingerprint.OutputFileNameFingerprints) cmd := &cobra.Command{ - Use: "fingerprint [path]", - Short: short, - Long: long, + Use: "fingerprint [path]", + Short: short, + Hidden: true, + Long: long, PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlags(cmd.Flags()) }, diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index 638b385d..58bfc8c3 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -85,6 +85,8 @@ $ debricked scan . `+exampleFlags) cmd.Flags().BoolVar(&noResolve, NoResolveFlag, false, `disables resolution of manifest files that lack lock files. Resolving manifest files enables more accurate dependency scanning since the whole dependency tree will be analysed. For example, if there is a "go.mod" in the target path, its dependencies are going to get resolved onto a lock file, and latter scanned.`) cmd.Flags().BoolVar(&noFingerprint, FingerprintFlag, false, "enables fingerprinting for undeclared component identification. Can be run as a standalone command [files fingerprint] with more granular options. [beta feature]") + cmd.Flags().MarkHidden(FingerprintFlag) //nolint:errcheck + viper.MustBindEnv(RepositoryFlag) viper.MustBindEnv(CommitFlag) viper.MustBindEnv(BranchFlag)