File fingerprinting and save to file

.github/workflows/test.yml

@@ -14,6 +14,11 @@ jobs:
         os: [ 'ubuntu-latest', 'windows-latest', 'macos-latest' ]
     runs-on: ${{ matrix.os }}
     steps:
+      - name: Set git to use LF
+        run: |
+          git config --global core.autocrlf input
+          git config --global core.eol lf
+
       - uses: actions/checkout@v3
 
       - name: Set up Go

.gitignore

@@ -15,3 +15,4 @@ internal/resolution/pm/gradle/.gradle-init-script.debricked.groovy
 test/resolve/testdata/npm/yarn.lock
 test/resolve/testdata/nuget/packages.lock.json
 test/resolve/testdata/nuget/obj
+.debricked.fingerprints.wfp

internal/cmd/files/files.go

@@ -2,12 +2,13 @@ package files
 
 import (
 	"github.com/debricked/cli/internal/cmd/files/find"
+	"github.com/debricked/cli/internal/cmd/files/fingerprint"
 	"github.com/debricked/cli/internal/file"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
-func NewFilesCmd(finder file.IFinder) *cobra.Command {
+func NewFilesCmd(finder file.IFinder, fingerprinter file.IFingerprint) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "files",
 		Short: "Analyze files",
@@ -18,6 +19,7 @@ func NewFilesCmd(finder file.IFinder) *cobra.Command {
 	}
 
 	cmd.AddCommand(find.NewFindCmd(finder))
+	cmd.AddCommand(fingerprint.NewFingerprintCmd(fingerprinter))
 
 	return cmd
 }

internal/cmd/files/files_test.go

@@ -9,13 +9,13 @@ import (
 
 func TestNewFilesCmd(t *testing.T) {
 	finder, _ := file.NewFinder(nil)
-	cmd := NewFilesCmd(finder)
+	cmd := NewFilesCmd(finder, nil)
 	commands := cmd.Commands()
-	nbrOfCommands := 1
+	nbrOfCommands := 2
 	assert.Lenf(t, commands, nbrOfCommands, "failed to assert that there were %d sub commands connected", nbrOfCommands)
 }
 
 func TestPreRun(t *testing.T) {
-	cmd := NewFilesCmd(nil)
+	cmd := NewFilesCmd(nil, nil)
 	cmd.PreRun(cmd, nil)
 }

internal/cmd/files/fingerprint/fingerprint.go

@@ -0,0 +1,71 @@
+package fingerprint
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/debricked/cli/internal/file"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var exclusions = file.DefaultExclusionsFingerprint()
+
+const (
+	ExclusionFlag = "exclusion-fingerprint"
+)
+
+func NewFingerprintCmd(fingerprinter file.IFingerprint) *cobra.Command {
+
+	short := fmt.Sprintf("Fingerprint files for identification in a given path and writes it to %s. [beta feature]", file.OutputFileNameFingerprints)
+	long := fmt.Sprintf("Fingerprint files for identification in a given path and writes it to %s. [beta feature]\nThis hashes all files and matches them against the Debricked knowledge base.", file.OutputFileNameFingerprints)
+	cmd := &cobra.Command{
+		Use:   "fingerprint [path]",
+		Short: short,
+		Long:  long,
+		PreRun: func(cmd *cobra.Command, _ []string) {
+			_ = viper.BindPFlags(cmd.Flags())
+		},
+		RunE: RunE(fingerprinter),
+	}
+	fileExclusionExample := filepath.Join("*", "**.pyc")
+	dirExclusionExample := filepath.Join("**", "node_modules", "**")
+	exampleFlags := fmt.Sprintf("-%s \"%s\" -%s \"%s\"", ExclusionFlag, fileExclusionExample, ExclusionFlag, dirExclusionExample)
+	cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "", exclusions, `The following terms are supported to exclude paths:
+Special Terms | Meaning
+------------- | -------
+"*"           | matches any sequence of non-Separator characters 
+"/**/"        | matches zero or multiple directories
+"?"           | matches any single non-Separator character
+"[class]"     | matches any single non-Separator character against a class of characters ([see "character classes"])
+"{alt1,...}"  | matches a sequence of characters if one of the comma-separated alternatives matches
+
+Example: 
+$ debricked files fingerprint . `+exampleFlags)
+
+	viper.MustBindEnv(ExclusionFlag)
+
+	return cmd
+}
+
+func RunE(f file.IFingerprint) func(_ *cobra.Command, args []string) error {
+	return func(_ *cobra.Command, args []string) error {
+		path := ""
+		if len(args) > 0 {
+			path = args[0]
+		}
+
+		output, err := f.FingerprintFiles(path, exclusions)
+
+		if err != nil {
+			return err
+		}
+
+		err = output.ToFile(file.OutputFileNameFingerprints)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}

internal/cmd/files/fingerprint/fingerprint_test.go

@@ -0,0 +1,56 @@
+package fingerprint
+
+import (
+	"os"
+	"testing"
+
+	"github.com/debricked/cli/internal/file"
+	"github.com/debricked/cli/internal/file/testdata"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewFingerprintCmd(t *testing.T) {
+	var f file.IFingerprint
+	cmd := NewFingerprintCmd(f)
+
+	commands := cmd.Commands()
+	nbrOfCommands := 0
+	assert.Len(t, commands, nbrOfCommands)
+
+	flags := cmd.Flags()
+	flagAssertions := map[string]string{}
+	for name, shorthand := range flagAssertions {
+		flag := flags.Lookup(name)
+		assert.NotNil(t, flag)
+		assert.Equal(t, shorthand, flag.Shorthand)
+	}
+
+	var flagKeys = []string{
+		ExclusionFlag,
+	}
+	viperKeys := viper.AllKeys()
+	for _, flagKey := range flagKeys {
+		match := false
+		for _, key := range viperKeys {
+			if key == flagKey {
+				match = true
+			}
+		}
+		assert.Truef(t, match, "failed to assert that flag was present: "+flagKey)
+	}
+
+}
+
+func TestRunE(t *testing.T) {
+	defer func() {
+		os.Remove(file.OutputFileNameFingerprints)
+	}()
+	fingerprintMock := testdata.NewFingerprintMock()
+	runE := RunE(fingerprintMock)
+
+	err := runE(nil, []string{"."})
+
+	assert.NoError(t, err)
+
+}

internal/cmd/root/root.go

@@ -40,7 +40,7 @@ Read more: https://portal.debricked.com/administration-47/how-do-i-generate-an-a
 	debClient.SetAccessToken(&accessToken)
 
 	rootCmd.AddCommand(report.NewReportCmd(container.LicenseReporter(), container.VulnerabilityReporter()))
-	rootCmd.AddCommand(files.NewFilesCmd(container.Finder()))
+	rootCmd.AddCommand(files.NewFilesCmd(container.Finder(), container.Fingerprinter()))
 	rootCmd.AddCommand(scan.NewScanCmd(container.Scanner()))
 	rootCmd.AddCommand(resolve.NewResolveCmd(container.Resolver()))
 

internal/cmd/root/root_test.go

@@ -31,7 +31,7 @@ func TestNewRootCmd(t *testing.T) {
 		}
 	}
 	assert.Truef(t, match, "failed to assert that flag was present: "+AccessTokenFlag)
-	assert.Len(t, viperKeys, 13)
+	assert.Len(t, viperKeys, 14)
 }
 
 func TestPreRun(t *testing.T) {

internal/cmd/scan/scan.go

@@ -20,6 +20,7 @@ var repositoryUrl string
 var integrationName string
 var exclusions = file.DefaultExclusions()
 var noResolve bool
+var noFingerprint bool
 var passOnDowntime bool
 
 const (
@@ -31,6 +32,7 @@ const (
 	IntegrationFlag   = "integration"
 	ExclusionFlag     = "exclusion"
 	NoResolveFlag     = "no-resolve"
+	FingerprintFlag   = "fingerprint"
 	PassOnTimeOut     = "pass-on-timeout"
 )
 
@@ -82,7 +84,7 @@ $ debricked scan . `+exampleFlags)
 	cmd.Flags().BoolVarP(&passOnDowntime, PassOnTimeOut, "p", false, "pass scan if there is a service access timeout")
 	cmd.Flags().BoolVar(&noResolve, NoResolveFlag, false, `disables resolution of manifest files that lack lock files. Resolving manifest files enables more accurate dependency scanning since the whole dependency tree will be analysed.
 For example, if there is a "go.mod" in the target path, its dependencies are going to get resolved onto a lock file, and latter scanned.`)
-
+	cmd.Flags().BoolVar(&noFingerprint, FingerprintFlag, false, "enables fingerprinting for undeclared component identification. Can be run as a standalone command [files fingerprint] with more granular options. [beta feature]")
 	viper.MustBindEnv(RepositoryFlag)
 	viper.MustBindEnv(CommitFlag)
 	viper.MustBindEnv(BranchFlag)
@@ -103,6 +105,7 @@ func RunE(s *scan.IScanner) func(_ *cobra.Command, args []string) error {
 		options := scan.DebrickedOptions{
 			Path:            path,
 			Resolve:         !viper.GetBool(NoResolveFlag),
+			Fingerprint:     viper.GetBool(FingerprintFlag),
 			Exclusions:      viper.GetStringSlice(ExclusionFlag),
 			RepositoryName:  viper.GetString(RepositoryFlag),
 			CommitName:      viper.GetString(CommitFlag),

internal/file/default_exclusion.go

@@ -10,3 +10,22 @@ func DefaultExclusions() []string {
 		filepath.Join("**", "obj", "**"), // nuget
 	}
 }
+
+var EXCLUDED_DIRS_FINGERPRINT = []string{
+	"nbproject", "nbbuild", "nbdist", "node_modules",
+	"__pycache__", "_yardoc", "eggs",
+	"wheels", "htmlcov", "__pypackages__"}
+
+var EXCLUDED_DIRS_FINGERPRINT_RAW = []string{"**/*.egg-info/**", "**/*venv/**"}
+
+func DefaultExclusionsFingerprint() []string {
+	output := []string{}
+
+	for _, pattern := range EXCLUDED_DIRS_FINGERPRINT {
+		output = append(output, filepath.Join("**", pattern, "**"))
+	}
+
+	output = append(output, EXCLUDED_DIRS_FINGERPRINT_RAW...)
+
+	return output
+}

internal/file/default_exclusion_test.go

@@ -2,6 +2,7 @@ package file
 
 import (
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -15,3 +16,23 @@ func TestDefaultExclusions(t *testing.T) {
 		assert.Greaterf(t, len(exParts), 0, "failed to assert that %s used correct separator. Proper separator %s", ex, separator)
 	}
 }
+func TestDefaultExclusionsFingerprint(t *testing.T) {
+	expectedExclusions := []string{
+		filepath.Join("**", "nbproject", "**"),
+		filepath.Join("**", "nbbuild", "**"),
+		filepath.Join("**", "nbdist", "**"),
+		filepath.Join("**", "node_modules", "**"),
+		filepath.Join("**", "__pycache__", "**"),
+		filepath.Join("**", "_yardoc", "**"),
+		filepath.Join("**", "eggs", "**"),
+		filepath.Join("**", "wheels", "**"),
+		filepath.Join("**", "htmlcov", "**"),
+		filepath.Join("**", "__pypackages__", "**"),
+		"**/*.egg-info/**",
+		"**/*venv/**",
+	}
+
+	exclusions := DefaultExclusionsFingerprint()
+
+	assert.ElementsMatch(t, expectedExclusions, exclusions, "DefaultExclusionsFingerprint did not return the expected exclusions")
+}