From dc55f5a436b2e865ef389037bb7e275b5a863651 Mon Sep 17 00:00:00 2001 From: filip Date: Sat, 23 Mar 2024 12:17:24 +0100 Subject: [PATCH 1/9] Add initial inclusion logic --- internal/callgraph/finder/finder.go | 2 +- .../callgraph/finder/golangfinder/finder.go | 4 +- .../finder/golangfinder/finder_test.go | 4 +- .../callgraph/finder/javafinder/finder.go | 4 +- .../finder/javafinder/finder_test.go | 14 +++---- .../callgraph/finder/testdata/finder_mock.go | 2 +- internal/callgraph/generator.go | 12 +++--- internal/callgraph/generator_test.go | 10 ++--- .../callgraph/language/golang/strategy.go | 7 ++-- .../language/golang/strategy_test.go | 22 +++++----- .../callgraph/language/java11/strategy.go | 9 +++-- .../language/java11/strategy_test.go | 20 +++++----- .../callgraph/strategy/strategy_factory.go | 8 ++-- .../strategy/strategy_factory_test.go | 6 +-- .../testdata/strategy_mock_factory.go | 4 +- internal/callgraph/testdata/generator_mock.go | 4 +- internal/cmd/callgraph/callgraph.go | 34 ++++++++++------ internal/cmd/callgraph/callgraph_test.go | 1 + internal/cmd/files/find/find.go | 12 +++++- internal/cmd/fingerprint/fingerprint.go | 11 ++++- internal/cmd/resolve/resolve.go | 10 +++++ internal/cmd/root/root_test.go | 2 +- internal/cmd/scan/scan.go | 10 ++++- internal/file/exclusion.go | 10 ++++- internal/file/exclusion_test.go | 2 +- internal/file/finder.go | 12 +++--- internal/file/finder_test.go | 15 +++---- internal/file/testdata/finder_mock.go | 4 +- internal/fingerprint/fingerprint.go | 40 +++++++++---------- internal/fingerprint/fingerprint_test.go | 23 +++++++++-- .../testdata/fingerprinter_mock.go | 2 +- internal/resolution/resolver.go | 14 ++++--- internal/scan/scanner.go | 14 ++++--- internal/scan/scanner_test.go | 5 +++ 34 files changed, 217 insertions(+), 136 deletions(-) diff --git a/internal/callgraph/finder/finder.go b/internal/callgraph/finder/finder.go index 16d870ec..a50dcb16 100644 --- a/internal/callgraph/finder/finder.go +++ b/internal/callgraph/finder/finder.go @@ -3,5 +3,5 @@ package finder type IFinder interface { FindRoots(files []string) ([]string, error) FindDependencyDirs(files []string, findJars bool) ([]string, error) - FindFiles(paths []string, exclusions []string) ([]string, error) + FindFiles(paths []string, exclusions []string, inclusions []string) ([]string, error) } diff --git a/internal/callgraph/finder/golangfinder/finder.go b/internal/callgraph/finder/golangfinder/finder.go index a4ca5884..4c93feb4 100644 --- a/internal/callgraph/finder/golangfinder/finder.go +++ b/internal/callgraph/finder/golangfinder/finder.go @@ -67,7 +67,7 @@ func (f GolangFinder) FindDependencyDirs(files []string, findJars bool) ([]strin return []string{}, nil } -func (f GolangFinder) FindFiles(roots []string, exclusions []string) ([]string, error) { +func (f GolangFinder) FindFiles(roots []string, exclusions []string, inclusions []string) ([]string, error) { files := make(map[string]bool) var err error = nil @@ -78,7 +78,7 @@ func (f GolangFinder) FindFiles(roots []string, exclusions []string) ([]string, return err } - excluded := file.Excluded(exclusions, path) + excluded := file.Excluded(exclusions, inclusions, path) if info.IsDir() && excluded { return filepath.SkipDir diff --git a/internal/callgraph/finder/golangfinder/finder_test.go b/internal/callgraph/finder/golangfinder/finder_test.go index b67122fb..0a8d50c7 100644 --- a/internal/callgraph/finder/golangfinder/finder_test.go +++ b/internal/callgraph/finder/golangfinder/finder_test.go @@ -24,7 +24,7 @@ func TestGolangFindEntrypointNoMain(t *testing.T) { func TestFindFiles(t *testing.T) { f := GolangFinder{} - files, err := f.FindFiles([]string{"testdata"}, nil) + files, err := f.FindFiles([]string{"testdata"}, nil, nil) assert.Nil(t, err) assert.NotEmpty(t, files) assert.Contains(t, files, filepath.Join("testdata", "app.go")) @@ -40,7 +40,7 @@ func TestFindDependencyDirs(t *testing.T) { func TestFindFilesExclusions(t *testing.T) { finder := GolangFinder{} - files, err := finder.FindFiles([]string{"testdata"}, []string{"testdata"}) + files, err := finder.FindFiles([]string{"testdata"}, []string{"testdata"}, nil) assert.Nil(t, err) assert.Empty(t, files) } diff --git a/internal/callgraph/finder/javafinder/finder.go b/internal/callgraph/finder/javafinder/finder.go index 9a18622f..f22bf4f6 100644 --- a/internal/callgraph/finder/javafinder/finder.go +++ b/internal/callgraph/finder/javafinder/finder.go @@ -38,7 +38,7 @@ func (f JavaFinder) FindDependencyDirs(files []string, findJars bool) ([]string, return dirJarFiles, nil } -func (f JavaFinder) FindFiles(roots []string, exclusions []string) ([]string, error) { +func (f JavaFinder) FindFiles(roots []string, exclusions []string, inclusions []string) ([]string, error) { files := make(map[string]bool) var err error = nil @@ -48,7 +48,7 @@ func (f JavaFinder) FindFiles(roots []string, exclusions []string) ([]string, er return err } - excluded := file.Excluded(exclusions, path) + excluded := file.Excluded(exclusions, inclusions, path) if info.IsDir() && excluded { return filepath.SkipDir diff --git a/internal/callgraph/finder/javafinder/finder_test.go b/internal/callgraph/finder/javafinder/finder_test.go index 0603e740..08341edd 100644 --- a/internal/callgraph/finder/javafinder/finder_test.go +++ b/internal/callgraph/finder/javafinder/finder_test.go @@ -36,14 +36,14 @@ func TestFindDependencyDirs(t *testing.T) { func TestFindFiles(t *testing.T) { f := JavaFinder{} - files, err := f.FindFiles([]string{"."}, nil) + files, err := f.FindFiles([]string{"."}, nil, nil) assert.Nil(t, err) assert.NotEmpty(t, files) } func TestFindFilesErr(t *testing.T) { f := JavaFinder{} - files, err := f.FindFiles([]string{"totaly-not-a-valid-path-123123123"}, nil) + files, err := f.FindFiles([]string{"totaly-not-a-valid-path-123123123"}, nil, nil) assert.NotNil(t, err) assert.Empty(t, files) } @@ -52,19 +52,19 @@ func TestFindFilesExcluded(t *testing.T) { f := JavaFinder{} project_path, err := filepath.Abs("testdata/test_project") assert.Nil(t, err) - files, err := f.FindFiles([]string{project_path}, nil) + files, err := f.FindFiles([]string{project_path}, nil, nil) assert.Nil(t, err) assert.Len(t, files, 2) - files, err = f.FindFiles([]string{project_path}, []string{"**/excluded_folder/**"}) + files, err = f.FindFiles([]string{project_path}, []string{"**/excluded_folder/**"}, nil) assert.Nil(t, err) assert.Len(t, files, 1) - files, err = f.FindFiles([]string{project_path}, []string{"excluded_folder"}) + files, err = f.FindFiles([]string{project_path}, []string{"excluded_folder"}, nil) assert.Nil(t, err) assert.Len(t, files, 2) - files, err = f.FindFiles([]string{project_path}, []string{"**/excluded*/**"}) + files, err = f.FindFiles([]string{project_path}, []string{"**/excluded*/**"}, nil) assert.Nil(t, err) assert.Len(t, files, 1) - files, err = f.FindFiles([]string{project_path}, []string{"**/excluded_file.txt"}) + files, err = f.FindFiles([]string{project_path}, []string{"**/excluded_file.txt"}, nil) assert.Nil(t, err) assert.Len(t, files, 1) } diff --git a/internal/callgraph/finder/testdata/finder_mock.go b/internal/callgraph/finder/testdata/finder_mock.go index 870efb89..9886644f 100644 --- a/internal/callgraph/finder/testdata/finder_mock.go +++ b/internal/callgraph/finder/testdata/finder_mock.go @@ -25,6 +25,6 @@ func (f FinderMock) FindRoots(_ []string) ([]string, error) { return f.FindRootsNames, f.FindRootsErr } -func (f FinderMock) FindFiles(_ []string, _ []string) ([]string, error) { +func (f FinderMock) FindFiles(_ []string, _ []string, _ []string) ([]string, error) { return f.FindFilesNames, f.FindFilesErr } diff --git a/internal/callgraph/generator.go b/internal/callgraph/generator.go index cb29d889..8279c58f 100644 --- a/internal/callgraph/generator.go +++ b/internal/callgraph/generator.go @@ -11,8 +11,8 @@ import ( ) type IGenerator interface { - GenerateWithTimer(paths []string, exclusions []string, configs []config.IConfig, timeout int) error - Generate(paths []string, exclusions []string, configs []config.IConfig, ctx cgexec.IContext) error + GenerateWithTimer(paths []string, exclusions []string, inclusions []string, configs []config.IConfig, timeout int) error + Generate(paths []string, exclusions []string, inclusions []string, configs []config.IConfig, ctx cgexec.IContext) error } type Generator struct { @@ -32,13 +32,13 @@ func NewGenerator( } } -func (g *Generator) GenerateWithTimer(paths []string, exclusions []string, configs []config.IConfig, timeout int) error { +func (g *Generator) GenerateWithTimer(paths []string, exclusions []string, inclusions []string, configs []config.IConfig, timeout int) error { result := make(chan error) ctx, cancel := cgexec.NewContext(timeout) defer cancel() go func() { - result <- g.Generate(paths, exclusions, configs, &ctx) + result <- g.Generate(paths, exclusions, inclusions, configs, &ctx) }() // Wait for the result or timeout @@ -47,14 +47,14 @@ func (g *Generator) GenerateWithTimer(paths []string, exclusions []string, confi return err } -func (g *Generator) Generate(paths []string, exclusions []string, configs []config.IConfig, ctx cgexec.IContext) error { +func (g *Generator) Generate(paths []string, exclusions []string, inclusions []string, configs []config.IConfig, ctx cgexec.IContext) error { targetPath := ".debrickedTmpFolder" debrickedExclusions := []string{targetPath} exclusions = append(exclusions, debrickedExclusions...) var jobs []job.IJob for _, config := range configs { - s, strategyErr := g.strategyFactory.Make(config, paths, exclusions, ctx) + s, strategyErr := g.strategyFactory.Make(config, paths, exclusions, inclusions, ctx) if strategyErr == nil { newJobs, err := s.Invoke() if err != nil { diff --git a/internal/callgraph/generator_test.go b/internal/callgraph/generator_test.go index b3ed5e39..9dbebafb 100644 --- a/internal/callgraph/generator_test.go +++ b/internal/callgraph/generator_test.go @@ -33,7 +33,7 @@ func TestGenerate(t *testing.T) { config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}, true, "maven"), } ctx, _ := ctxTestdata.NewContextMock() - err := g.Generate([]string{"../../go.mod"}, nil, configs, ctx) + err := g.Generate([]string{"../../go.mod"}, []string{}, nil, configs, ctx) assert.NoError(t, err) assert.NotEmpty(t, g.Generation.Jobs()) } @@ -47,7 +47,7 @@ func TestGenerateWithTimer(t *testing.T) { configs := []config.IConfig{ config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}, true, "maven"), } - err := g.GenerateWithTimer([]string{"../../go.mod"}, nil, configs, 1000) + err := g.GenerateWithTimer([]string{"../../go.mod"}, []string{}, nil, configs, 1000) assert.NoError(t, err) assert.NotEmpty(t, g.Generation.Jobs()) } @@ -62,7 +62,7 @@ func TestGenerateInvokeError(t *testing.T) { config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}, true, "maven"), } ctx, _ := ctxTestdata.NewContextMock() - err := g.Generate([]string{"../../go.mod"}, nil, configs, ctx) + err := g.Generate([]string{"../../go.mod"}, []string{}, nil, configs, ctx) assert.NotNil(t, err) } @@ -77,7 +77,7 @@ func TestGenerateScheduleError(t *testing.T) { config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}, true, "maven"), } ctx, _ := ctxTestdata.NewContextMock() - err := g.Generate([]string{"../../go.mod"}, nil, configs, ctx) + err := g.Generate([]string{"../../go.mod"}, []string{}, nil, configs, ctx) assert.NotEmpty(t, g.Generation.Jobs()) assert.ErrorIs(t, err, errAssertion) } @@ -89,7 +89,7 @@ func TestGenerateDirWithoutConfig(t *testing.T) { ) ctx, _ := ctxTestdata.NewContextMock() - err := g.Generate([]string{"invalid-dir"}, nil, nil, ctx) + err := g.Generate([]string{"invalid-dir"}, []string{}, nil, nil, ctx) assert.Empty(t, g.Generation.Jobs()) assert.NoError(t, err) } diff --git a/internal/callgraph/language/golang/strategy.go b/internal/callgraph/language/golang/strategy.go index 2d5b9de3..cd6d302d 100644 --- a/internal/callgraph/language/golang/strategy.go +++ b/internal/callgraph/language/golang/strategy.go @@ -17,6 +17,7 @@ type Strategy struct { config conf.IConfig paths []string exclusions []string + inclusions []string finder finder.IFinder ctx cgexec.IContext } @@ -32,7 +33,7 @@ func (s Strategy) Invoke() ([]job.IJob, error) { for _, path := range s.paths { - files, err := s.finder.FindFiles([]string{path}, s.exclusions) + files, err := s.finder.FindFiles([]string{path}, s.exclusions, s.inclusions) if err != nil { strategyWarning("Error while finding files: " + err.Error()) @@ -71,8 +72,8 @@ func (s Strategy) Invoke() ([]job.IJob, error) { return jobs, nil } -func NewStrategy(config conf.IConfig, paths []string, exclusions []string, finder finder.IFinder, ctx cgexec.IContext) Strategy { - return Strategy{config, paths, exclusions, finder, ctx} +func NewStrategy(config conf.IConfig, paths []string, exclusions []string, inclusions []string, finder finder.IFinder, ctx cgexec.IContext) Strategy { + return Strategy{config, paths, exclusions, inclusions, finder, ctx} } func strategyWarning(errMsg string) { diff --git a/internal/callgraph/language/golang/strategy_test.go b/internal/callgraph/language/golang/strategy_test.go index cb983882..fa1e1ec3 100644 --- a/internal/callgraph/language/golang/strategy_test.go +++ b/internal/callgraph/language/golang/strategy_test.go @@ -10,16 +10,16 @@ import ( ) func TestNewStrategy(t *testing.T) { - s := NewStrategy(nil, nil, nil, nil, nil) + s := NewStrategy(nil, nil, nil, nil, nil, nil) assert.NotNil(t, s) - s = NewStrategy(nil, []string{}, []string{}, nil, nil) + s = NewStrategy(nil, []string{}, []string{}, []string{}, nil, nil) assert.NotNil(t, s) - s = NewStrategy(nil, []string{"file"}, []string{}, nil, nil) + s = NewStrategy(nil, []string{"file"}, []string{}, []string{}, nil, nil) assert.NotNil(t, s) - s = NewStrategy(nil, []string{"file-1", "file-2"}, []string{}, nil, nil) + s = NewStrategy(nil, []string{"file-1", "file-2"}, []string{}, []string{}, nil, nil) assert.NotNil(t, s) conf := config.NewConfig("golang", []string{"arg1"}, map[string]string{"kwarg": "val"}, true, "go") @@ -27,13 +27,13 @@ func TestNewStrategy(t *testing.T) { testFiles := []string{"file-1"} finder.FindRootsNames = testFiles ctx, _ := ctxTestdata.NewContextMock() - s = NewStrategy(conf, []string{"."}, []string{}, finder, ctx) + s = NewStrategy(conf, []string{"."}, []string{}, []string{}, finder, ctx) assert.NotNil(t, s) assert.Equal(t, s.config, conf) } func TestInvokeNoFiles(t *testing.T) { - s := NewStrategy(nil, []string{}, []string{}, nil, nil) + s := NewStrategy(nil, []string{}, []string{}, []string{}, nil, nil) jobs, _ := s.Invoke() assert.Empty(t, jobs) } @@ -44,7 +44,7 @@ func TestInvokeOneFile(t *testing.T) { testFiles := []string{"file-1"} finder.FindRootsNames = testFiles ctx, _ := ctxTestdata.NewContextMock() - s := NewStrategy(conf, []string{"."}, []string{}, finder, ctx) + s := NewStrategy(conf, []string{"."}, []string{}, []string{}, finder, ctx) jobs, err := s.Invoke() assert.NoError(t, err) assert.Len(t, jobs, 1) @@ -56,7 +56,7 @@ func TestInvokeManyFiles(t *testing.T) { testFiles := []string{"file-1", "file-2"} finder.FindRootsNames = testFiles ctx, _ := ctxTestdata.NewContextMock() - s := NewStrategy(conf, []string{"."}, []string{}, finder, ctx) + s := NewStrategy(conf, []string{"."}, []string{}, []string{}, finder, ctx) jobs, _ := s.Invoke() assert.Len(t, jobs, 2) } @@ -68,14 +68,14 @@ func TestInvokeWithErrors(t *testing.T) { finder.FindRootsNames = testFiles finder.FindRootsErr = assert.AnError ctx, _ := ctxTestdata.NewContextMock() - s := NewStrategy(conf, []string{"."}, []string{}, finder, ctx) + s := NewStrategy(conf, []string{"."}, []string{}, []string{}, finder, ctx) jobs, err := s.Invoke() assert.Error(t, err) assert.Empty(t, jobs) finder.FindRootsErr = nil finder.FindFilesErr = assert.AnError - s = NewStrategy(conf, []string{"."}, []string{}, finder, ctx) + s = NewStrategy(conf, []string{"."}, []string{}, []string{}, finder, ctx) jobs, err = s.Invoke() assert.Error(t, err) assert.Empty(t, jobs) @@ -87,7 +87,7 @@ func TestInvokeNoRoots(t *testing.T) { testFiles := []string{} finder.FindRootsNames = testFiles ctx, _ := ctxTestdata.NewContextMock() - s := NewStrategy(conf, []string{"."}, []string{}, finder, ctx) + s := NewStrategy(conf, []string{"."}, []string{}, []string{}, finder, ctx) jobs, err := s.Invoke() assert.NoError(t, err) assert.Empty(t, jobs) diff --git a/internal/callgraph/language/java11/strategy.go b/internal/callgraph/language/java11/strategy.go index 194cc5f1..0bd41b15 100644 --- a/internal/callgraph/language/java11/strategy.go +++ b/internal/callgraph/language/java11/strategy.go @@ -20,6 +20,7 @@ type Strategy struct { cmdFactory ICmdFactory paths []string exclusions []string + inclusions []string finder finder.IFinder ctx cgexec.IContext } @@ -36,7 +37,7 @@ func (s Strategy) Invoke() ([]job.IJob, error) { var roots []string var err error - files, err := s.finder.FindFiles(s.paths, s.exclusions) + files, err := s.finder.FindFiles(s.paths, s.exclusions, s.inclusions) if err != nil { strategyWarning("Error while finding files: " + err.Error()) @@ -67,7 +68,7 @@ func (s Strategy) Invoke() ([]job.IJob, error) { } // If build, then we need to find the newly built files - files, _ = s.finder.FindFiles(s.paths, s.exclusions) + files, _ = s.finder.FindFiles(s.paths, s.exclusions, s.inclusions) } javaClassDirs, _ := s.finder.FindDependencyDirs(files, false) @@ -104,8 +105,8 @@ func (s Strategy) Invoke() ([]job.IJob, error) { return jobs, nil } -func NewStrategy(config conf.IConfig, paths []string, exclusions []string, finder finder.IFinder, ctx cgexec.IContext) Strategy { - return Strategy{config, CmdFactory{}, paths, exclusions, finder, ctx} +func NewStrategy(config conf.IConfig, paths []string, exclusions []string, inclusions []string, finder finder.IFinder, ctx cgexec.IContext) Strategy { + return Strategy{config, CmdFactory{}, paths, exclusions, inclusions, finder, ctx} } func strategyWarning(errMsg string) { diff --git a/internal/callgraph/language/java11/strategy_test.go b/internal/callgraph/language/java11/strategy_test.go index 45ea90a6..25281e28 100644 --- a/internal/callgraph/language/java11/strategy_test.go +++ b/internal/callgraph/language/java11/strategy_test.go @@ -13,16 +13,16 @@ import ( ) func TestNewStrategy(t *testing.T) { - s := NewStrategy(nil, nil, nil, nil, nil) + s := NewStrategy(nil, nil, nil, nil, nil, nil) assert.NotNil(t, s) - s = NewStrategy(nil, []string{}, []string{}, nil, nil) + s = NewStrategy(nil, []string{}, []string{}, []string{}, nil, nil) assert.NotNil(t, s) - s = NewStrategy(nil, []string{"file"}, []string{}, nil, nil) + s = NewStrategy(nil, []string{"file"}, []string{}, []string{}, nil, nil) assert.NotNil(t, s) - s = NewStrategy(nil, []string{"file-1", "file-2"}, []string{}, nil, nil) + s = NewStrategy(nil, []string{"file-1", "file-2"}, []string{}, []string{}, nil, nil) assert.NotNil(t, s) conf := config.NewConfig("java", []string{"arg1"}, map[string]string{"kwarg": "val"}, true, "maven") @@ -30,13 +30,13 @@ func TestNewStrategy(t *testing.T) { testFiles := []string{"file-1"} finder.FindRootsNames = testFiles ctx, _ := ctxTestdata.NewContextMock() - s = NewStrategy(conf, testFiles, []string{}, finder, ctx) + s = NewStrategy(conf, testFiles, []string{}, []string{}, finder, ctx) assert.NotNil(t, s) assert.Equal(t, s.config, conf) } func TestInvokeNoFiles(t *testing.T) { - s := NewStrategy(nil, []string{}, []string{}, nil, nil) + s := NewStrategy(nil, []string{}, []string{}, []string{}, nil, nil) jobs, _ := s.Invoke() assert.Empty(t, jobs) } @@ -47,7 +47,7 @@ func TestInvokeOneFile(t *testing.T) { testFiles := []string{"file-1"} finder.FindRootsNames = testFiles ctx, _ := ctxTestdata.NewContextMock() - s := NewStrategy(conf, testFiles, []string{}, finder, ctx) + s := NewStrategy(conf, testFiles, []string{}, []string{}, finder, ctx) jobs, _ := s.Invoke() assert.Len(t, jobs, 0) } @@ -58,7 +58,7 @@ func TestInvokeManyFiles(t *testing.T) { testFiles := []string{"file-1", "file-2"} finder.FindRootsNames = testFiles ctx, _ := ctxTestdata.NewContextMock() - s := NewStrategy(conf, testFiles, []string{}, finder, ctx) + s := NewStrategy(conf, testFiles, []string{}, []string{}, finder, ctx) jobs, _ := s.Invoke() assert.Len(t, jobs, 0) } @@ -70,7 +70,7 @@ func TestInvokeManyFilesWCorrectFilters(t *testing.T) { finder.FindRootsNames = []string{"file-3/pom.xml"} finder.FindDependencyDirsNames = []string{"file-3/test.class"} ctx, _ := ctxTestdata.NewContextMock() - s := NewStrategy(conf, testFiles, []string{"test"}, finder, ctx) + s := NewStrategy(conf, testFiles, []string{"test"}, []string{}, finder, ctx) jobs, _ := s.Invoke() assert.Len(t, jobs, 1) for _, job := range jobs { @@ -89,7 +89,7 @@ func TestBuildProjectsError(t *testing.T) { finder.FindRootsNames = []string{"file-3/pom.xml"} finder.FindDependencyDirsNames = []string{"file-3/test.class"} ctx, _ := ctxTestdata.NewContextMock() - s := NewStrategy(conf, testFiles, []string{"test"}, finder, ctx) + s := NewStrategy(conf, testFiles, []string{"test"}, []string{}, finder, ctx) factoryMock := javaTestdata.NewEchoCmdFactory() factoryMock.BuildMavenErr = fmt.Errorf("build-error") s.cmdFactory = factoryMock diff --git a/internal/callgraph/strategy/strategy_factory.go b/internal/callgraph/strategy/strategy_factory.go index 5e9e6c5d..97da8154 100644 --- a/internal/callgraph/strategy/strategy_factory.go +++ b/internal/callgraph/strategy/strategy_factory.go @@ -12,7 +12,7 @@ import ( ) type IFactory interface { - Make(config conf.IConfig, paths []string, exclusions []string, ctx cgexec.IContext) (IStrategy, error) + Make(config conf.IConfig, paths []string, exclusions []string, inclusions []string, ctx cgexec.IContext) (IStrategy, error) } type Factory struct{} @@ -21,13 +21,13 @@ func NewStrategyFactory() Factory { return Factory{} } -func (sf Factory) Make(config conf.IConfig, paths []string, exclusions []string, ctx cgexec.IContext) (IStrategy, error) { +func (sf Factory) Make(config conf.IConfig, paths []string, exclusions []string, inclusions []string, ctx cgexec.IContext) (IStrategy, error) { name := config.Language() switch name { case java.Name: - return java.NewStrategy(config, paths, exclusions, javafinder.JavaFinder{}, ctx), nil + return java.NewStrategy(config, paths, exclusions, inclusions, javafinder.JavaFinder{}, ctx), nil case golang.Name: - return golang.NewStrategy(config, paths, exclusions, golangfinder.GolangFinder{}, ctx), nil + return golang.NewStrategy(config, paths, exclusions, inclusions, golangfinder.GolangFinder{}, ctx), nil default: return nil, fmt.Errorf("failed to make strategy from %s", name) } diff --git a/internal/callgraph/strategy/strategy_factory_test.go b/internal/callgraph/strategy/strategy_factory_test.go index 3730d8a6..15172fd1 100644 --- a/internal/callgraph/strategy/strategy_factory_test.go +++ b/internal/callgraph/strategy/strategy_factory_test.go @@ -17,7 +17,7 @@ func TestNewStrategyFactory(t *testing.T) { func TestMakeErr(t *testing.T) { f := NewStrategyFactory() conf := config.NewConfig("test", nil, nil, true, "") - s, err := f.Make(conf, nil, nil, nil) + s, err := f.Make(conf, nil, nil, nil, nil) assert.Nil(t, s) assert.ErrorContains(t, err, "failed to make strategy from test") } @@ -25,12 +25,12 @@ func TestMakeErr(t *testing.T) { func TestMake(t *testing.T) { conf := config.NewConfig(java.Name, nil, nil, true, "") cases := map[string]IStrategy{ - java.Name: java.NewStrategy(conf, []string{}, []string{}, javafinder.JavaFinder{}, nil), + java.Name: java.NewStrategy(conf, []string{}, []string{}, []string{}, javafinder.JavaFinder{}, nil), } f := NewStrategyFactory() for name, strategy := range cases { t.Run(name, func(t *testing.T) { - s, err := f.Make(conf, []string{}, []string{}, nil) + s, err := f.Make(conf, []string{}, []string{}, []string{}, nil) assert.NoError(t, err) assert.Equal(t, strategy, s) }) diff --git a/internal/callgraph/strategy/testdata/strategy_mock_factory.go b/internal/callgraph/strategy/testdata/strategy_mock_factory.go index 49b69c30..2cba3b70 100644 --- a/internal/callgraph/strategy/testdata/strategy_mock_factory.go +++ b/internal/callgraph/strategy/testdata/strategy_mock_factory.go @@ -13,7 +13,7 @@ func NewStrategyFactoryMock() FactoryMock { return FactoryMock{} } -func (sf FactoryMock) Make(config config.IConfig, paths []string, exclusions []string, ctx cgexec.IContext) (strategy.IStrategy, error) { +func (sf FactoryMock) Make(config config.IConfig, paths []string, exclusions []string, inclusions []string, ctx cgexec.IContext) (strategy.IStrategy, error) { return NewStrategyMock(config, paths, testdata.FinderMock{}, ctx), nil } @@ -23,6 +23,6 @@ func NewStrategyFactoryErrorMock() FactoryErrorMock { return FactoryErrorMock{} } -func (sf FactoryErrorMock) Make(config config.IConfig, paths []string, exclusions []string, ctx cgexec.IContext) (strategy.IStrategy, error) { +func (sf FactoryErrorMock) Make(config config.IConfig, paths []string, exclusions []string, inclusions []string, ctx cgexec.IContext) (strategy.IStrategy, error) { return NewStrategyErrorMock(config, paths, testdata.FinderMock{}, ctx), nil } diff --git a/internal/callgraph/testdata/generator_mock.go b/internal/callgraph/testdata/generator_mock.go index aebe0c7d..3fa3203d 100644 --- a/internal/callgraph/testdata/generator_mock.go +++ b/internal/callgraph/testdata/generator_mock.go @@ -13,11 +13,11 @@ type GeneratorMock struct { files []string } -func (r *GeneratorMock) GenerateWithTimer(_ []string, _ []string, _ []config.IConfig, _ int) error { +func (r *GeneratorMock) GenerateWithTimer(_ []string, _ []string, _ []string, _ []config.IConfig, _ int) error { return r.Err } -func (r *GeneratorMock) Generate(_ []string, _ []string, _ []config.IConfig, _ cgexec.IContext) error { +func (r *GeneratorMock) Generate(_ []string, _ []string, _ []string, _ []config.IConfig, _ cgexec.IContext) error { for _, f := range r.files { createdFile, err := os.Create(f) if err != nil { diff --git a/internal/cmd/callgraph/callgraph.go b/internal/cmd/callgraph/callgraph.go index 551bc869..5e97413e 100644 --- a/internal/cmd/callgraph/callgraph.go +++ b/internal/cmd/callgraph/callgraph.go @@ -13,26 +13,27 @@ import ( "github.com/spf13/viper" ) -var exclusions = file.DefaultExclusions() +var ( + exclusions = file.DefaultExclusions() + inclusions []string + buildDisabled bool + generateTimeout int + languages string + supportedLanguages = []string{"java", "golang"} + languageMap = map[string]string{ + "java": "maven", + "golang": "go", + } +) const ( ExclusionFlag = "exclusion" + InclusionFlag = "inclusion" NoBuildFlag = "no-build" GenerateTimeoutFlag = "generate-timeout" LanguagesFlag = "languages" ) -var buildDisabled bool -var generateTimeout int -var languages string - -var supportedLanguages = []string{"java", "golang"} - -var languageMap = map[string]string{ - "java": "maven", - "golang": "go", -} - func NewCallgraphCmd(generator callgraph.IGenerator) *cobra.Command { cmd := &cobra.Command{ Use: "callgraph [path]", @@ -68,6 +69,13 @@ Exclude flags could alternatively be set using DEBRICKED_EXCLUSIONS="path1,path2 Example: $ debricked callgraph . `+exampleFlags) + cmd.Flags().StringArrayVar( + &inclusions, + InclusionFlag, + []string{}, + `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. +Examples: +$ debricked scan . --include /node_modules/`) cmd.Flags().BoolVar(&buildDisabled, NoBuildFlag, false, `Do not automatically build all source code in the project to enable call graph generation. This option requires a pre-built project. For more detailed documentation on the callgraph generation, visit our portal: https://portal.debricked.com/debricked-cli-63/debricked-cli-documentation-298?tid=298&fid=63#callgraph`) @@ -123,7 +131,7 @@ func RunE(callgraph callgraph.IGenerator) func(_ *cobra.Command, args []string) configs = append(configs, conf.NewConfig(language, args, map[string]string{}, !buildDisabled, languageMap[language])) } - err = callgraph.GenerateWithTimer(args, viper.GetStringSlice(ExclusionFlag), configs, viper.GetInt(GenerateTimeoutFlag)) + err = callgraph.GenerateWithTimer(args, viper.GetStringSlice(ExclusionFlag), viper.GetStringSlice(InclusionFlag), configs, viper.GetInt(GenerateTimeoutFlag)) return err } diff --git a/internal/cmd/callgraph/callgraph_test.go b/internal/cmd/callgraph/callgraph_test.go index 420421c9..ec854537 100644 --- a/internal/cmd/callgraph/callgraph_test.go +++ b/internal/cmd/callgraph/callgraph_test.go @@ -21,6 +21,7 @@ func TestNewCallgraphCmd(t *testing.T) { flags := cmd.Flags() flagAssertions := map[string]string{ ExclusionFlag: "e", + InclusionFlag: "", NoBuildFlag: "", GenerateTimeoutFlag: "", } diff --git a/internal/cmd/files/find/find.go b/internal/cmd/files/find/find.go index 91e5f3ce..dec52426 100644 --- a/internal/cmd/files/find/find.go +++ b/internal/cmd/files/find/find.go @@ -12,12 +12,14 @@ import ( ) var exclusions = file.Exclusions() +var inclusions []string var jsonPrint bool var lockfileOnly bool var strictness int const ( ExclusionFlag = "exclusion" + InclusionFlag = "inclusion" JsonFlag = "json" LockfileOnlyFlag = "lockfile" StrictFlag = "strict" @@ -50,7 +52,13 @@ Exclude flags could alternatively be set using DEBRICKED_EXCLUSIONS="path1,path2 Example: $ debricked files find . `+exampleFlags) - + cmd.Flags().StringArrayVar( + &inclusions, + InclusionFlag, + []string{}, + `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. +Examples: +$ debricked scan . --include /node_modules/`) cmd.Flags().BoolVarP(&jsonPrint, JsonFlag, "j", false, `Print files in JSON format Format: [ @@ -72,6 +80,7 @@ Strictness Level | Meaning `) viper.MustBindEnv(ExclusionFlag) + viper.MustBindEnv(InclusionFlag) viper.MustBindEnv(JsonFlag) viper.MustBindEnv(LockfileOnlyFlag) viper.MustBindEnv(StrictFlag) @@ -94,6 +103,7 @@ func RunE(f file.IFinder) func(_ *cobra.Command, args []string) error { fileGroups, err := f.GetGroups( path, viper.GetStringSlice(ExclusionFlag), + viper.GetStringSlice(InclusionFlag), viper.GetBool(LockfileOnlyFlag), viper.GetInt(StrictFlag), ) diff --git a/internal/cmd/fingerprint/fingerprint.go b/internal/cmd/fingerprint/fingerprint.go index 337a6638..97995d73 100644 --- a/internal/cmd/fingerprint/fingerprint.go +++ b/internal/cmd/fingerprint/fingerprint.go @@ -11,12 +11,14 @@ import ( ) var exclusions = file.DefaultExclusionsFingerprint() +var inclusions []string var shouldFingerprintCompressedContent bool var outputDir string var minFingerprintContentLength int const ( ExclusionFlag = "exclusion" + InclusionFlag = "inclusion" FingerprintCompressedContent = "fingerprint-compressed-content" OutputDirFlag = "output-dir" MinFingerprintContentLengthFlag = "min-fingerprint-content-length" @@ -49,6 +51,13 @@ Special Terms | Meaning Example: $ debricked files fingerprint . `+exampleFlags) + cmd.Flags().StringArrayVar( + &inclusions, + InclusionFlag, + []string{}, + `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. +Examples: +$ debricked scan . --include /node_modules/`) cmd.Flags().BoolVar(&shouldFingerprintCompressedContent, FingerprintCompressedContent, false, `Fingerprint the contents of compressed files by unpacking them in memory, Supported files: `+fmt.Sprintf("%v", fingerprint.ZIP_FILE_ENDINGS)) cmd.Flags().StringVar(&outputDir, OutputDirFlag, ".", "The directory to write the output file to") cmd.Flags().IntVar(&minFingerprintContentLength, MinFingerprintContentLengthFlag, 45, "Set minimum content length (in bytes) for files to fingerprint. Defaults to 45 bytes.") @@ -64,7 +73,7 @@ func RunE(f fingerprint.IFingerprint) func(_ *cobra.Command, args []string) erro path = args[0] } - output, err := f.FingerprintFiles(path, exclusions, shouldFingerprintCompressedContent, minFingerprintContentLength) + output, err := f.FingerprintFiles(path, exclusions, inclusions, shouldFingerprintCompressedContent, minFingerprintContentLength) if err != nil { return err diff --git a/internal/cmd/resolve/resolve.go b/internal/cmd/resolve/resolve.go index 10266f86..4b0f7313 100644 --- a/internal/cmd/resolve/resolve.go +++ b/internal/cmd/resolve/resolve.go @@ -13,6 +13,7 @@ import ( var ( exclusions = file.Exclusions() + inclusions []string verbose bool npmPreferred bool regenerate int @@ -21,6 +22,7 @@ var ( const ( ExclusionFlag = "exclusion" + InclusionFlag = "inclusion" VerboseFlag = "verbose" NpmPreferredFlag = "prefer-npm" RegenerateFlag = "regenerate" @@ -56,6 +58,13 @@ Exclude flags could alternatively be set using DEBRICKED_EXCLUSIONS="path1,path2 Example: $ debricked resolve . `+exampleFlags) + cmd.Flags().StringArrayVar( + &inclusions, + InclusionFlag, + []string{}, + `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. +Examples: +$ debricked scan . --include /node_modules/`) regenerateDoc := strings.Join( []string{ "Toggles regeneration of already existing lock files between 3 modes:\n", @@ -107,6 +116,7 @@ func RunE(resolver resolution.IResolver) func(_ *cobra.Command, args []string) e } options := resolution.DebrickedOptions{ Exclusions: viper.GetStringSlice(ExclusionFlag), + Inclusions: viper.GetStringSlice(InclusionFlag), Verbose: viper.GetBool(VerboseFlag), Regenerate: viper.GetInt(RegenerateFlag), NpmPreferred: viper.GetBool(NpmPreferredFlag), diff --git a/internal/cmd/root/root_test.go b/internal/cmd/root/root_test.go index 38914922..a9a49731 100644 --- a/internal/cmd/root/root_test.go +++ b/internal/cmd/root/root_test.go @@ -31,7 +31,7 @@ func TestNewRootCmd(t *testing.T) { } } assert.Truef(t, match, "failed to assert that flag was present: "+AccessTokenFlag) - assert.Len(t, viperKeys, 14) + assert.Len(t, viperKeys, 15) } func TestPreRun(t *testing.T) { diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index e07c279d..e96fa359 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -21,6 +21,7 @@ var repositoryUrl string var integrationName string var jsonFilePath string var exclusions = file.Exclusions() +var inclusions = file.Exclusions() var verbose bool var regenerate int var versionHint bool @@ -41,6 +42,7 @@ const ( RepositoryUrlFlag = "repository-url" IntegrationFlag = "integration" ExclusionFlag = "exclusion" + InclusionFlag = "inclusion" VerboseFlag = "verbose" VersionHintFlag = "version-hint" RegenerateFlag = "regenerate" @@ -108,6 +110,13 @@ Exclude flags could alternatively be set using DEBRICKED_EXCLUSIONS="path1,path2 Examples: $ debricked scan . `+exampleFlags) + cmd.Flags().StringArrayVar( + &inclusions, + InclusionFlag, + inclusions, + `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. +Examples: +$ debricked scan . --include /node_modules/`) regenerateDoc := strings.Join( []string{ "Toggles regeneration of already existing lock files between 3 modes:\n", @@ -144,7 +153,6 @@ For example, if there is a "go.mod" in the target path, its dependencies are goi "This flag allows you to select which package manager will be used as a resolver: Yarn (default) or NPM.", "Example: debricked resolve --prefer-npm", }, "\n") - cmd.Flags().BoolP(NpmPreferredFlag, "", npmPreferred, npmPreferredDoc) viper.MustBindEnv(RepositoryFlag) diff --git a/internal/file/exclusion.go b/internal/file/exclusion.go index 45dacb0d..8b7ec33b 100644 --- a/internal/file/exclusion.go +++ b/internal/file/exclusion.go @@ -9,6 +9,7 @@ import ( ) const debrickedExclusionEnvVar = "DEBRICKED_EXCLUSIONS" +const debrickedInclusionEnvVar = "DEBRICKED_INCLUSIONS" func DefaultExclusions() []string { return []string{ @@ -50,7 +51,14 @@ func DefaultExclusionsFingerprint() []string { return output } -func Excluded(exclusions []string, path string) bool { +func Excluded(exclusions []string, inclusions []string, path string) bool { + for _, inclusion := range inclusions { + ex := filepath.Clean(inclusion) + matched, _ := doublestar.PathMatch(ex, path) + if matched { + return false + } + } for _, exclusion := range exclusions { ex := filepath.Clean(exclusion) matched, _ := doublestar.PathMatch(ex, path) diff --git a/internal/file/exclusion_test.go b/internal/file/exclusion_test.go index df216b8d..685377d4 100644 --- a/internal/file/exclusion_test.go +++ b/internal/file/exclusion_test.go @@ -145,7 +145,7 @@ func TestExclude(t *testing.T) { t.Run(c.name, func(t *testing.T) { var excludedFiles []string for _, file := range files { - if Excluded(c.exclusions, file) { + if Excluded(c.exclusions, []string{}, file) { excludedFiles = append(excludedFiles, file) } } diff --git a/internal/file/finder.go b/internal/file/finder.go index 9d5f7a9d..a8ee51d5 100644 --- a/internal/file/finder.go +++ b/internal/file/finder.go @@ -23,9 +23,9 @@ const SupportedFormatsFallbackFilePath = "embedded/supported_formats.json" const SupportedFormatsUri = "/api/1.0/open/files/supported-formats" type IFinder interface { - GetGroups(rootPath string, exclusions []string, lockfileOnly bool, strictness int) (Groups, error) + GetGroups(rootPath string, exclusions []string, inclusions []string, lockfileOnly bool, strictness int) (Groups, error) GetSupportedFormats() ([]*CompiledFormat, error) - GetConfigPath(rootPath string, exclusions []string) string + GetConfigPath(rootPath string, exclusions []string, inclusions []string) string } type Finder struct { @@ -41,7 +41,7 @@ func NewFinder(c client.IDebClient, fs ioFs.IFileSystem) (*Finder, error) { return &Finder{c, fs}, nil } -func (finder *Finder) GetConfigPath(rootPath string, exclusions []string) string { +func (finder *Finder) GetConfigPath(rootPath string, exclusions []string, inclusions []string) string { var configPath string if len(rootPath) == 0 { @@ -53,7 +53,7 @@ func (finder *Finder) GetConfigPath(rootPath string, exclusions []string) string if err != nil { return err } - if !fileInfo.IsDir() && !Excluded(exclusions, path) { + if !fileInfo.IsDir() && !Excluded(exclusions, inclusions, path) { if filepath.Base(path) == "debricked-config.yaml" { configPath = path } @@ -70,7 +70,7 @@ func (finder *Finder) GetConfigPath(rootPath string, exclusions []string) string } // GetGroups return all file groups in specified path recursively. -func (finder *Finder) GetGroups(rootPath string, exclusions []string, lockfileOnly bool, strictness int) (Groups, error) { +func (finder *Finder) GetGroups(rootPath string, exclusions []string, inclusions []string, lockfileOnly bool, strictness int) (Groups, error) { var groups Groups formats, err := finder.GetSupportedFormats() @@ -88,7 +88,7 @@ func (finder *Finder) GetGroups(rootPath string, exclusions []string, lockfileOn if err != nil { return err } - if !fileInfo.IsDir() && !Excluded(exclusions, path) { + if !fileInfo.IsDir() && !Excluded(exclusions, inclusions, path) { for _, format := range formats { if groups.Match(format, path, lockfileOnly) { diff --git a/internal/file/finder_test.go b/internal/file/finder_test.go index cffc14f1..daf5f1f6 100644 --- a/internal/file/finder_test.go +++ b/internal/file/finder_test.go @@ -108,10 +108,11 @@ func TestGetGroups(t *testing.T) { path := "" exclusions := []string{"testdata/go/*.mod", "testdata/misc/**"} + inclusions := []string{"**/package.json"} excludedFiles := []string{"testdata/go/go.mod", "testdata/misc/requirements.txt", "testdata/misc/Cargo.lock"} - const nbrOfGroups = 5 + const nbrOfGroups = 6 - fileGroups, err := finder.GetGroups(path, exclusions, false, StrictAll) + fileGroups, err := finder.GetGroups(path, exclusions, inclusions, false, StrictAll) assert.NoError(t, err) assert.Equalf(t, nbrOfGroups, fileGroups.Size(), "failed to assert that %d groups were created. %d was found", nbrOfGroups, fileGroups.Size()) @@ -136,7 +137,7 @@ func TestGetGroupsPIP(t *testing.T) { const nbrOfGroups = 3 lockfileOnly := false - fileGroups, err := finder.GetGroups(path, []string{}, lockfileOnly, StrictAll) + fileGroups, err := finder.GetGroups(path, []string{}, []string{}, lockfileOnly, StrictAll) assert.NoError(t, err) assert.Equalf(t, nbrOfGroups, fileGroups.Size(), "failed to assert that %d groups were created. %d was found", nbrOfGroups, fileGroups.Size()) @@ -176,7 +177,7 @@ func TestGetGroupsWithOnlyLockFiles(t *testing.T) { setUp(true) path := "testdata/misc" const nbrOfGroups = 2 - fileGroups, err := finder.GetGroups(path, []string{"**/requirements*.txt", "**/composer.json", "**/composer.lock", "**/go.mod"}, false, StrictAll) + fileGroups, err := finder.GetGroups(path, []string{"**/requirements*.txt", "**/composer.json", "**/composer.lock", "**/go.mod"}, []string{}, false, StrictAll) assert.NoError(t, err) assert.Equalf(t, nbrOfGroups, fileGroups.Size(), "failed to assert that %d groups were created. %d was found", nbrOfGroups, fileGroups.Size()) @@ -193,7 +194,7 @@ func TestGetGroupsWithTwoFileMatchesInSameDir(t *testing.T) { setUp(true) path := "testdata/pip" const nbrOfGroups = 3 - fileGroups, err := finder.GetGroups(path, []string{}, false, StrictAll) + fileGroups, err := finder.GetGroups(path, []string{}, []string{}, false, StrictAll) assert.NoError(t, err) assert.Equalf(t, nbrOfGroups, fileGroups.Size(), "failed to assert that %d groups were created. %d was found", nbrOfGroups, fileGroups.Size()) @@ -220,7 +221,7 @@ func TestGetGroupsWithTwoFileMatchesInSameDir(t *testing.T) { func TestGetDebrickedConfig(t *testing.T) { path := "testdata" - configPath := finder.GetConfigPath(path, nil) + configPath := finder.GetConfigPath(path, nil, nil) assert.Equal(t, filepath.Join("testdata", "misc", "debricked-config.yaml"), configPath) } @@ -267,7 +268,7 @@ func TestGetGroupsWithStrictFlag(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { filePath := "testdata" - fileGroups, err := finder.GetGroups(filePath, []string{}, false, c.strictness) + fileGroups, err := finder.GetGroups(filePath, []string{}, []string{}, false, c.strictness) fileGroup := fileGroups.groups[c.testedGroupIndex] assert.Nilf(t, err, "failed to assert that no error occurred. Error: %s", err) diff --git a/internal/file/testdata/finder_mock.go b/internal/file/testdata/finder_mock.go index b9fdf814..d4139d18 100644 --- a/internal/file/testdata/finder_mock.go +++ b/internal/file/testdata/finder_mock.go @@ -19,11 +19,11 @@ func NewFinderMock() *FinderMock { } // GetGroups return all file groups in specified path recursively. -func (f *FinderMock) GetGroups(_ string, _ []string, _ bool, _ int) (file.Groups, error) { +func (f *FinderMock) GetGroups(_ string, _ []string, _ []string, _ bool, _ int) (file.Groups, error) { return f.groups, f.error } -func (f *FinderMock) GetConfigPath(_ string, _ []string) string { +func (f *FinderMock) GetConfigPath(_ string, _ []string, _ []string) string { return "" } diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index f6dc1057..64fd4314 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -139,7 +139,7 @@ func isInExcludedDir(path string) bool { type IFingerprint interface { FingerprintFiles( - rootPath string, exclusions []string, fingerprintCompressedContent bool, minFingerprintContentLength int, + rootPath string, exclusions []string, inclusions []string, fingerprintCompressedContent bool, minFingerprintContentLength int, ) (Fingerprints, error) } @@ -166,7 +166,7 @@ func (f FileFingerprint) ToString() string { } func (f *Fingerprinter) FingerprintFiles( - rootPath string, exclusions []string, fingerprintCompressedContent bool, minFingerprintContentLength int, + rootPath string, exclusions []string, inclusions []string, fingerprintCompressedContent bool, minFingerprintContentLength int, ) (Fingerprints, error) { if len(rootPath) == 0 { rootPath = filepath.Base("") @@ -185,7 +185,7 @@ func (f *Fingerprinter) FingerprintFiles( return err } - fileFingerprints, err := computeHashForFileAndZip(fileInfo, path, exclusions, fingerprintCompressedContent) + fileFingerprints, err := computeHashForFileAndZip(fileInfo, path, exclusions, inclusions, fingerprintCompressedContent) if err != nil { return err } @@ -223,31 +223,31 @@ func (f *Fingerprinter) FingerprintFiles( return fingerprints, err } -func computeHashForArchive(path string, exclusions []string) ([]FileFingerprint, error) { +func computeHashForArchive(path string, exclusions []string, inclusions []string) ([]FileFingerprint, error) { if isZipFile(path) { - return inMemFingerprintZipContent(path, exclusions) + return inMemFingerprintZipContent(path, exclusions, inclusions) } if isTarGZipFile(path) { - return inMemFingerprintTarGZipContent(path, exclusions) + return inMemFingerprintTarGZipContent(path, exclusions, inclusions) } if isTarBZip2File(path) { - return inMemFingerprintTarBZip2Content(path, exclusions) + return inMemFingerprintTarBZip2Content(path, exclusions, inclusions) } return nil, nil } func computeHashForFileAndZip( - fileInfo os.FileInfo, path string, exclusions []string, fingerprintCompressedContent bool, + fileInfo os.FileInfo, path string, exclusions []string, inclusions []string, fingerprintCompressedContent bool, ) ([]FileFingerprint, error) { - if !shouldProcessFile(fileInfo, exclusions, path) { + if !shouldProcessFile(fileInfo, exclusions, inclusions, path) { return nil, nil } var fingerprints []FileFingerprint if fingerprintCompressedContent { - fingerprintsArchive, err := computeHashForArchive(path, exclusions) + fingerprintsArchive, err := computeHashForArchive(path, exclusions, inclusions) if err != nil { if errors.Is(err, zip.ErrFormat) { fmt.Printf("WARNING: Could not unpack and fingerprint contents of compressed file [%s]. Error: %v\n", path, err) @@ -277,12 +277,12 @@ func isSymlink(filename string) (bool, error) { var isSymlinkFunc = isSymlink -func shouldProcessFile(fileInfo os.FileInfo, exclusions []string, path string) bool { +func shouldProcessFile(fileInfo os.FileInfo, exclusions []string, inclusions []string, path string) bool { if fileInfo.IsDir() { return false } - if file.Excluded(exclusions, path) { + if file.Excluded(exclusions, inclusions, path) { return false } @@ -402,21 +402,21 @@ func isTarBZip2File(filename string) bool { return false } -func shouldProcessTarHeader(header tar.Header, exclusions []string, longPath string) bool { +func shouldProcessTarHeader(header tar.Header, exclusions []string, inclusions []string, longPath string) bool { if header.Typeflag != tar.TypeReg { return false } if filepath.IsAbs(header.Name) || strings.HasPrefix(header.Name, "..") { return false } - if !shouldProcessFile(header.FileInfo(), exclusions, longPath) { + if !shouldProcessFile(header.FileInfo(), exclusions, inclusions, longPath) { return false } return true } -func inMemFingerprintTarBZip2Content(filename string, exclusions []string) ([]FileFingerprint, error) { +func inMemFingerprintTarBZip2Content(filename string, exclusions []string, inclusions []string) ([]FileFingerprint, error) { file, err := os.Open(filename) if err != nil { return nil, err @@ -434,7 +434,7 @@ func inMemFingerprintTarBZip2Content(filename string, exclusions []string) ([]Fi } longPath := filepath.Join(filename, header.Name) // #nosec fmt.Println("Extracted:", longPath) - if !shouldProcessTarHeader(*header, exclusions, longPath) { + if !shouldProcessTarHeader(*header, exclusions, inclusions, longPath) { continue } hasher := newHasher() @@ -454,7 +454,7 @@ func inMemFingerprintTarBZip2Content(filename string, exclusions []string) ([]Fi return fingerprints, nil } -func inMemFingerprintTarGZipContent(filename string, exclusions []string) ([]FileFingerprint, error) { +func inMemFingerprintTarGZipContent(filename string, exclusions []string, inclusions []string) ([]FileFingerprint, error) { file, err := os.Open(filename) if err != nil { return nil, err @@ -474,7 +474,7 @@ func inMemFingerprintTarGZipContent(filename string, exclusions []string) ([]Fil return nil, err } longPath := filepath.Join(filename, header.Name) // #nosec - if !shouldProcessTarHeader(*header, exclusions, longPath) { + if !shouldProcessTarHeader(*header, exclusions, inclusions, longPath) { continue } hasher := newHasher() @@ -494,7 +494,7 @@ func inMemFingerprintTarGZipContent(filename string, exclusions []string) ([]Fil return fingerprints, nil } -func inMemFingerprintZipContent(filename string, exclusions []string) ([]FileFingerprint, error) { +func inMemFingerprintZipContent(filename string, exclusions []string, inclusions []string) ([]FileFingerprint, error) { r, err := zip.OpenReader(filename) if err != nil { @@ -510,7 +510,7 @@ func inMemFingerprintZipContent(filename string, exclusions []string) ([]FileFin } longFileName := filepath.Join(filename, f.Name) // #nosec - if !shouldProcessFile(f.FileInfo(), exclusions, longFileName) { + if !shouldProcessFile(f.FileInfo(), exclusions, inclusions, longFileName) { continue } rc, err := f.Open() diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index 3c5a0368..933a352e 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -85,6 +85,7 @@ func TestShouldProcessFile(t *testing.T) { name string filePath string excludes []string + includes []string mock func() want bool }{ @@ -99,6 +100,7 @@ func TestShouldProcessFile(t *testing.T) { name: "Test with a symbolic link", filePath: testLink, excludes: []string{}, + includes: []string{}, mock: func() {}, want: false, }, @@ -106,13 +108,23 @@ func TestShouldProcessFile(t *testing.T) { name: "Test Excluded", filePath: testFile, excludes: []string{"**/test.py"}, + includes: []string{}, mock: func() {}, want: false, }, + { + name: "Test Excluded and Included", + filePath: testFile, + excludes: []string{"**/test.py"}, + includes: []string{"**/test.py"}, + mock: func() {}, + want: true, + }, { name: "Test with mockSymlink", filePath: testFile, excludes: []string{}, + includes: []string{}, mock: func() { isSymlinkFunc = mockSymlink }, want: false, }, @@ -120,6 +132,7 @@ func TestShouldProcessFile(t *testing.T) { name: "Test with errorString: The system cannot find the path specified.", filePath: testFile, excludes: []string{}, + includes: []string{}, mock: func() { errorString = "The system cannot find the path specified." }, want: true, }, @@ -127,6 +140,7 @@ func TestShouldProcessFile(t *testing.T) { name: "Test with errorString: not a directory", filePath: testFile, excludes: []string{}, + includes: []string{}, mock: func() { errorString = "not a directory" }, want: true, }, @@ -134,6 +148,7 @@ func TestShouldProcessFile(t *testing.T) { name: "Test with generic error", filePath: testFile, excludes: []string{}, + includes: []string{}, mock: func() { errorString = "generic error" }, want: false, }, @@ -147,7 +162,7 @@ func TestShouldProcessFile(t *testing.T) { if err != nil { t.Fatalf("Failed to get file info for %s: %v", tt.filePath, err) } - if got := shouldProcessFile(fileInfo, tt.excludes, tt.filePath); got != tt.want { + if got := shouldProcessFile(fileInfo, tt.excludes, tt.includes, tt.filePath); got != tt.want { t.Errorf("Expected shouldProcessFile to return %v for %s, but it returned %v", tt.want, tt.filePath, got) } }) @@ -167,7 +182,7 @@ func TestFingerprinterInterface(t *testing.T) { func TestFingerprintFiles(t *testing.T) { fingerprinter := NewFingerprinter() - fingerprints, err := fingerprinter.FingerprintFiles("testdata/fingerprinter", []string{}, true, 0) + fingerprints, err := fingerprinter.FingerprintFiles("testdata/fingerprinter", []string{}, []string{}, true, 0) assert.NoError(t, err) assert.NotNil(t, fingerprints) assert.NotEmpty(t, fingerprints) @@ -175,7 +190,7 @@ func TestFingerprintFiles(t *testing.T) { assert.Equal(t, "file=634c5485de8e22b27094affadd8a6e3b,21,testdata/fingerprinter/testfile.py", fingerprints.Entries[0].ToString()) // Test no file - fingerprints, err = fingerprinter.FingerprintFiles("", []string{}, true, 0) + fingerprints, err = fingerprinter.FingerprintFiles("", []string{}, []string{}, true, 0) assert.NoError(t, err) assert.NotNil(t, fingerprints) assert.NotEmpty(t, fingerprints) @@ -499,7 +514,7 @@ func TestInMemFingerprintingCompressedContent(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fingerprinter := NewFingerprinter() - fingerprints, err := fingerprinter.FingerprintFiles(tt.path, []string{}, tt.shouldUnzip, 0) + fingerprints, err := fingerprinter.FingerprintFiles(tt.path, []string{}, []string{}, tt.shouldUnzip, 0) assert.NoError(t, err) assert.NotNil(t, fingerprints) assert.NotEmpty(t, fingerprints) diff --git a/internal/fingerprint/testdata/fingerprinter_mock.go b/internal/fingerprint/testdata/fingerprinter_mock.go index 38058242..cdb2286e 100644 --- a/internal/fingerprint/testdata/fingerprinter_mock.go +++ b/internal/fingerprint/testdata/fingerprinter_mock.go @@ -15,7 +15,7 @@ func NewFingerprintMock() *FingerprintMock { } func (f *FingerprintMock) FingerprintFiles( - rootPath string, exclusions []string, fingerprintCompressedContent bool, minFingerprintContentLength int, + rootPath string, exclusions []string, inclusions []string, fingerprintCompressedContent bool, minFingerprintContentLength int, ) (fingerprint.Fingerprints, error) { return fingerprint.Fingerprints{}, f.error } diff --git a/internal/resolution/resolver.go b/internal/resolution/resolver.go index 78839495..3f36c2fe 100644 --- a/internal/resolution/resolver.go +++ b/internal/resolution/resolver.go @@ -60,6 +60,7 @@ type IOptions interface{} type DebrickedOptions struct { Path string Exclusions []string + Inclusions []string Verbose bool Regenerate int NpmPreferred bool @@ -146,7 +147,7 @@ func (r Resolver) Resolve(paths []string, options IOptions) (IResolution, error) if !ok { return nil, ErrBadOpts } - files, err := r.refinePaths(paths, dOptions.Exclusions, dOptions.Regenerate) + files, err := r.refinePaths(paths, dOptions.Exclusions, dOptions.Inclusions, dOptions.Regenerate) if err != nil { return nil, err } @@ -191,7 +192,7 @@ func (r Resolver) Resolve(paths []string, options IOptions) (IResolution, error) return resolution, err } -func (r Resolver) refinePaths(paths []string, exclusions []string, regenerate int) ([]string, error) { +func (r Resolver) refinePaths(paths []string, exclusions []string, inclusions []string, regenerate int) ([]string, error) { var fileSet = map[string]bool{} var dirs []string for _, arg := range paths { @@ -214,7 +215,7 @@ func (r Resolver) refinePaths(paths []string, exclusions []string, regenerate in } } - err := r.searchDirs(fileSet, dirs, exclusions, regenerate) + err := r.searchDirs(fileSet, dirs, exclusions, inclusions, regenerate) if err != nil { return nil, err } @@ -227,9 +228,9 @@ func (r Resolver) refinePaths(paths []string, exclusions []string, regenerate in return files, nil } -func (r Resolver) searchDirs(fileSet map[string]bool, dirs []string, exclusions []string, regenerate int) error { +func (r Resolver) searchDirs(fileSet map[string]bool, dirs []string, exclusions []string, inclusions []string, regenerate int) error { for _, dir := range dirs { - err := r.processDir(fileSet, dir, exclusions, regenerate) + err := r.processDir(fileSet, dir, exclusions, inclusions, regenerate) if err != nil { return err } @@ -238,10 +239,11 @@ func (r Resolver) searchDirs(fileSet map[string]bool, dirs []string, exclusions return nil } -func (r Resolver) processDir(fileSet map[string]bool, dir string, exclusions []string, regenerate int) error { +func (r Resolver) processDir(fileSet map[string]bool, dir string, exclusions []string, inclusions []string, regenerate int) error { fileGroups, err := r.finder.GetGroups( dir, exclusions, + inclusions, false, file.StrictAll, ) diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index fc31ccc7..9fcb520f 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -48,6 +48,7 @@ type DebrickedOptions struct { Fingerprint bool CallGraph bool Exclusions []string + Inclusions []string Verbose bool Regenerate int VersionHint bool @@ -145,6 +146,7 @@ func (dScanner *DebrickedScanner) scanResolve(options DebrickedOptions) error { Verbose: options.Verbose, Regenerate: options.Regenerate, Exclusions: options.Exclusions, + Inclusions: options.Inclusions, NpmPreferred: options.NpmPreferred, } if options.Resolve { @@ -164,7 +166,7 @@ func (dScanner *DebrickedScanner) scanFingerprint(options DebrickedOptions) erro return nil } fingerprints, err := dScanner.fingerprint.FingerprintFiles( - options.Path, file.DefaultExclusionsFingerprint(), false, options.MinFingerprintContentLength, + options.Path, file.DefaultExclusionsFingerprint(), options.Inclusions, false, options.MinFingerprintContentLength, ) if err != nil { return err @@ -199,13 +201,13 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g if path == "" { path = "." } - resErr := dScanner.callgraph.GenerateWithTimer([]string{path}, options.Exclusions, configs, timeout) + resErr := dScanner.callgraph.GenerateWithTimer([]string{path}, options.Exclusions, options.Inclusions, configs, timeout) if resErr != nil { return nil, resErr } } - fileGroups, err := dScanner.finder.GetGroups(options.Path, options.Exclusions, false, file.StrictAll) + fileGroups, err := dScanner.finder.GetGroups(options.Path, options.Exclusions, options.Inclusions, false, file.StrictAll) if err != nil { return nil, err } @@ -216,7 +218,7 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g IntegrationsName: options.IntegrationName, CallGraphUploadTimeout: options.CallGraphUploadTimeout, VersionHint: options.VersionHint, - DebrickedConfig: dScanner.getDebrickedConfig(options.Path, options.Exclusions), + DebrickedConfig: dScanner.getDebrickedConfig(options.Path, options.Exclusions, options.Inclusions), } result, err := (*dScanner.uploader).Upload(uploaderOptions) if err != nil { @@ -226,8 +228,8 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g return result, nil } -func (dScanner *DebrickedScanner) getDebrickedConfig(path string, exclusions []string) *upload.DebrickedConfig { - configPath := dScanner.finder.GetConfigPath(path, exclusions) +func (dScanner *DebrickedScanner) getDebrickedConfig(path string, exclusions []string, inclusions []string) *upload.DebrickedConfig { + configPath := dScanner.finder.GetConfigPath(path, exclusions, inclusions) if configPath == "" { return nil } diff --git a/internal/scan/scanner_test.go b/internal/scan/scanner_test.go index 38a3fa16..f95f23ad 100644 --- a/internal/scan/scanner_test.go +++ b/internal/scan/scanner_test.go @@ -385,6 +385,7 @@ func TestScanWithResolveErr(t *testing.T) { var dOptionsTemplate = DebrickedOptions{ Path: "path", Exclusions: nil, + Inclusions: nil, RepositoryName: "repository", CommitName: "commit", BranchName: "branch", @@ -519,6 +520,7 @@ func TestMapEnvToOptions(t *testing.T) { MapEnvToOptions(&c.opts, c.env) assert.Equal(t, c.template.Path, c.opts.Path) assert.Nil(t, c.opts.Exclusions) + assert.Nil(t, c.opts.Inclusions) assert.Equal(t, c.template.RepositoryName, c.opts.RepositoryName) assert.Equal(t, c.template.CommitName, c.opts.CommitName) assert.Equal(t, c.template.BranchName, c.opts.BranchName) @@ -653,6 +655,7 @@ func TestScanWithFingerprint(t *testing.T) { Resolve: true, Fingerprint: true, Exclusions: nil, + Inclusions: nil, RepositoryName: repositoryName, CommitName: commitName, BranchName: "", @@ -694,6 +697,7 @@ func TestScanWithFingerprintNoEnterprise(t *testing.T) { Resolve: true, Fingerprint: true, Exclusions: nil, + Inclusions: nil, RepositoryName: repositoryName, CommitName: commitName, BranchName: "", @@ -734,6 +738,7 @@ func TestScanWithCallgraph(t *testing.T) { Fingerprint: false, CallGraph: true, Exclusions: nil, + Inclusions: nil, RepositoryName: repositoryName, CommitName: commitName, BranchName: "", From 37c72a541f0535764614ffd6524291a6e87104ee Mon Sep 17 00:00:00 2001 From: filip Date: Sun, 24 Mar 2024 15:23:55 +0100 Subject: [PATCH 2/9] Refactor exclusion --- internal/cmd/fingerprint/fingerprint.go | 14 +- internal/file/exclusion.go | 44 +++--- internal/file/exclusion_test.go | 23 --- internal/fingerprint/exclusion.go | 82 +++++++++++ internal/fingerprint/fingerprint.go | 134 +++--------------- internal/fingerprint/fingerprint_test.go | 78 ++++------ .../testdata/fingerprinter_mock.go | 2 +- internal/scan/scanner.go | 8 +- 8 files changed, 163 insertions(+), 222 deletions(-) create mode 100644 internal/fingerprint/exclusion.go diff --git a/internal/cmd/fingerprint/fingerprint.go b/internal/cmd/fingerprint/fingerprint.go index 97995d73..4279e16c 100644 --- a/internal/cmd/fingerprint/fingerprint.go +++ b/internal/cmd/fingerprint/fingerprint.go @@ -4,13 +4,12 @@ import ( "fmt" "path/filepath" - "github.com/debricked/cli/internal/file" "github.com/debricked/cli/internal/fingerprint" "github.com/spf13/cobra" "github.com/spf13/viper" ) -var exclusions = file.DefaultExclusionsFingerprint() +var exclusions = fingerprint.DefaultExclusionsFingerprint() var inclusions []string var shouldFingerprintCompressedContent bool var outputDir string @@ -61,6 +60,7 @@ $ debricked scan . --include /node_modules/`) cmd.Flags().BoolVar(&shouldFingerprintCompressedContent, FingerprintCompressedContent, false, `Fingerprint the contents of compressed files by unpacking them in memory, Supported files: `+fmt.Sprintf("%v", fingerprint.ZIP_FILE_ENDINGS)) cmd.Flags().StringVar(&outputDir, OutputDirFlag, ".", "The directory to write the output file to") cmd.Flags().IntVar(&minFingerprintContentLength, MinFingerprintContentLengthFlag, 45, "Set minimum content length (in bytes) for files to fingerprint. Defaults to 45 bytes.") + viper.MustBindEnv(ExclusionFlag) return cmd @@ -72,8 +72,14 @@ func RunE(f fingerprint.IFingerprint) func(_ *cobra.Command, args []string) erro if len(args) > 0 { path = args[0] } - - output, err := f.FingerprintFiles(path, exclusions, inclusions, shouldFingerprintCompressedContent, minFingerprintContentLength) + options := fingerprint.DebrickedOptions{ + Path: path, + Exclusions: exclusions, + Inclusions: inclusions, + FingerprintCompressedContent: shouldFingerprintCompressedContent, + MinFingerprintContentLength: minFingerprintContentLength, + } + output, err := f.FingerprintFiles(options) if err != nil { return err diff --git a/internal/file/exclusion.go b/internal/file/exclusion.go index 8b7ec33b..23918665 100644 --- a/internal/file/exclusion.go +++ b/internal/file/exclusion.go @@ -9,16 +9,27 @@ import ( ) const debrickedExclusionEnvVar = "DEBRICKED_EXCLUSIONS" -const debrickedInclusionEnvVar = "DEBRICKED_INCLUSIONS" + +type DefaultExclusionList struct { + Directories []string +} + +var defaultExclusions = DefaultExclusionList{ + Directories: []string{ + "node_modules", + "vendor", + ".git", + "obj", // nuget + "bower_components", // bower + }, +} func DefaultExclusions() []string { - return []string{ - filepath.Join("**", "node_modules", "**"), - filepath.Join("**", "vendor", "**"), - filepath.Join("**", ".git", "**"), - filepath.Join("**", "obj", "**"), // nuget - filepath.Join("**", "bower_components", "**"), // bower + var exclusions []string + for _, excluded_dir := range defaultExclusions.Directories { + exclusions = append(exclusions, filepath.Join("**", excluded_dir, "**")) } + return exclusions } func Exclusions() []string { @@ -32,25 +43,6 @@ func Exclusions() []string { return values } -var EXCLUDED_DIRS_FINGERPRINT = []string{ - "nbproject", "nbbuild", "nbdist", "node_modules", - "__pycache__", "_yardoc", "eggs", - "wheels", "htmlcov", "__pypackages__", ".git"} - -var EXCLUDED_DIRS_FINGERPRINT_RAW = []string{"**/*.egg-info/**", "**/*venv/**", "**/*venv3/**"} - -func DefaultExclusionsFingerprint() []string { - output := []string{} - - for _, pattern := range EXCLUDED_DIRS_FINGERPRINT { - output = append(output, filepath.Join("**", pattern, "**")) - } - - output = append(output, EXCLUDED_DIRS_FINGERPRINT_RAW...) - - return output -} - func Excluded(exclusions []string, inclusions []string, path string) bool { for _, inclusion := range inclusions { ex := filepath.Clean(inclusion) diff --git a/internal/file/exclusion_test.go b/internal/file/exclusion_test.go index 685377d4..cf9bd623 100644 --- a/internal/file/exclusion_test.go +++ b/internal/file/exclusion_test.go @@ -64,29 +64,6 @@ func TestExclusionsWithEmptyTokenEnvVariable(t *testing.T) { assert.Equal(t, gt, defaultExclusions) } -func TestDefaultExclusionsFingerprint(t *testing.T) { - expectedExclusions := []string{ - filepath.Join("**", "nbproject", "**"), - filepath.Join("**", "nbbuild", "**"), - filepath.Join("**", "nbdist", "**"), - filepath.Join("**", "node_modules", "**"), - filepath.Join("**", "__pycache__", "**"), - filepath.Join("**", "_yardoc", "**"), - filepath.Join("**", "eggs", "**"), - filepath.Join("**", "wheels", "**"), - filepath.Join("**", "htmlcov", "**"), - filepath.Join("**", "__pypackages__", "**"), - filepath.Join("**", ".git", "**"), - "**/*.egg-info/**", - "**/*venv/**", - "**/*venv3/**", - } - - exclusions := DefaultExclusionsFingerprint() - - assert.ElementsMatch(t, expectedExclusions, exclusions, "DefaultExclusionsFingerprint did not return the expected exclusions") -} - func TestExclude(t *testing.T) { var files []string _ = filepath.Walk(".", diff --git a/internal/fingerprint/exclusion.go b/internal/fingerprint/exclusion.go new file mode 100644 index 00000000..8fa14e48 --- /dev/null +++ b/internal/fingerprint/exclusion.go @@ -0,0 +1,82 @@ +package fingerprint + +import "path/filepath" + +type DefaultFingerprintExclusionList struct { + Directories []string + Files []string + Endings []string + Extensions []string +} + +var defaultFingerprintExclusions = DefaultFingerprintExclusionList{ + Directories: []string{ + ".idea", + "nbproject", + "nbbuild", + "nddist", + "__pycache__", + "_yardoc", + "eggs", + "wheels", + "htmlcov", + "__pypackages__", + ".git", + "*.egg-info", + "*venv", + "*venv3", + }, + Files: []string{ + "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar", + "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile", + "\\[content_types\\].xml", "\\[Content_Types\\].xml", "py.typed", "LICENSE.APACHE2", "LICENSE.MIT", + }, + Endings: []string{ + "-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice", + "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3", + "nycrc", "targ", "eslintrc", "prettierrc", + }, + Extensions: []string{ + ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am", + ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".cmake", ".cnf", + ".conf", ".config", ".contributors", ".copying", ".crt", ".csproj", ".css", + ".csv", ".dat", ".data", ".doc", ".docx", ".dtd", ".dts", ".iws", ".c9", ".c9revisions", + ".dtsi", ".dump", ".eot", ".eps", ".geojson", ".gdoc", ".gif", + ".glif", ".gmo", ".gradle", ".guess", ".hex", ".htm", ".html", ".ico", ".iml", + ".in", ".inc", ".info", ".ini", ".ipynb", ".jpeg", ".jpg", ".json", ".jsonld", ".lock", + ".log", ".m4", ".map", ".markdown", ".md", ".md5", ".meta", ".mk", ".mxml", + ".o", ".otf", ".out", ".pbtxt", ".pdf", ".pem", ".phtml", ".plist", ".png", + ".po", ".ppt", ".prefs", ".properties", ".pyc", ".qdoc", ".result", ".rgb", + ".rst", ".scss", ".sha", ".sha1", ".sha2", ".sha256", ".sln", ".spec", ".sql", + ".sub", ".svg", ".svn-base", ".tab", ".template", ".test", ".tex", ".tiff", + ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl", ".woff", ".woff2", ".xht", + ".xhtml", ".xls", ".xlsx", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp", + ".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover", + ".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", ".pyi", + }, +} + +func DefaultExclusionsFingerprint() []string { + var default_exclusions []string + for _, excluded_dir := range defaultFingerprintExclusions.Directories { + default_exclusions = append(default_exclusions, filepath.Join("**", excluded_dir, "**")) + } + for _, excluded_file := range defaultFingerprintExclusions.Files { + default_exclusions = append(default_exclusions, filepath.Join("**", excluded_file)) + } + for _, excluded_extension := range defaultFingerprintExclusions.Extensions { + default_exclusions = append(default_exclusions, filepath.Join("**", "*"+excluded_extension)) + } + for _, excluded_ending := range defaultFingerprintExclusions.Extensions { + default_exclusions = append(default_exclusions, filepath.Join("**", "*"+excluded_ending)) + } + + return default_exclusions +} + +func DefaultInclusionsFingerprint() []string { + + return []string{ + "package.json", + } +} diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index 64fd4314..37ae1828 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -18,43 +18,12 @@ import ( "lukechampine.com/blake3" ) -var EXCLUDED_EXT = []string{ - ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am", - ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".cmake", ".cnf", - ".conf", ".config", ".contributors", ".copying", ".crt", ".csproj", ".css", - ".csv", ".dat", ".data", ".doc", ".docx", ".dtd", ".dts", ".iws", ".c9", ".c9revisions", - ".dtsi", ".dump", ".eot", ".eps", ".geojson", ".gdoc", ".gif", - ".glif", ".gmo", ".gradle", ".guess", ".hex", ".htm", ".html", ".ico", ".iml", - ".in", ".inc", ".info", ".ini", ".ipynb", ".jpeg", ".jpg", ".json", ".jsonld", ".lock", - ".log", ".m4", ".map", ".markdown", ".md", ".md5", ".meta", ".mk", ".mxml", - ".o", ".otf", ".out", ".pbtxt", ".pdf", ".pem", ".phtml", ".plist", ".png", - ".po", ".ppt", ".prefs", ".properties", ".pyc", ".qdoc", ".result", ".rgb", - ".rst", ".scss", ".sha", ".sha1", ".sha2", ".sha256", ".sln", ".spec", ".sql", - ".sub", ".svg", ".svn-base", ".tab", ".template", ".test", ".tex", ".tiff", - ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".woff", ".woff2", ".xht", - ".xhtml", ".xls", ".xlsx", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp", - ".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover", - ".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", "", ".pyi", -} - -var EXCLUDED_FILE_ENDINGS = []string{ - "-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice", - "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3", - "nycrc", "targ", "eslintrc", "prettierrc", -} - -var EXCLUDED_FILES = []string{ - "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar", - "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile", - "[content_types].xml", "py.typed", "LICENSE.APACHE2", "LICENSE.MIT", -} - -var EXCLUDED_DIRS = []string{ - ".idea", -} - -var INCLUDED_FILES = []string{ - "package.json", +type DebrickedOptions struct { + Path string + Exclusions []string + Inclusions []string + FingerprintCompressedContent bool + MinFingerprintContentLength int } var ZIP_FILE_ENDINGS = []string{".jar", ".nupkg", ".war", ".zip", ".ear", ".whl"} @@ -74,72 +43,9 @@ const ( OutputFileNameFingerprints = "debricked.fingerprints.txt" ) -func isExcludedFile(path string) bool { - - return (isExcludedByExtension(path) || - isExcludedByFilename(path) || - isExcludedByEnding(path) || - isInExcludedDir(path)) && !isIncludedFile(path) -} - -func isIncludedFile(path string) bool { - filename := filepath.Base(path) - for _, file := range INCLUDED_FILES { - if filename == file { - return true - } - } - - return false -} - -func isExcludedByExtension(path string) bool { - pathLower := strings.ToLower(path) - for _, format := range EXCLUDED_EXT { - if filepath.Ext(pathLower) == format { - return true - } - } - - return false -} - -func isExcludedByFilename(path string) bool { - filename := filepath.Base(path) - filenameLower := strings.ToLower(filename) - for _, file := range EXCLUDED_FILES { - if filenameLower == file { - return true - } - } - - return false -} - -func isExcludedByEnding(path string) bool { - pathLower := strings.ToLower(path) - for _, ending := range EXCLUDED_FILE_ENDINGS { - if strings.HasSuffix(pathLower, ending) { - return true - } - } - - return false -} - -func isInExcludedDir(path string) bool { - for _, dirname := range EXCLUDED_DIRS { - if strings.Contains(path, dirname) { - return true - } - } - - return false -} - type IFingerprint interface { FingerprintFiles( - rootPath string, exclusions []string, inclusions []string, fingerprintCompressedContent bool, minFingerprintContentLength int, + options DebrickedOptions, ) (Fingerprints, error) } @@ -165,11 +71,9 @@ func (f FileFingerprint) ToString() string { return fmt.Sprintf("file=%x,%d,%s", f.fingerprint, f.contentLength, path) } -func (f *Fingerprinter) FingerprintFiles( - rootPath string, exclusions []string, inclusions []string, fingerprintCompressedContent bool, minFingerprintContentLength int, -) (Fingerprints, error) { - if len(rootPath) == 0 { - rootPath = filepath.Base("") +func (f *Fingerprinter) FingerprintFiles(options DebrickedOptions) (Fingerprints, error) { + if len(options.Path) == 0 { + options.Path = filepath.Base("") } fingerprints := Fingerprints{} @@ -180,19 +84,19 @@ func (f *Fingerprinter) FingerprintFiles( nbFiles := 0 - err := filepath.Walk(rootPath, func(path string, fileInfo os.FileInfo, err error) error { + err := filepath.Walk(options.Path, func(path string, fileInfo os.FileInfo, err error) error { if err != nil { return err } - fileFingerprints, err := computeHashForFileAndZip(fileInfo, path, exclusions, inclusions, fingerprintCompressedContent) + fileFingerprints, err := computeHashForFileAndZip(fileInfo, path, options.Exclusions, options.Inclusions, options.FingerprintCompressedContent) if err != nil { return err } var filteredFileFingerprints []FileFingerprint for _, fileFingerprint := range fileFingerprints { - if fileFingerprint.contentLength >= int64(minFingerprintContentLength) { + if fileFingerprint.contentLength >= int64(options.MinFingerprintContentLength) { filteredFileFingerprints = append(filteredFileFingerprints, fileFingerprint) } } @@ -278,17 +182,17 @@ func isSymlink(filename string) (bool, error) { var isSymlinkFunc = isSymlink func shouldProcessFile(fileInfo os.FileInfo, exclusions []string, inclusions []string, path string) bool { + inclusions = append(inclusions, DefaultInclusionsFingerprint()...) + exclusions = append(exclusions, DefaultExclusionsFingerprint()...) if fileInfo.IsDir() { return false } - if file.Excluded(exclusions, inclusions, path) { return false } - - if isExcludedFile(path) { + if !strings.Contains(filepath.Base(path), ".") { return false - } + } // If no extension isSymlink, err := isSymlinkFunc(path) if err != nil { @@ -318,10 +222,6 @@ func computeHashForFile(filename string) (FileFingerprint, error) { contentLength := int64(len(data)) - if err != nil { - return FileFingerprint{}, err - } - return FileFingerprint{ path: filename, contentLength: contentLength, diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index 933a352e..a59320d6 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -11,52 +11,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestIsExcludedFile(t *testing.T) { - - // Test excluded file extensions - excludedExts := []string{".doc", ".pdf", ".txt", ""} - for _, ext := range excludedExts { - filename := "file" + ext - assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename) - } - - // Test excluded files - excludedFiles := []string{"LICENSE", "README.md", "Makefile", "mvnw", "[content_types].xml", "Stockholm", "hello.json"} - for _, filename := range excludedFiles { - assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename) - filepath := "foo/bar/" + filename - assert.True(t, isExcludedFile(filepath), "Expected %q to be excluded", filepath) - } - - // Test excluded file endings - excludedEndings := []string{"-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice", - "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3"} - for _, ending := range excludedEndings { - filename := "file." + ending - assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename) - } - - // Test excluded dirnames - filesInExcludedDir := []string{"package/.idea/test.txt"} - for _, filename := range filesInExcludedDir { - assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename) - } - - // Test included files - includedFiles := []string{"package.json"} - for _, filename := range includedFiles { - assert.False(t, isExcludedFile(filename), "Expected %q to not be excluded", filename) - filepath := "foo/bar/" + filename - assert.False(t, isExcludedFile(filepath), "Expected %q to not be excluded", filepath) - } - - // Test non-excluded files - assert.False(t, isExcludedFile("file.py"), "Expected file.txt to not be excluded") - assert.False(t, isExcludedFile("file.go"), "Expected .go to not be excluded") - assert.False(t, isExcludedFile("file.dll"), "Expected .dll to not be excluded") - assert.False(t, isExcludedFile("file.jar"), "Expected .jar to not be excluded") -} - var errorString = "mock error" // Test errors in symlink @@ -182,7 +136,15 @@ func TestFingerprinterInterface(t *testing.T) { func TestFingerprintFiles(t *testing.T) { fingerprinter := NewFingerprinter() - fingerprints, err := fingerprinter.FingerprintFiles("testdata/fingerprinter", []string{}, []string{}, true, 0) + fingerprints, err := fingerprinter.FingerprintFiles( + DebrickedOptions{ + Path: "testdata/fingerprinter", + Exclusions: []string{}, + Inclusions: []string{}, + FingerprintCompressedContent: true, + MinFingerprintContentLength: 0, + }, + ) assert.NoError(t, err) assert.NotNil(t, fingerprints) assert.NotEmpty(t, fingerprints) @@ -190,7 +152,15 @@ func TestFingerprintFiles(t *testing.T) { assert.Equal(t, "file=634c5485de8e22b27094affadd8a6e3b,21,testdata/fingerprinter/testfile.py", fingerprints.Entries[0].ToString()) // Test no file - fingerprints, err = fingerprinter.FingerprintFiles("", []string{}, []string{}, true, 0) + fingerprints, err = fingerprinter.FingerprintFiles( + DebrickedOptions{ + Path: "", + Exclusions: []string{}, + Inclusions: []string{}, + FingerprintCompressedContent: true, + MinFingerprintContentLength: 0, + }, + ) assert.NoError(t, err) assert.NotNil(t, fingerprints) assert.NotEmpty(t, fingerprints) @@ -484,7 +454,7 @@ func TestInMemFingerprintingCompressedContent(t *testing.T) { { name: "TGz", path: "testdata/archive/tgz", - expected: 1051, + expected: 984, suffix: "lodash.tgz", shouldUnzip: true, }, @@ -514,7 +484,15 @@ func TestInMemFingerprintingCompressedContent(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fingerprinter := NewFingerprinter() - fingerprints, err := fingerprinter.FingerprintFiles(tt.path, []string{}, []string{}, tt.shouldUnzip, 0) + fingerprints, err := fingerprinter.FingerprintFiles( + DebrickedOptions{ + Path: tt.path, + Exclusions: []string{}, + Inclusions: []string{}, + FingerprintCompressedContent: tt.shouldUnzip, + MinFingerprintContentLength: 45, + }, + ) assert.NoError(t, err) assert.NotNil(t, fingerprints) assert.NotEmpty(t, fingerprints) diff --git a/internal/fingerprint/testdata/fingerprinter_mock.go b/internal/fingerprint/testdata/fingerprinter_mock.go index cdb2286e..cf9df001 100644 --- a/internal/fingerprint/testdata/fingerprinter_mock.go +++ b/internal/fingerprint/testdata/fingerprinter_mock.go @@ -15,7 +15,7 @@ func NewFingerprintMock() *FingerprintMock { } func (f *FingerprintMock) FingerprintFiles( - rootPath string, exclusions []string, inclusions []string, fingerprintCompressedContent bool, minFingerprintContentLength int, + options fingerprint.DebrickedOptions, ) (fingerprint.Fingerprints, error) { return fingerprint.Fingerprints{}, f.error } diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index 9fcb520f..cc41c366 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -166,7 +166,13 @@ func (dScanner *DebrickedScanner) scanFingerprint(options DebrickedOptions) erro return nil } fingerprints, err := dScanner.fingerprint.FingerprintFiles( - options.Path, file.DefaultExclusionsFingerprint(), options.Inclusions, false, options.MinFingerprintContentLength, + fingerprint.DebrickedOptions{ + Path: options.Path, + Exclusions: append(options.Exclusions, fingerprint.DefaultExclusionsFingerprint()...), + Inclusions: append(options.Inclusions, fingerprint.DefaultInclusionsFingerprint()...), + MinFingerprintContentLength: options.MinFingerprintContentLength, + FingerprintCompressedContent: false, + }, ) if err != nil { return err From 1c868e331b9152628b09eec6ef43fb009d55f522 Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 25 Mar 2024 11:20:46 +0100 Subject: [PATCH 3/9] Refactor to reduce nbr of arguments --- internal/callgraph/generator.go | 26 ++++++--- internal/callgraph/generator_test.go | 42 ++++++++++++-- internal/callgraph/testdata/generator_mock.go | 6 +- internal/cmd/callgraph/callgraph.go | 13 ++++- internal/cmd/files/find/find.go | 12 ++-- internal/file/finder.go | 24 +++++--- internal/file/finder_test.go | 56 +++++++++++++++---- internal/file/testdata/finder_mock.go | 2 +- internal/fingerprint/fingerprint.go | 44 +++++++++------ internal/resolution/resolver.go | 26 +++++---- internal/scan/scanner.go | 20 ++++++- 11 files changed, 195 insertions(+), 76 deletions(-) diff --git a/internal/callgraph/generator.go b/internal/callgraph/generator.go index 8279c58f..55c71a9c 100644 --- a/internal/callgraph/generator.go +++ b/internal/callgraph/generator.go @@ -10,9 +10,17 @@ import ( "github.com/debricked/cli/internal/tui" ) +type DebrickedOptions struct { + Paths []string + Exclusions []string + Inclusions []string + Configs []config.IConfig + Timeout int +} + type IGenerator interface { - GenerateWithTimer(paths []string, exclusions []string, inclusions []string, configs []config.IConfig, timeout int) error - Generate(paths []string, exclusions []string, inclusions []string, configs []config.IConfig, ctx cgexec.IContext) error + GenerateWithTimer(options DebrickedOptions) error + Generate(options DebrickedOptions, ctx cgexec.IContext) error } type Generator struct { @@ -32,13 +40,13 @@ func NewGenerator( } } -func (g *Generator) GenerateWithTimer(paths []string, exclusions []string, inclusions []string, configs []config.IConfig, timeout int) error { +func (g *Generator) GenerateWithTimer(options DebrickedOptions) error { result := make(chan error) - ctx, cancel := cgexec.NewContext(timeout) + ctx, cancel := cgexec.NewContext(options.Timeout) defer cancel() go func() { - result <- g.Generate(paths, exclusions, inclusions, configs, &ctx) + result <- g.Generate(options, &ctx) }() // Wait for the result or timeout @@ -47,14 +55,14 @@ func (g *Generator) GenerateWithTimer(paths []string, exclusions []string, inclu return err } -func (g *Generator) Generate(paths []string, exclusions []string, inclusions []string, configs []config.IConfig, ctx cgexec.IContext) error { +func (g *Generator) Generate(options DebrickedOptions, ctx cgexec.IContext) error { targetPath := ".debrickedTmpFolder" debrickedExclusions := []string{targetPath} - exclusions = append(exclusions, debrickedExclusions...) + exclusions := append(options.Exclusions, debrickedExclusions...) var jobs []job.IJob - for _, config := range configs { - s, strategyErr := g.strategyFactory.Make(config, paths, exclusions, inclusions, ctx) + for _, config := range options.Configs { + s, strategyErr := g.strategyFactory.Make(config, options.Paths, exclusions, options.Inclusions, ctx) if strategyErr == nil { newJobs, err := s.Invoke() if err != nil { diff --git a/internal/callgraph/generator_test.go b/internal/callgraph/generator_test.go index 9dbebafb..b764d4a0 100644 --- a/internal/callgraph/generator_test.go +++ b/internal/callgraph/generator_test.go @@ -33,7 +33,13 @@ func TestGenerate(t *testing.T) { config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}, true, "maven"), } ctx, _ := ctxTestdata.NewContextMock() - err := g.Generate([]string{"../../go.mod"}, []string{}, nil, configs, ctx) + err := g.Generate( + DebrickedOptions{ + Paths: []string{"../../go.mod"}, + Exclusions: []string{}, + Inclusions: []string{}, + Configs: configs, + }, ctx) assert.NoError(t, err) assert.NotEmpty(t, g.Generation.Jobs()) } @@ -47,7 +53,15 @@ func TestGenerateWithTimer(t *testing.T) { configs := []config.IConfig{ config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}, true, "maven"), } - err := g.GenerateWithTimer([]string{"../../go.mod"}, []string{}, nil, configs, 1000) + err := g.GenerateWithTimer( + DebrickedOptions{ + Paths: []string{"../../go.mod"}, + Exclusions: nil, + Inclusions: nil, + Configs: configs, + Timeout: 1000, + }, + ) assert.NoError(t, err) assert.NotEmpty(t, g.Generation.Jobs()) } @@ -62,7 +76,13 @@ func TestGenerateInvokeError(t *testing.T) { config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}, true, "maven"), } ctx, _ := ctxTestdata.NewContextMock() - err := g.Generate([]string{"../../go.mod"}, []string{}, nil, configs, ctx) + err := g.Generate( + DebrickedOptions{ + Paths: []string{"../../go.mod"}, + Exclusions: []string{}, + Inclusions: []string{}, + Configs: configs, + }, ctx) assert.NotNil(t, err) } @@ -77,7 +97,13 @@ func TestGenerateScheduleError(t *testing.T) { config.NewConfig("java", []string{}, map[string]string{"pm": "maven"}, true, "maven"), } ctx, _ := ctxTestdata.NewContextMock() - err := g.Generate([]string{"../../go.mod"}, []string{}, nil, configs, ctx) + err := g.Generate( + DebrickedOptions{ + Paths: []string{"../../go.mod"}, + Exclusions: []string{}, + Inclusions: []string{}, + Configs: configs, + }, ctx) assert.NotEmpty(t, g.Generation.Jobs()) assert.ErrorIs(t, err, errAssertion) } @@ -89,7 +115,13 @@ func TestGenerateDirWithoutConfig(t *testing.T) { ) ctx, _ := ctxTestdata.NewContextMock() - err := g.Generate([]string{"invalid-dir"}, []string{}, nil, nil, ctx) + err := g.Generate( + DebrickedOptions{ + Paths: []string{"invalid-dir"}, + Exclusions: []string{}, + Inclusions: []string{}, + Configs: nil, + }, ctx) assert.Empty(t, g.Generation.Jobs()) assert.NoError(t, err) } diff --git a/internal/callgraph/testdata/generator_mock.go b/internal/callgraph/testdata/generator_mock.go index 3fa3203d..3ca85623 100644 --- a/internal/callgraph/testdata/generator_mock.go +++ b/internal/callgraph/testdata/generator_mock.go @@ -4,8 +4,8 @@ import ( "os" "path/filepath" + "github.com/debricked/cli/internal/callgraph" "github.com/debricked/cli/internal/callgraph/cgexec" - "github.com/debricked/cli/internal/callgraph/config" ) type GeneratorMock struct { @@ -13,11 +13,11 @@ type GeneratorMock struct { files []string } -func (r *GeneratorMock) GenerateWithTimer(_ []string, _ []string, _ []string, _ []config.IConfig, _ int) error { +func (r *GeneratorMock) GenerateWithTimer(_ callgraph.DebrickedOptions) error { return r.Err } -func (r *GeneratorMock) Generate(_ []string, _ []string, _ []string, _ []config.IConfig, _ cgexec.IContext) error { +func (r *GeneratorMock) Generate(_ callgraph.DebrickedOptions, _ cgexec.IContext) error { for _, f := range r.files { createdFile, err := os.Create(f) if err != nil { diff --git a/internal/cmd/callgraph/callgraph.go b/internal/cmd/callgraph/callgraph.go index 5e97413e..c07da02d 100644 --- a/internal/cmd/callgraph/callgraph.go +++ b/internal/cmd/callgraph/callgraph.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/debricked/cli/internal/callgraph" + cg "github.com/debricked/cli/internal/callgraph" conf "github.com/debricked/cli/internal/callgraph/config" "github.com/debricked/cli/internal/file" "github.com/spf13/cobra" @@ -34,7 +35,7 @@ const ( LanguagesFlag = "languages" ) -func NewCallgraphCmd(generator callgraph.IGenerator) *cobra.Command { +func NewCallgraphCmd(generator cg.IGenerator) *cobra.Command { cmd := &cobra.Command{ Use: "callgraph [path]", Short: "Generate a static call graph for the given directory and subdirectories", @@ -131,8 +132,14 @@ func RunE(callgraph callgraph.IGenerator) func(_ *cobra.Command, args []string) configs = append(configs, conf.NewConfig(language, args, map[string]string{}, !buildDisabled, languageMap[language])) } - err = callgraph.GenerateWithTimer(args, viper.GetStringSlice(ExclusionFlag), viper.GetStringSlice(InclusionFlag), configs, viper.GetInt(GenerateTimeoutFlag)) + options := cg.DebrickedOptions{ + Paths: args, + Exclusions: viper.GetStringSlice(ExclusionFlag), + Inclusions: viper.GetStringSlice(InclusionFlag), + Configs: configs, + Timeout: viper.GetInt(GenerateTimeoutFlag), + } - return err + return callgraph.GenerateWithTimer(options) } } diff --git a/internal/cmd/files/find/find.go b/internal/cmd/files/find/find.go index dec52426..bfeb652e 100644 --- a/internal/cmd/files/find/find.go +++ b/internal/cmd/files/find/find.go @@ -101,11 +101,13 @@ func RunE(f file.IFinder) func(_ *cobra.Command, args []string) error { } fileGroups, err := f.GetGroups( - path, - viper.GetStringSlice(ExclusionFlag), - viper.GetStringSlice(InclusionFlag), - viper.GetBool(LockfileOnlyFlag), - viper.GetInt(StrictFlag), + file.DebrickedOptions{ + RootPath: path, + Exclusions: viper.GetStringSlice(ExclusionFlag), + Inclusions: viper.GetStringSlice(InclusionFlag), + LockFileOnly: viper.GetBool(LockfileOnlyFlag), + Strictness: viper.GetInt(StrictFlag), + }, ) if err != nil { return err diff --git a/internal/file/finder.go b/internal/file/finder.go index a8ee51d5..982decef 100644 --- a/internal/file/finder.go +++ b/internal/file/finder.go @@ -22,8 +22,16 @@ var supportedFormats embed.FS const SupportedFormatsFallbackFilePath = "embedded/supported_formats.json" const SupportedFormatsUri = "/api/1.0/open/files/supported-formats" +type DebrickedOptions struct { + RootPath string + Exclusions []string + Inclusions []string + LockFileOnly bool + Strictness int +} + type IFinder interface { - GetGroups(rootPath string, exclusions []string, inclusions []string, lockfileOnly bool, strictness int) (Groups, error) + GetGroups(options DebrickedOptions) (Groups, error) GetSupportedFormats() ([]*CompiledFormat, error) GetConfigPath(rootPath string, exclusions []string, inclusions []string) string } @@ -70,27 +78,27 @@ func (finder *Finder) GetConfigPath(rootPath string, exclusions []string, inclus } // GetGroups return all file groups in specified path recursively. -func (finder *Finder) GetGroups(rootPath string, exclusions []string, inclusions []string, lockfileOnly bool, strictness int) (Groups, error) { +func (finder *Finder) GetGroups(options DebrickedOptions) (Groups, error) { var groups Groups formats, err := finder.GetSupportedFormats() if err != nil { return groups, err } - if len(rootPath) == 0 { - rootPath = filepath.Base("") + if len(options.RootPath) == 0 { + options.RootPath = filepath.Base("") } // Traverse files to find dependency file groups err = filepath.Walk( - rootPath, + options.RootPath, func(path string, fileInfo os.FileInfo, err error) error { if err != nil { return err } - if !fileInfo.IsDir() && !Excluded(exclusions, inclusions, path) { + if !fileInfo.IsDir() && !Excluded(options.Exclusions, options.Inclusions, path) { for _, format := range formats { - if groups.Match(format, path, lockfileOnly) { + if groups.Match(format, path, options.LockFileOnly) { break } @@ -101,7 +109,7 @@ func (finder *Finder) GetGroups(rootPath string, exclusions []string, inclusions }, ) - groups.FilterGroupsByStrictness(strictness) + groups.FilterGroupsByStrictness(options.Strictness) return groups, err } diff --git a/internal/file/finder_test.go b/internal/file/finder_test.go index daf5f1f6..85838bda 100644 --- a/internal/file/finder_test.go +++ b/internal/file/finder_test.go @@ -107,13 +107,18 @@ func TestGetGroups(t *testing.T) { setUp(true) path := "" - exclusions := []string{"testdata/go/*.mod", "testdata/misc/**"} - inclusions := []string{"**/package.json"} excludedFiles := []string{"testdata/go/go.mod", "testdata/misc/requirements.txt", "testdata/misc/Cargo.lock"} const nbrOfGroups = 6 - fileGroups, err := finder.GetGroups(path, exclusions, inclusions, false, StrictAll) - + fileGroups, err := finder.GetGroups( + DebrickedOptions{ + RootPath: path, + Exclusions: []string{"testdata/go/*.mod", "testdata/misc/**"}, + Inclusions: []string{"**/package.json"}, + LockFileOnly: false, + Strictness: StrictAll, + }, + ) assert.NoError(t, err) assert.Equalf(t, nbrOfGroups, fileGroups.Size(), "failed to assert that %d groups were created. %d was found", nbrOfGroups, fileGroups.Size()) @@ -136,8 +141,15 @@ func TestGetGroupsPIP(t *testing.T) { path := "testdata/pip" const nbrOfGroups = 3 - lockfileOnly := false - fileGroups, err := finder.GetGroups(path, []string{}, []string{}, lockfileOnly, StrictAll) + fileGroups, err := finder.GetGroups( + DebrickedOptions{ + RootPath: path, + Exclusions: []string{}, + Inclusions: []string{}, + LockFileOnly: false, + Strictness: StrictAll, + }, + ) assert.NoError(t, err) assert.Equalf(t, nbrOfGroups, fileGroups.Size(), "failed to assert that %d groups were created. %d was found", nbrOfGroups, fileGroups.Size()) @@ -177,7 +189,16 @@ func TestGetGroupsWithOnlyLockFiles(t *testing.T) { setUp(true) path := "testdata/misc" const nbrOfGroups = 2 - fileGroups, err := finder.GetGroups(path, []string{"**/requirements*.txt", "**/composer.json", "**/composer.lock", "**/go.mod"}, []string{}, false, StrictAll) + fileGroups, err := finder.GetGroups( + DebrickedOptions{ + RootPath: path, + Exclusions: []string{"**/requirements*.txt", "**/composer.json", "**/composer.lock", "**/go.mod"}, + Inclusions: []string{}, + LockFileOnly: false, + Strictness: StrictAll, + }, + ) + assert.NoError(t, err) assert.Equalf(t, nbrOfGroups, fileGroups.Size(), "failed to assert that %d groups were created. %d was found", nbrOfGroups, fileGroups.Size()) @@ -192,9 +213,16 @@ func TestGetGroupsWithOnlyLockFiles(t *testing.T) { func TestGetGroupsWithTwoFileMatchesInSameDir(t *testing.T) { setUp(true) - path := "testdata/pip" const nbrOfGroups = 3 - fileGroups, err := finder.GetGroups(path, []string{}, []string{}, false, StrictAll) + fileGroups, err := finder.GetGroups( + DebrickedOptions{ + RootPath: "testdata/pip", + Exclusions: []string{}, + Inclusions: []string{}, + LockFileOnly: false, + Strictness: StrictAll, + }, + ) assert.NoError(t, err) assert.Equalf(t, nbrOfGroups, fileGroups.Size(), "failed to assert that %d groups were created. %d was found", nbrOfGroups, fileGroups.Size()) @@ -268,7 +296,15 @@ func TestGetGroupsWithStrictFlag(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { filePath := "testdata" - fileGroups, err := finder.GetGroups(filePath, []string{}, []string{}, false, c.strictness) + fileGroups, err := finder.GetGroups( + DebrickedOptions{ + RootPath: filePath, + Exclusions: []string{}, + Inclusions: []string{}, + LockFileOnly: false, + Strictness: c.strictness, + }, + ) fileGroup := fileGroups.groups[c.testedGroupIndex] assert.Nilf(t, err, "failed to assert that no error occurred. Error: %s", err) diff --git a/internal/file/testdata/finder_mock.go b/internal/file/testdata/finder_mock.go index d4139d18..135d30d4 100644 --- a/internal/file/testdata/finder_mock.go +++ b/internal/file/testdata/finder_mock.go @@ -19,7 +19,7 @@ func NewFinderMock() *FinderMock { } // GetGroups return all file groups in specified path recursively. -func (f *FinderMock) GetGroups(_ string, _ []string, _ []string, _ bool, _ int) (file.Groups, error) { +func (f *FinderMock) GetGroups(_ file.DebrickedOptions) (file.Groups, error) { return f.groups, f.error } diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index 37ae1828..95549b13 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -88,27 +88,19 @@ func (f *Fingerprinter) FingerprintFiles(options DebrickedOptions) (Fingerprints if err != nil { return err } - - fileFingerprints, err := computeHashForFileAndZip(fileInfo, path, options.Exclusions, options.Inclusions, options.FingerprintCompressedContent) + fileFingerprints, err := fingerprintFile(path, fileInfo, options) if err != nil { return err } + nbFiles += len(fileFingerprints) - var filteredFileFingerprints []FileFingerprint - for _, fileFingerprint := range fileFingerprints { - if fileFingerprint.contentLength >= int64(options.MinFingerprintContentLength) { - filteredFileFingerprints = append(filteredFileFingerprints, fileFingerprint) - } + if nbFiles%100 == 0 { + f.spinnerManager.SetSpinnerMessage(spinner, spinnerMessage, fmt.Sprintf("%d", nbFiles)) } - if len(filteredFileFingerprints) != 0 { - fingerprints.Entries = append(fingerprints.Entries, filteredFileFingerprints...) + if len(fileFingerprints) != 0 { + fingerprints.Entries = append(fingerprints.Entries, fileFingerprints...) - nbFiles += len(filteredFileFingerprints) - - if nbFiles%100 == 0 { - f.spinnerManager.SetSpinnerMessage(spinner, spinnerMessage, fmt.Sprintf("%d", nbFiles)) - } } return nil @@ -127,6 +119,22 @@ func (f *Fingerprinter) FingerprintFiles(options DebrickedOptions) (Fingerprints return fingerprints, err } +func fingerprintFile(path string, fileInfo os.FileInfo, options DebrickedOptions) ([]FileFingerprint, error) { + fileFingerprints, err := computeHashForFileAndZip(fileInfo, path, options) + if err != nil { + return nil, err + } + + var filteredFileFingerprints []FileFingerprint + for _, fileFingerprint := range fileFingerprints { + if fileFingerprint.contentLength >= int64(options.MinFingerprintContentLength) { + filteredFileFingerprints = append(filteredFileFingerprints, fileFingerprint) + } + } + + return filteredFileFingerprints, nil +} + func computeHashForArchive(path string, exclusions []string, inclusions []string) ([]FileFingerprint, error) { if isZipFile(path) { return inMemFingerprintZipContent(path, exclusions, inclusions) @@ -142,16 +150,16 @@ func computeHashForArchive(path string, exclusions []string, inclusions []string } func computeHashForFileAndZip( - fileInfo os.FileInfo, path string, exclusions []string, inclusions []string, fingerprintCompressedContent bool, + fileInfo os.FileInfo, path string, options DebrickedOptions, ) ([]FileFingerprint, error) { - if !shouldProcessFile(fileInfo, exclusions, inclusions, path) { + if !shouldProcessFile(fileInfo, options.Exclusions, options.Inclusions, path) { return nil, nil } var fingerprints []FileFingerprint - if fingerprintCompressedContent { - fingerprintsArchive, err := computeHashForArchive(path, exclusions, inclusions) + if options.FingerprintCompressedContent { + fingerprintsArchive, err := computeHashForArchive(path, options.Exclusions, options.Inclusions) if err != nil { if errors.Is(err, zip.ErrFormat) { fmt.Printf("WARNING: Could not unpack and fingerprint contents of compressed file [%s]. Error: %v\n", path, err) diff --git a/internal/resolution/resolver.go b/internal/resolution/resolver.go index 3f36c2fe..e824d9d3 100644 --- a/internal/resolution/resolver.go +++ b/internal/resolution/resolver.go @@ -147,7 +147,7 @@ func (r Resolver) Resolve(paths []string, options IOptions) (IResolution, error) if !ok { return nil, ErrBadOpts } - files, err := r.refinePaths(paths, dOptions.Exclusions, dOptions.Inclusions, dOptions.Regenerate) + files, err := r.refinePaths(paths, dOptions) if err != nil { return nil, err } @@ -192,7 +192,7 @@ func (r Resolver) Resolve(paths []string, options IOptions) (IResolution, error) return resolution, err } -func (r Resolver) refinePaths(paths []string, exclusions []string, inclusions []string, regenerate int) ([]string, error) { +func (r Resolver) refinePaths(paths []string, options DebrickedOptions) ([]string, error) { var fileSet = map[string]bool{} var dirs []string for _, arg := range paths { @@ -215,7 +215,7 @@ func (r Resolver) refinePaths(paths []string, exclusions []string, inclusions [] } } - err := r.searchDirs(fileSet, dirs, exclusions, inclusions, regenerate) + err := r.searchDirs(fileSet, dirs, options) if err != nil { return nil, err } @@ -228,9 +228,9 @@ func (r Resolver) refinePaths(paths []string, exclusions []string, inclusions [] return files, nil } -func (r Resolver) searchDirs(fileSet map[string]bool, dirs []string, exclusions []string, inclusions []string, regenerate int) error { +func (r Resolver) searchDirs(fileSet map[string]bool, dirs []string, options DebrickedOptions) error { for _, dir := range dirs { - err := r.processDir(fileSet, dir, exclusions, inclusions, regenerate) + err := r.processDir(fileSet, dir, options) if err != nil { return err } @@ -239,18 +239,20 @@ func (r Resolver) searchDirs(fileSet map[string]bool, dirs []string, exclusions return nil } -func (r Resolver) processDir(fileSet map[string]bool, dir string, exclusions []string, inclusions []string, regenerate int) error { +func (r Resolver) processDir(fileSet map[string]bool, dir string, options DebrickedOptions) error { fileGroups, err := r.finder.GetGroups( - dir, - exclusions, - inclusions, - false, - file.StrictAll, + file.DebrickedOptions{ + RootPath: dir, + Exclusions: options.Exclusions, + Inclusions: options.Inclusions, + LockFileOnly: false, + Strictness: file.StrictAll, + }, ) if err != nil { return err } - r.processFileGroups(fileSet, fileGroups, regenerate) + r.processFileGroups(fileSet, fileGroups, options.Regenerate) return nil } diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index cc41c366..d55958c5 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -207,13 +207,29 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g if path == "" { path = "." } - resErr := dScanner.callgraph.GenerateWithTimer([]string{path}, options.Exclusions, options.Inclusions, configs, timeout) + resErr := dScanner.callgraph.GenerateWithTimer( + callgraph.DebrickedOptions{ + Paths: []string{path}, + Exclusions: options.Exclusions, + Inclusions: options.Inclusions, + Configs: configs, + Timeout: timeout, + }, + ) if resErr != nil { return nil, resErr } } - fileGroups, err := dScanner.finder.GetGroups(options.Path, options.Exclusions, options.Inclusions, false, file.StrictAll) + fileGroups, err := dScanner.finder.GetGroups( + file.DebrickedOptions{ + RootPath: options.Path, + Exclusions: options.Exclusions, + Inclusions: options.Inclusions, + LockFileOnly: false, + Strictness: file.StrictAll, + }, + ) if err != nil { return nil, err } From bb8477cf74f4f0979d457760898725f9cf14fab1 Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 25 Mar 2024 11:20:46 +0100 Subject: [PATCH 4/9] Refactor to reduce nbr of arguments --- internal/file/exclusion.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/file/exclusion.go b/internal/file/exclusion.go index 23918665..82632dd1 100644 --- a/internal/file/exclusion.go +++ b/internal/file/exclusion.go @@ -29,6 +29,7 @@ func DefaultExclusions() []string { for _, excluded_dir := range defaultExclusions.Directories { exclusions = append(exclusions, filepath.Join("**", excluded_dir, "**")) } + return exclusions } From 750df98e4e5275f8f53438af7bd45aa50139198b Mon Sep 17 00:00:00 2001 From: filip Date: Tue, 26 Mar 2024 19:02:35 +0100 Subject: [PATCH 5/9] Rebase fixes and windows fix --- internal/cmd/callgraph/callgraph.go | 6 ++-- internal/cmd/files/find/find.go | 6 ++-- internal/cmd/fingerprint/fingerprint.go | 8 ++--- internal/cmd/resolve/resolve.go | 6 ++-- internal/cmd/scan/scan.go | 6 ++-- internal/file/exclusion.go | 11 ++++--- internal/file/exclusion_test.go | 42 +++++++++++++++++++++--- internal/file/finder_test.go | 2 +- internal/fingerprint/exclusion.go | 15 +++++---- internal/fingerprint/fingerprint_test.go | 2 +- internal/scan/scanner.go | 12 +++---- internal/scan/testdata/npm/result.json | 7 ++++ 12 files changed, 82 insertions(+), 41 deletions(-) create mode 100644 internal/scan/testdata/npm/result.json diff --git a/internal/cmd/callgraph/callgraph.go b/internal/cmd/callgraph/callgraph.go index c07da02d..f1427ea0 100644 --- a/internal/cmd/callgraph/callgraph.go +++ b/internal/cmd/callgraph/callgraph.go @@ -53,8 +53,8 @@ $ debricked callgraph }, RunE: RunE(generator), } - fileExclusionExample := filepath.Join("*", "**.lock") - dirExclusionExample := filepath.Join("**", "node_modules", "**") + fileExclusionExample := filepath.Join("'*", "**.lock'") + dirExclusionExample := filepath.Join("'**", "node_modules", "**'") exampleFlags := fmt.Sprintf("-e \"%s\" -e \"%s\"", fileExclusionExample, dirExclusionExample) cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "e", exclusions, `The following terms are supported to exclude paths: @@ -76,7 +76,7 @@ $ debricked callgraph . `+exampleFlags) []string{}, `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. Examples: -$ debricked scan . --include /node_modules/`) +$ debricked scan . --include '**/node_modules/**'`) cmd.Flags().BoolVar(&buildDisabled, NoBuildFlag, false, `Do not automatically build all source code in the project to enable call graph generation. This option requires a pre-built project. For more detailed documentation on the callgraph generation, visit our portal: https://portal.debricked.com/debricked-cli-63/debricked-cli-documentation-298?tid=298&fid=63#callgraph`) diff --git a/internal/cmd/files/find/find.go b/internal/cmd/files/find/find.go index bfeb652e..f5e49639 100644 --- a/internal/cmd/files/find/find.go +++ b/internal/cmd/files/find/find.go @@ -36,8 +36,8 @@ For example ` + "`package.json`" + ` with ` + "`package-lock.json`.", }, RunE: RunE(finder), } - fileExclusionExample := filepath.Join("*", "**.lock") - dirExclusionExample := filepath.Join("**", "node_modules", "**") + fileExclusionExample := filepath.Join("'*", "**.lock'") + dirExclusionExample := filepath.Join("'**", "node_modules", "**'") exampleFlags := fmt.Sprintf("-e \"%s\" -e \"%s\"", fileExclusionExample, dirExclusionExample) cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "e", exclusions, `The following terms are supported to exclude paths: Special Terms | Meaning @@ -58,7 +58,7 @@ $ debricked files find . `+exampleFlags) []string{}, `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. Examples: -$ debricked scan . --include /node_modules/`) +$ debricked scan . --include '**/node_modules/**'`) cmd.Flags().BoolVarP(&jsonPrint, JsonFlag, "j", false, `Print files in JSON format Format: [ diff --git a/internal/cmd/fingerprint/fingerprint.go b/internal/cmd/fingerprint/fingerprint.go index 4279e16c..536b5cd8 100644 --- a/internal/cmd/fingerprint/fingerprint.go +++ b/internal/cmd/fingerprint/fingerprint.go @@ -36,10 +36,10 @@ func NewFingerprintCmd(fingerprinter fingerprint.IFingerprint) *cobra.Command { }, RunE: RunE(fingerprinter), } - fileExclusionExample := filepath.Join("*", "**.pyc") - dirExclusionExample := filepath.Join("**", "node_modules", "**") + fileExclusionExample := filepath.Join("'*", "**.pyc'") + dirExclusionExample := filepath.Join("'**", "node_modules", "**'") exampleFlags := fmt.Sprintf("-%s \"%s\" -%s \"%s\"", ExclusionFlag, fileExclusionExample, ExclusionFlag, dirExclusionExample) - cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "", exclusions, `The following terms are supported to exclude paths: + cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "e", exclusions, `The following terms are supported to exclude paths: Special Terms | Meaning ------------- | ------- "*" | matches any sequence of non-Separator characters @@ -56,7 +56,7 @@ $ debricked files fingerprint . `+exampleFlags) []string{}, `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. Examples: -$ debricked scan . --include /node_modules/`) +$ debricked scan . --include '**/node_modules/**'`) cmd.Flags().BoolVar(&shouldFingerprintCompressedContent, FingerprintCompressedContent, false, `Fingerprint the contents of compressed files by unpacking them in memory, Supported files: `+fmt.Sprintf("%v", fingerprint.ZIP_FILE_ENDINGS)) cmd.Flags().StringVar(&outputDir, OutputDirFlag, ".", "The directory to write the output file to") cmd.Flags().IntVar(&minFingerprintContentLength, MinFingerprintContentLengthFlag, 45, "Set minimum content length (in bytes) for files to fingerprint. Defaults to 45 bytes.") diff --git a/internal/cmd/resolve/resolve.go b/internal/cmd/resolve/resolve.go index 4b0f7313..282a76f6 100644 --- a/internal/cmd/resolve/resolve.go +++ b/internal/cmd/resolve/resolve.go @@ -42,8 +42,8 @@ $ debricked resolve go.mod pkg/ }, RunE: RunE(resolver), } - fileExclusionExample := filepath.Join("*", "**.lock") - dirExclusionExample := filepath.Join("**", "node_modules", "**") + fileExclusionExample := filepath.Join("'*", "**.lock'") + dirExclusionExample := filepath.Join("'**", "node_modules", "**'") exampleFlags := fmt.Sprintf("-e \"%s\" -e \"%s\"", fileExclusionExample, dirExclusionExample) cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "e", exclusions, `The following terms are supported to exclude paths: Special Terms | Meaning @@ -64,7 +64,7 @@ $ debricked resolve . `+exampleFlags) []string{}, `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. Examples: -$ debricked scan . --include /node_modules/`) +$ debricked scan . --include '**/node_modules/**'`) regenerateDoc := strings.Join( []string{ "Toggles regeneration of already existing lock files between 3 modes:\n", diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index e96fa359..85368c34 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -89,8 +89,8 @@ If the given path contains a git repository all flags but "integration" will be `name of integration used to trigger scan. For example "GitHub Actions"`, ) cmd.Flags().StringVarP(&jsonFilePath, JsonFilePathFlag, "j", "", "write upload result as json to provided path") - fileExclusionExample := filepath.Join("*", "**.lock") - dirExclusionExample := filepath.Join("**", "node_modules", "**") + fileExclusionExample := filepath.Join("'*", "**.lock'") + dirExclusionExample := filepath.Join("'**", "node_modules", "**'") exampleFlags := fmt.Sprintf("-e \"%s\" -e \"%s\"", fileExclusionExample, dirExclusionExample) cmd.Flags().StringArrayVarP( &exclusions, @@ -116,7 +116,7 @@ $ debricked scan . `+exampleFlags) inclusions, `Forces inclusion of specified terms, see exclusion flag for more information on supported terms. Examples: -$ debricked scan . --include /node_modules/`) +$ debricked scan . --include '**/node_modules/**'`) regenerateDoc := strings.Join( []string{ "Toggles regeneration of already existing lock files between 3 modes:\n", diff --git a/internal/file/exclusion.go b/internal/file/exclusion.go index 82632dd1..727cee8b 100644 --- a/internal/file/exclusion.go +++ b/internal/file/exclusion.go @@ -27,7 +27,7 @@ var defaultExclusions = DefaultExclusionList{ func DefaultExclusions() []string { var exclusions []string for _, excluded_dir := range defaultExclusions.Directories { - exclusions = append(exclusions, filepath.Join("**", excluded_dir, "**")) + exclusions = append(exclusions, "**/"+excluded_dir+"/**") } return exclusions @@ -45,16 +45,17 @@ func Exclusions() []string { } func Excluded(exclusions []string, inclusions []string, path string) bool { + path = filepath.ToSlash(path) for _, inclusion := range inclusions { - ex := filepath.Clean(inclusion) - matched, _ := doublestar.PathMatch(ex, path) + inclusion = filepath.ToSlash(inclusion) + matched, _ := doublestar.Match(inclusion, path) if matched { return false } } for _, exclusion := range exclusions { - ex := filepath.Clean(exclusion) - matched, _ := doublestar.PathMatch(ex, path) + exclusion = filepath.ToSlash(exclusion) + matched, _ := doublestar.Match(exclusion, path) if matched { return true } diff --git a/internal/file/exclusion_test.go b/internal/file/exclusion_test.go index cf9bd623..762b4fd4 100644 --- a/internal/file/exclusion_test.go +++ b/internal/file/exclusion_test.go @@ -54,11 +54,11 @@ func TestExclusionsWithEmptyTokenEnvVariable(t *testing.T) { }(debrickedExclusionEnvVar, oldEnvValue) gt := []string{ - filepath.Join("**", "node_modules", "**"), - filepath.Join("**", "vendor", "**"), - filepath.Join("**", ".git", "**"), - filepath.Join("**", "obj", "**"), - filepath.Join("**", "bower_components", "**"), + "**/node_modules/**", + "**/vendor/**", + "**/.git/**", + "**/obj/**", + "**/bower_components/**", } defaultExclusions := Exclusions() assert.Equal(t, gt, defaultExclusions) @@ -145,3 +145,35 @@ func TestExclude(t *testing.T) { }) } } + +func TestExcluded(t *testing.T) { + cases := []struct { + name string + exclusions []string + inclusions []string + path string + expected bool + }{ + { + name: "NodeModules", + exclusions: []string{"**/node_modules/**"}, + inclusions: []string{}, + path: "node_modules/package.json", + expected: true, + }, + { + name: "Inclusions", + exclusions: []string{"**/node_modules/**"}, + inclusions: []string{"**/package.json"}, + path: "node_modules/package.json", + expected: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.expected, Excluded(c.exclusions, c.inclusions, c.path)) + }) + } + +} diff --git a/internal/file/finder_test.go b/internal/file/finder_test.go index 85838bda..d61e8c58 100644 --- a/internal/file/finder_test.go +++ b/internal/file/finder_test.go @@ -299,7 +299,7 @@ func TestGetGroupsWithStrictFlag(t *testing.T) { fileGroups, err := finder.GetGroups( DebrickedOptions{ RootPath: filePath, - Exclusions: []string{}, + Exclusions: []string{"**/node_modules/**"}, Inclusions: []string{}, LockFileOnly: false, Strictness: c.strictness, diff --git a/internal/fingerprint/exclusion.go b/internal/fingerprint/exclusion.go index 8fa14e48..742fd562 100644 --- a/internal/fingerprint/exclusion.go +++ b/internal/fingerprint/exclusion.go @@ -1,7 +1,5 @@ package fingerprint -import "path/filepath" - type DefaultFingerprintExclusionList struct { Directories []string Files []string @@ -25,6 +23,11 @@ var defaultFingerprintExclusions = DefaultFingerprintExclusionList{ "*.egg-info", "*venv", "*venv3", + "node_modules", + "vendor", + ".git", + "obj", + "bower_components", }, Files: []string{ "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar", @@ -59,16 +62,16 @@ var defaultFingerprintExclusions = DefaultFingerprintExclusionList{ func DefaultExclusionsFingerprint() []string { var default_exclusions []string for _, excluded_dir := range defaultFingerprintExclusions.Directories { - default_exclusions = append(default_exclusions, filepath.Join("**", excluded_dir, "**")) + default_exclusions = append(default_exclusions, "**/"+excluded_dir+"/**") } for _, excluded_file := range defaultFingerprintExclusions.Files { - default_exclusions = append(default_exclusions, filepath.Join("**", excluded_file)) + default_exclusions = append(default_exclusions, "**/"+excluded_file) } for _, excluded_extension := range defaultFingerprintExclusions.Extensions { - default_exclusions = append(default_exclusions, filepath.Join("**", "*"+excluded_extension)) + default_exclusions = append(default_exclusions, "**/*"+excluded_extension) } for _, excluded_ending := range defaultFingerprintExclusions.Extensions { - default_exclusions = append(default_exclusions, filepath.Join("**", "*"+excluded_ending)) + default_exclusions = append(default_exclusions, "**/*"+excluded_ending) } return default_exclusions diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go index a59320d6..26baab33 100644 --- a/internal/fingerprint/fingerprint_test.go +++ b/internal/fingerprint/fingerprint_test.go @@ -446,7 +446,7 @@ func TestInMemFingerprintingCompressedContent(t *testing.T) { }, { name: "Nupkg", - path: "testdata/archive/nupkg", + path: filepath.Join("testdata", "archive", "nupkg"), expected: 21, suffix: "newtonsoft.json.13.0.3.nupkg", shouldUnzip: true, diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index d55958c5..c6ba0150 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -222,13 +222,11 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g } fileGroups, err := dScanner.finder.GetGroups( - file.DebrickedOptions{ - RootPath: options.Path, - Exclusions: options.Exclusions, - Inclusions: options.Inclusions, - LockFileOnly: false, - Strictness: file.StrictAll, - }, + options.Path, + options.Exclusions, + options.Inclusions, + false, + file.StrictAll, ) if err != nil { return nil, err diff --git a/internal/scan/testdata/npm/result.json b/internal/scan/testdata/npm/result.json new file mode 100644 index 00000000..65696454 --- /dev/null +++ b/internal/scan/testdata/npm/result.json @@ -0,0 +1,7 @@ +{ + "vulnerabilitiesFound": 0, + "unaffectedVulnerabilitiesFound": 0, + "automationsAction": "", + "automationRules": null, + "detailsUrl": "" +} \ No newline at end of file From 2d94325baf254faba5567eae22d2faef7dcb89ee Mon Sep 17 00:00:00 2001 From: filip-debricked <135233582+filip-debricked@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:40:36 +0100 Subject: [PATCH 6/9] Delete internal/scan/testdata/npm/result.json --- internal/scan/testdata/npm/result.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 internal/scan/testdata/npm/result.json diff --git a/internal/scan/testdata/npm/result.json b/internal/scan/testdata/npm/result.json deleted file mode 100644 index 65696454..00000000 --- a/internal/scan/testdata/npm/result.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "vulnerabilitiesFound": 0, - "unaffectedVulnerabilitiesFound": 0, - "automationsAction": "", - "automationRules": null, - "detailsUrl": "" -} \ No newline at end of file From 6226ebb6b9071e99bcac81b0860807748401a764 Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 1 Apr 2024 15:37:52 +0200 Subject: [PATCH 7/9] Windows fix --- internal/file/exclusion.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/file/exclusion.go b/internal/file/exclusion.go index 727cee8b..8d442466 100644 --- a/internal/file/exclusion.go +++ b/internal/file/exclusion.go @@ -47,14 +47,12 @@ func Exclusions() []string { func Excluded(exclusions []string, inclusions []string, path string) bool { path = filepath.ToSlash(path) for _, inclusion := range inclusions { - inclusion = filepath.ToSlash(inclusion) matched, _ := doublestar.Match(inclusion, path) if matched { return false } } for _, exclusion := range exclusions { - exclusion = filepath.ToSlash(exclusion) matched, _ := doublestar.Match(exclusion, path) if matched { return true From 717bed13f633bc0d2bd297f331b37b69b88bfe72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Hed=C3=A9n?= Date: Wed, 24 Apr 2024 15:14:48 +0200 Subject: [PATCH 8/9] Rebase fixes --- internal/fingerprint/exclusion.go | 2 +- internal/scan/scanner.go | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/fingerprint/exclusion.go b/internal/fingerprint/exclusion.go index 742fd562..ce79a96e 100644 --- a/internal/fingerprint/exclusion.go +++ b/internal/fingerprint/exclusion.go @@ -52,7 +52,7 @@ var defaultFingerprintExclusions = DefaultFingerprintExclusionList{ ".po", ".ppt", ".prefs", ".properties", ".pyc", ".qdoc", ".result", ".rgb", ".rst", ".scss", ".sha", ".sha1", ".sha2", ".sha256", ".sln", ".spec", ".sql", ".sub", ".svg", ".svn-base", ".tab", ".template", ".test", ".tex", ".tiff", - ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl", ".woff", ".woff2", ".xht", + ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".woff", ".woff2", ".xht", ".xhtml", ".xls", ".xlsx", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp", ".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover", ".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", ".pyi", diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index c6ba0150..d55958c5 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -222,11 +222,13 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g } fileGroups, err := dScanner.finder.GetGroups( - options.Path, - options.Exclusions, - options.Inclusions, - false, - file.StrictAll, + file.DebrickedOptions{ + RootPath: options.Path, + Exclusions: options.Exclusions, + Inclusions: options.Inclusions, + LockFileOnly: false, + Strictness: file.StrictAll, + }, ) if err != nil { return nil, err From dd2854231325c97230f22ffac86da3b24d73fab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Hed=C3=A9n?= Date: Wed, 24 Apr 2024 16:37:44 +0200 Subject: [PATCH 9/9] Updated release notes for inclusion --- UPGRADE-2.0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/UPGRADE-2.0.md b/UPGRADE-2.0.md index 7c555c63..ad72e78f 100644 --- a/UPGRADE-2.0.md +++ b/UPGRADE-2.0.md @@ -3,6 +3,8 @@ ## Changed behaviours - Changes default strictness of resolve command to 1 (Exit with code 1 if all files failed to resolve, otherwise exit with code 0 instead of always exiting with code 0) - File Fingerprint analysis is on by default, rolling roll-out starting with all repositories that start with the letter "C". +- Added inclusion option to commands to force include patterns which are by default ignored by the CLI +- Refactored how exclusion works for fingerprinting to align it with the rest of the CLI, this includes a breaking change for windows where Unix path separators must be used in patterns. ## Runtime upgrades