diff --git a/.gitignore b/.gitignore index 31742f5c..d6e0d762 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,9 @@ internal/cmd/scan/testdata/npm/yarn.lock internal/file/embedded/supported_formats.json internal/resolution/pm/gradle/.gradle-init-script.debricked.groovy internal/callgraph/language/java11/testdata/mvnproj/target +test/resolve/testdata/composer/composer.lock test/resolve/testdata/npm/yarn.lock +test/resolve/testdata/npm/package-lock.json test/resolve/testdata/nuget/packages.lock.json test/resolve/testdata/nuget/csproj/packages.lock.json test/resolve/testdata/nuget/packagesconfig/packages.config.nuget.debricked.lock @@ -25,9 +27,13 @@ debricked.fingerprints.txt test/resolve/testdata/gomod/gomod.debricked.lock test/resolve/testdata/maven/maven.debricked.lock test/callgraph/**/maven.debricked.lock +internal/file/testdata/**/go.sum internal/file/testdata/**/gomod.debricked.lock internal/file/testdata/**/yarn-error.log +internal/scan/composer/**/yarn.lock internal/scan/testdata/**/yarn.lock +internal/scan/testdata/**/package-lock.json +internal/scan/testdata/**/debricked.fingerprints.wfp test/resolve/testdata/gradle/*/** **.gradle-init-script.debricked.groovy test/resolve/testdata/gradle/gradle.debricked.lock diff --git a/build/docker/alpine.Dockerfile b/build/docker/alpine.Dockerfile index 5626c70f..be931486 100644 --- a/build/docker/alpine.Dockerfile +++ b/build/docker/alpine.Dockerfile @@ -46,12 +46,13 @@ RUN apk --no-cache --update add \ py3-pip \ go~=1.21 \ nodejs \ + npm \ yarn \ dotnet7-sdk \ g++ \ curl -RUN dotnet --version +RUN dotnet --version && npm -v && yarn -v RUN apk add --no-cache \ git \ diff --git a/build/docker/debian.Dockerfile b/build/docker/debian.Dockerfile index df9028f4..a75ec198 100644 --- a/build/docker/debian.Dockerfile +++ b/build/docker/debian.Dockerfile @@ -51,6 +51,8 @@ RUN apt -y update && apt -y upgrade && apt -y install nodejs && \ apt -y clean && rm -rf /var/lib/apt/lists/* RUN npm install --global npm@latest && npm install --global yarn +RUN npm -v && yarn -v + # https://learn.microsoft.com/en-us/dotnet/core/install/linux-scripted-manual#scripted-install # https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian # Package manager installs are only supported on the x64 architecture. Other architectures, such as Arm, must install .NET by some other means such as with Snap, an installer script, or through a manual binary installation. diff --git a/internal/cmd/resolve/resolve.go b/internal/cmd/resolve/resolve.go index 2067049a..59923783 100644 --- a/internal/cmd/resolve/resolve.go +++ b/internal/cmd/resolve/resolve.go @@ -3,6 +3,7 @@ package resolve import ( "fmt" "path/filepath" + "strings" "github.com/debricked/cli/internal/file" "github.com/debricked/cli/internal/resolution" @@ -10,12 +11,16 @@ import ( "github.com/spf13/viper" ) -var exclusions = file.Exclusions() -var verbose bool +var ( + exclusions = file.Exclusions() + verbose bool + npmPreferred bool +) const ( - ExclusionFlag = "exclusion" - VerboseFlag = "verbose" + ExclusionFlag = "exclusion" + VerboseFlag = "verbose" + NpmPreferredFlag = "prefer-npm" ) func NewResolveCmd(resolver resolution.IResolver) *cobra.Command { @@ -49,7 +54,16 @@ Example: $ debricked resolve . `+exampleFlags) cmd.Flags().BoolVar(&verbose, VerboseFlag, true, "set to false to disable extensive resolution error messages") + npmPreferredDoc := strings.Join( + []string{ + "This flag allows you to select which package manager will be used as a resolver: Yarn (default) or NPM.", + "Example: debricked resolve --prefer-npm", + }, "\n") + + cmd.Flags().BoolP(NpmPreferredFlag, "", npmPreferred, npmPreferredDoc) + viper.MustBindEnv(ExclusionFlag) + viper.MustBindEnv(NpmPreferredFlag) return cmd } @@ -59,6 +73,8 @@ func RunE(resolver resolution.IResolver) func(_ *cobra.Command, args []string) e if len(args) == 0 { args = append(args, ".") } + + resolver.SetNpmPreferred(viper.GetBool(NpmPreferredFlag)) _, err := resolver.Resolve(args, viper.GetStringSlice(ExclusionFlag), viper.GetBool(VerboseFlag)) return err diff --git a/internal/cmd/root/root_test.go b/internal/cmd/root/root_test.go index 38914922..a9a49731 100644 --- a/internal/cmd/root/root_test.go +++ b/internal/cmd/root/root_test.go @@ -31,7 +31,7 @@ func TestNewRootCmd(t *testing.T) { } } assert.Truef(t, match, "failed to assert that flag was present: "+AccessTokenFlag) - assert.Len(t, viperKeys, 14) + assert.Len(t, viperKeys, 15) } func TestPreRun(t *testing.T) { diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index 493281af..7d3f2fa5 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" "github.com/debricked/cli/internal/file" "github.com/debricked/cli/internal/scan" @@ -24,6 +25,7 @@ var noResolve bool var noFingerprint bool var passOnDowntime bool var callgraph bool +var npmPreferred bool var callgraphUploadTimeout int var callgraphGenerateTimeout int @@ -42,6 +44,7 @@ const ( CallGraphFlag = "callgraph" CallGraphUploadTimeoutFlag = "callgraph-upload-timeout" CallGraphGenerateTimeoutFlag = "callgraph-generate-timeout" + NpmPreferredFlag = "prefer-npm" ) var scanCmdError error @@ -101,6 +104,14 @@ For example, if there is a "go.mod" in the target path, its dependencies are goi cmd.Flags().IntVar(&callgraphUploadTimeout, CallGraphUploadTimeoutFlag, 10*60, "Set a timeout (in seconds) on call graph upload.") cmd.Flags().IntVar(&callgraphGenerateTimeout, CallGraphGenerateTimeoutFlag, 60*60, "Set a timeout (in seconds) on call graph generation.") + npmPreferredDoc := strings.Join( + []string{ + "This flag allows you to select which package manager will be used as a resolver: Yarn (default) or NPM.", + "Example: debricked resolve --prefer-npm", + }, "\n") + + cmd.Flags().BoolP(NpmPreferredFlag, "", npmPreferred, npmPreferredDoc) + viper.MustBindEnv(RepositoryFlag) viper.MustBindEnv(CommitFlag) viper.MustBindEnv(BranchFlag) @@ -108,6 +119,7 @@ For example, if there is a "go.mod" in the target path, its dependencies are goi viper.MustBindEnv(RepositoryUrlFlag) viper.MustBindEnv(IntegrationFlag) viper.MustBindEnv(PassOnTimeOut) + viper.MustBindEnv(NpmPreferredFlag) return cmd } @@ -130,6 +142,7 @@ func RunE(s *scan.IScanner) func(_ *cobra.Command, args []string) error { CommitAuthor: viper.GetString(CommitAuthorFlag), RepositoryUrl: viper.GetString(RepositoryUrlFlag), IntegrationName: viper.GetString(IntegrationFlag), + NpmPreferred: viper.GetBool(NpmPreferredFlag), PassOnTimeOut: viper.GetBool(PassOnTimeOut), CallGraph: viper.GetBool(CallGraphFlag), CallGraphUploadTimeout: viper.GetInt(CallGraphUploadTimeoutFlag), diff --git a/internal/resolution/file/file_batch_factory.go b/internal/resolution/file/file_batch_factory.go index 22818166..f2cfe9c1 100644 --- a/internal/resolution/file/file_batch_factory.go +++ b/internal/resolution/file/file_batch_factory.go @@ -5,26 +5,38 @@ import ( "regexp" "github.com/debricked/cli/internal/resolution/pm" + "github.com/debricked/cli/internal/resolution/pm/npm" + "github.com/debricked/cli/internal/resolution/pm/yarn" ) type IBatchFactory interface { Make(files []string) []IBatch + SetNpmPreferred(npmPreferred bool) } type BatchFactory struct { - pms []pm.IPm + pms []pm.IPm + npmPreferred bool } -func NewBatchFactory() BatchFactory { - return BatchFactory{ +func NewBatchFactory() *BatchFactory { + return &BatchFactory{ pms: pm.Pms(), } } -func (bf BatchFactory) Make(files []string) []IBatch { +func (bf *BatchFactory) SetNpmPreferred(npmPreferred bool) { + bf.npmPreferred = npmPreferred +} + +func (bf *BatchFactory) Make(files []string) []IBatch { batchMap := make(map[string]IBatch) for _, file := range files { for _, p := range bf.pms { + if bf.skipPackageManager(p) { + continue + } + for _, manifest := range p.Manifests() { compiledRegex, _ := regexp.Compile(manifest) if compiledRegex.MatchString(path.Base(file)) { @@ -47,3 +59,16 @@ func (bf BatchFactory) Make(files []string) []IBatch { return batches } + +func (bf *BatchFactory) skipPackageManager(p pm.IPm) bool { + name := p.Name() + + switch true { + case name == npm.Name && !bf.npmPreferred: + return true + case name == yarn.Name && bf.npmPreferred: + return true + } + + return false +} diff --git a/internal/resolution/file/testdata/file_batch_factory_mock.go b/internal/resolution/file/testdata/file_batch_factory_mock.go index c0ed5511..7ff29d53 100644 --- a/internal/resolution/file/testdata/file_batch_factory_mock.go +++ b/internal/resolution/file/testdata/file_batch_factory_mock.go @@ -15,6 +15,9 @@ func NewBatchFactoryMock() BatchFactoryMock { } } +func (bf BatchFactoryMock) SetNpmPreferred(_ bool) { +} + func (bf BatchFactoryMock) Make(_ []string) []file.IBatch { return []file.IBatch{} diff --git a/internal/resolution/pm/npm/cmd_factory.go b/internal/resolution/pm/npm/cmd_factory.go new file mode 100644 index 00000000..be62bc69 --- /dev/null +++ b/internal/resolution/pm/npm/cmd_factory.go @@ -0,0 +1,44 @@ +package npm + +import ( + "os/exec" + "path/filepath" +) + +type ICmdFactory interface { + MakeInstallCmd(command string, file string) (*exec.Cmd, error) +} + +type IExecPath interface { + LookPath(file string) (string, error) +} + +type ExecPath struct { +} + +func (ExecPath) LookPath(file string) (string, error) { + return exec.LookPath(file) +} + +type CmdFactory struct { + execPath IExecPath +} + +func (cmdf CmdFactory) MakeInstallCmd(command string, file string) (*exec.Cmd, error) { + path, err := cmdf.execPath.LookPath(command) + + fileDir := filepath.Dir(file) + + return &exec.Cmd{ + Path: path, + Args: []string{ + //"yes |", // Answer 'y' to any prompts... + command, + "install", + "--ignore-scripts", // Avoid risky scripts + "--audit=false", // Do not run audit + "--bin-links=false", // We don't need symlinks to binaries as we won't run any code + }, + Dir: fileDir, + }, err +} diff --git a/internal/resolution/pm/npm/cmd_factory_test.go b/internal/resolution/pm/npm/cmd_factory_test.go new file mode 100644 index 00000000..fc37ea15 --- /dev/null +++ b/internal/resolution/pm/npm/cmd_factory_test.go @@ -0,0 +1,19 @@ +package npm + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMakeInstallCmd(t *testing.T) { + npmCommand := "npm" + cmd, err := CmdFactory{ + execPath: ExecPath{}, + }.MakeInstallCmd(npmCommand, "file") + assert.NoError(t, err) + assert.NotNil(t, cmd) + args := cmd.Args + assert.Contains(t, args, "npm") + assert.Contains(t, args, "install") +} diff --git a/internal/resolution/pm/npm/job.go b/internal/resolution/pm/npm/job.go new file mode 100644 index 00000000..95febf61 --- /dev/null +++ b/internal/resolution/pm/npm/job.go @@ -0,0 +1,174 @@ +package npm + +import ( + "regexp" + "strings" + + "github.com/debricked/cli/internal/resolution/job" + "github.com/debricked/cli/internal/resolution/pm/util" +) + +const ( + npm = "npm" + versionNotFoundErrRegex = `notarget [\w\s]+ ([^"\s:]+).` + dependencyNotFoundErrRegex = `404\s+'([^"\s:]+)'` + registryUnavailableErrRegex = `EAI_AGAIN ([\w\.]+)` + permissionDeniedErrRegex = `Error: EACCES, open '([^"\s:]+)'` +) + +type Job struct { + job.BaseJob + install bool + npmCommand string + cmdFactory ICmdFactory +} + +func NewJob( + file string, + install bool, + cmdFactory ICmdFactory, +) *Job { + return &Job{ + BaseJob: job.NewBaseJob(file), + install: install, + cmdFactory: cmdFactory, + } +} + +func (j *Job) Install() bool { + return j.install +} + +func (j *Job) Run() { + if j.install { + status := "installing dependencies" + j.SendStatus(status) + j.npmCommand = npm + + installCmd, err := j.cmdFactory.MakeInstallCmd(j.npmCommand, j.GetFile()) + + if err != nil { + j.handleError(j.createError(err.Error(), installCmd.String(), status)) + + return + } + + if output, err := installCmd.Output(); err != nil { + error := strings.Join([]string{string(output), j.GetExitError(err, "").Error()}, "") + j.handleError(j.createError(error, installCmd.String(), status)) + + return + } + } +} + +func (j *Job) createError(error string, cmd string, status string) job.IError { + cmdError := util.NewPMJobError(error) + cmdError.SetCommand(cmd) + cmdError.SetStatus(status) + + return cmdError +} + +func (j *Job) handleError(cmdError job.IError) { + expressions := []string{ + versionNotFoundErrRegex, + dependencyNotFoundErrRegex, + registryUnavailableErrRegex, + permissionDeniedErrRegex, + } + + for _, expression := range expressions { + regex := regexp.MustCompile(expression) + matches := regex.FindAllStringSubmatch(cmdError.Error(), -1) + + if len(matches) > 0 { + cmdError = j.addDocumentation(expression, matches, cmdError) + j.Errors().Append(cmdError) + + return + } + } + + j.Errors().Append(cmdError) +} + +func (j *Job) addDocumentation(expr string, matches [][]string, cmdError job.IError) job.IError { + documentation := cmdError.Documentation() + + switch expr { + case versionNotFoundErrRegex: + documentation = getVersionNotFoundErrorDocumentation(matches) + case dependencyNotFoundErrRegex: + documentation = getDependencyNotFoundErrorDocumentation(matches) + case registryUnavailableErrRegex: + documentation = getRegistryUnavailableErrorDocumentation(matches) + case permissionDeniedErrRegex: + documentation = getPermissionDeniedErrorDocumentation(matches) + } + + cmdError.SetDocumentation(documentation) + + return cmdError +} + +func getDependencyNotFoundErrorDocumentation(matches [][]string) string { + dependency := "" + if len(matches) > 0 && len(matches[0]) > 1 { + dependency = matches[0][1] + } + + return strings.Join( + []string{ + "Failed to find package", + "\"" + dependency + "\"", + "that satisfies the requirement from dependencies.", + "Please check that dependencies are correct in your package.json file.", + "\n" + util.InstallPrivateDependencyMessage, + }, " ") +} + +func getVersionNotFoundErrorDocumentation(matches [][]string) string { + dependency := "" + if len(matches) > 0 && len(matches[0]) > 1 { + dependency = matches[0][1] + } + + return strings.Join( + []string{ + "Failed to find package", + "\"" + dependency + "\"", + "that satisfies the requirement from package.json file.", + "In most cases you or one of your dependencies are requesting a package version that doesn't exist.", + "Please check that package versions are correct in your package.json file.", + }, " ") +} + +func getRegistryUnavailableErrorDocumentation(matches [][]string) string { + registry := "" + if len(matches) > 0 && len(matches[0]) > 1 { + registry = matches[0][1] + } + + return strings.Join( + []string{ + "Package registry", + "\"" + registry + "\"", + "is not available at the moment.", + "There might be a trouble with your network connection.", + }, " ") +} + +func getPermissionDeniedErrorDocumentation(matches [][]string) string { + path := "" + if len(matches) > 0 && len(matches[0]) > 1 { + path = matches[0][1] + } + + return strings.Join( + []string{ + "Couldn't get access to", + "\"" + path + "\".", + "Please check permissions or try running this command again as root/Administrator.", + }, " ") +} diff --git a/internal/resolution/pm/npm/job_test.go b/internal/resolution/pm/npm/job_test.go new file mode 100644 index 00000000..a526687e --- /dev/null +++ b/internal/resolution/pm/npm/job_test.go @@ -0,0 +1,100 @@ +package npm + +import ( + "errors" + "testing" + + jobTestdata "github.com/debricked/cli/internal/resolution/job/testdata" + "github.com/debricked/cli/internal/resolution/pm/npm/testdata" + "github.com/debricked/cli/internal/resolution/pm/util" + "github.com/stretchr/testify/assert" +) + +const ( + badName = "bad-name" +) + +func TestNewJob(t *testing.T) { + j := NewJob("file", false, CmdFactory{ + execPath: ExecPath{}, + }) + assert.Equal(t, "file", j.GetFile()) + assert.False(t, j.Errors().HasError()) +} + +func TestInstall(t *testing.T) { + j := Job{install: true} + assert.Equal(t, true, j.Install()) + + j = Job{install: false} + assert.Equal(t, false, j.Install()) +} + +func TestRunInstallCmdErr(t *testing.T) { + cases := []struct { + name string + error string + doc string + }{ + { + name: "General error", + error: "cmd-error", + doc: util.UnknownError, + }, + { + name: "Invalid package version", + error: "npm ERR! code ETARGET\nnpm ERR! notarget No matching version found for chalk@^113.0.0.\nnpm ERR! notarget In most cases you or one of your dependencies are requesting\nnpm ERR! notarget a package version that doesn't exist.", + doc: "Failed to find package \"chalk@^113.0.0\" that satisfies the requirement from package.json file. In most cases you or one of your dependencies are requesting a package version that doesn't exist. Please check that package versions are correct in your package.json file.", + }, + { + name: "Invalid package name or private package", + error: "npm ERR! code E404\nnpm ERR! 404 Not Found - GET https://registry.npmjs.org/chalke - Not found\nnpm ERR! 404 \nnpm ERR! 404 'chalke@^3.0.0' is not in this registry.\nnpm ERR! 404 You should bug the author to publish it (or use the name yourself!)\nnpm ERR! 404 \nnpm ERR! 404 Note that you can also install from a\nnpm ERR! 404 tarball, folder, http url, or git url.", + doc: "Failed to find package \"chalke@^3.0.0\" that satisfies the requirement from dependencies. Please check that dependencies are correct in your package.json file. \nIf this is a private dependency, please make sure that the debricked CLI has access to install it or pre-install it before running the debricked CLI.", + }, + { + name: "No internet connection", + error: "npm ERR! code EAI_AGAIN\nnpm ERR! syscall getaddrinfo\nnpm ERR! errno EAI_AGAIN\nnpm ERR! request to https://registry.npmjs.org/chalke failed, reason: getaddrinfo EAI_AGAIN registry.npmjs.org", + doc: "Package registry \"registry.npmjs.org\" is not available at the moment. There might be a trouble with your network connection.", + }, + { + name: "Permission denied", + error: "npm ERR! Error: EACCES, open '/home/me/.npm/semver/3.0.1/package/package.json'\nnpm ERR! { [Error: EACCES, open '/home/me/.npm/semver/3.0.1/package/package.json']\nnpm ERR! errno: 3,\nnpm ERR! code: 'EACCES',\nnpm ERR! path: '/home/me/.npm/semver/3.0.1/package/package.json',\nnpm ERR! parent: 'gulp' }\nnpm ERR! \nnpm ERR! Please try running this command again as root/Administrator.\n", + doc: "Couldn't get access to \"/home/me/.npm/semver/3.0.1/package/package.json\". Please check permissions or try running this command again as root/Administrator.", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cmdErr := errors.New(c.error) + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MakeInstallErr = cmdErr + cmd, _ := cmdFactoryMock.MakeInstallCmd("echo", "package.json") + + expectedError := util.NewPMJobError(c.error) + expectedError.SetDocumentation(c.doc) + expectedError.SetStatus("installing dependencies") + expectedError.SetCommand(cmd.String()) + + j := NewJob("file", true, cmdFactoryMock) + + go jobTestdata.WaitStatus(j) + j.Run() + + allErrors := j.Errors().GetAll() + + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, allErrors, expectedError) + }) + } +} + +func TestRunInstallCmdOutputErr(t *testing.T) { + cmdMock := testdata.NewEchoCmdFactory() + cmdMock.InstallCmdName = badName + j := NewJob("file", true, cmdMock) + + go jobTestdata.WaitStatus(j) + j.Run() + + jobTestdata.AssertPathErr(t, j.Errors()) +} diff --git a/internal/resolution/pm/npm/pm.go b/internal/resolution/pm/npm/pm.go new file mode 100644 index 00000000..828aad13 --- /dev/null +++ b/internal/resolution/pm/npm/pm.go @@ -0,0 +1,23 @@ +package npm + +const Name = "npm" + +type Pm struct { + name string +} + +func NewPm() Pm { + return Pm{ + name: Name, + } +} + +func (pm Pm) Name() string { + return pm.name +} + +func (Pm) Manifests() []string { + return []string{ + `package\.json$`, + } +} diff --git a/internal/resolution/pm/npm/pm_test.go b/internal/resolution/pm/npm/pm_test.go new file mode 100644 index 00000000..4ed12f07 --- /dev/null +++ b/internal/resolution/pm/npm/pm_test.go @@ -0,0 +1,40 @@ +package npm + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewPm(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.name) +} + +func TestName(t *testing.T) { + pm := NewPm() + assert.Equal(t, Name, pm.Name()) +} + +func TestManifests(t *testing.T) { + pm := Pm{} + manifests := pm.Manifests() + assert.Len(t, manifests, 1) + manifest := manifests[0] + assert.Equal(t, `package\.json$`, manifest) + _, err := regexp.Compile(manifest) + assert.NoError(t, err) + + cases := map[string]bool{ + "package.json": true, + "package-lock.json": false, + "npm.lock": false, + } + for file, isMatch := range cases { + t.Run(file, func(t *testing.T) { + matched, _ := regexp.MatchString(manifest, file) + assert.Equal(t, isMatch, matched) + }) + } +} diff --git a/internal/resolution/pm/npm/strategy.go b/internal/resolution/pm/npm/strategy.go new file mode 100644 index 00000000..31c757ef --- /dev/null +++ b/internal/resolution/pm/npm/strategy.go @@ -0,0 +1,29 @@ +package npm + +import ( + "github.com/debricked/cli/internal/resolution/job" +) + +type Strategy struct { + files []string +} + +func (s Strategy) Invoke() ([]job.IJob, error) { + var jobs []job.IJob + for _, file := range s.files { + jobs = append(jobs, NewJob( + file, + true, + CmdFactory{ + execPath: ExecPath{}, + }, + ), + ) + } + + return jobs, nil +} + +func NewStrategy(files []string) Strategy { + return Strategy{files} +} diff --git a/internal/resolution/pm/npm/strategy_test.go b/internal/resolution/pm/npm/strategy_test.go new file mode 100644 index 00000000..29fd6a43 --- /dev/null +++ b/internal/resolution/pm/npm/strategy_test.go @@ -0,0 +1,43 @@ +package npm + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewStrategy(t *testing.T) { + s := NewStrategy(nil) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{}) + assert.NotNil(t, s) + assert.Len(t, s.files, 0) + + s = NewStrategy([]string{"file"}) + assert.NotNil(t, s) + assert.Len(t, s.files, 1) + + s = NewStrategy([]string{"file-1", "file-2"}) + assert.NotNil(t, s) + assert.Len(t, s.files, 2) +} + +func TestInvokeNoFiles(t *testing.T) { + s := NewStrategy([]string{}) + jobs, _ := s.Invoke() + assert.Empty(t, jobs) +} + +func TestInvokeOneFile(t *testing.T) { + s := NewStrategy([]string{"file"}) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 1) +} + +func TestInvokeManyFiles(t *testing.T) { + s := NewStrategy([]string{"file-1", "file-2"}) + jobs, _ := s.Invoke() + assert.Len(t, jobs, 2) +} diff --git a/internal/resolution/pm/npm/testdata/cmd_factory_mock.go b/internal/resolution/pm/npm/testdata/cmd_factory_mock.go new file mode 100644 index 00000000..093ba6b2 --- /dev/null +++ b/internal/resolution/pm/npm/testdata/cmd_factory_mock.go @@ -0,0 +1,20 @@ +package testdata + +import ( + "os/exec" +) + +type CmdFactoryMock struct { + InstallCmdName string + MakeInstallErr error +} + +func NewEchoCmdFactory() CmdFactoryMock { + return CmdFactoryMock{ + InstallCmdName: "echo", + } +} + +func (f CmdFactoryMock) MakeInstallCmd(command string, file string) (*exec.Cmd, error) { + return exec.Command(f.InstallCmdName), f.MakeInstallErr +} diff --git a/internal/resolution/pm/pm.go b/internal/resolution/pm/pm.go index d9503c85..0a87125c 100644 --- a/internal/resolution/pm/pm.go +++ b/internal/resolution/pm/pm.go @@ -5,6 +5,7 @@ import ( "github.com/debricked/cli/internal/resolution/pm/gomod" "github.com/debricked/cli/internal/resolution/pm/gradle" "github.com/debricked/cli/internal/resolution/pm/maven" + "github.com/debricked/cli/internal/resolution/pm/npm" "github.com/debricked/cli/internal/resolution/pm/nuget" "github.com/debricked/cli/internal/resolution/pm/pip" "github.com/debricked/cli/internal/resolution/pm/yarn" @@ -22,6 +23,7 @@ func Pms() []IPm { gomod.NewPm(), pip.NewPm(), yarn.NewPm(), + npm.NewPm(), nuget.NewPm(), composer.NewPm(), } diff --git a/internal/resolution/pm/yarn/job.go b/internal/resolution/pm/yarn/job.go index cd239872..b4c1e0d0 100644 --- a/internal/resolution/pm/yarn/job.go +++ b/internal/resolution/pm/yarn/job.go @@ -10,9 +10,6 @@ import ( const ( yarn = "yarn" - invalidJsonErrRegex = "error SyntaxError.*package.json: (.*)" - invalidSchemaErrRegex = "error package.json: (.*)" - invalidArgumentErrRegex = "error TypeError \\[\\w+\\]: (.*)" versionNotFoundErrRegex = "error (Couldn\\'t find any versions for .*)" dependencyNotFoundErrRegex = `error.*? "?(https?://[^"\s:]+)?: Not found` registryUnavailableErrRegex = "error Error: getaddrinfo ENOTFOUND ([\\w\\.]+)" @@ -75,9 +72,6 @@ func (j *Job) createError(error string, cmd string, status string) job.IError { func (j *Job) handleError(cmdError job.IError) { expressions := []string{ - invalidJsonErrRegex, - invalidSchemaErrRegex, - invalidArgumentErrRegex, versionNotFoundErrRegex, dependencyNotFoundErrRegex, registryUnavailableErrRegex, @@ -103,12 +97,6 @@ func (j *Job) addDocumentation(expr string, matches [][]string, cmdError job.IEr documentation := cmdError.Documentation() switch expr { - case invalidJsonErrRegex: - documentation = getInvalidJsonErrorDocumentation(matches) - case invalidSchemaErrRegex: - documentation = getInvalidSchemaErrorDocumentation(matches) - case invalidArgumentErrRegex: - documentation = getInvalidArgumentErrorDocumentation(matches) case versionNotFoundErrRegex: documentation = getVersionNotFoundErrorDocumentation(matches) case dependencyNotFoundErrRegex: @@ -124,46 +112,6 @@ func (j *Job) addDocumentation(expr string, matches [][]string, cmdError job.IEr return cmdError } -func getInvalidJsonErrorDocumentation(matches [][]string) string { - message := "" - if len(matches) > 0 && len(matches[0]) > 1 { - message = matches[0][1] - } - - return strings.Join( - []string{ - "Your package.json file contains invalid JSON:", - message + ".", - }, " ") -} - -func getInvalidSchemaErrorDocumentation(matches [][]string) string { - message := "" - if len(matches) > 0 && len(matches[0]) > 1 { - message = matches[0][1] - } - - return strings.Join( - []string{ - "Your package.json file is not valid:", - message + ".", - "Please make sure it follows the schema.", - }, " ") -} - -func getInvalidArgumentErrorDocumentation(matches [][]string) string { - message := "" - if len(matches) > 0 && len(matches[0]) > 1 { - message = matches[0][1] - } - - return strings.Join( - []string{ - message + ".", - "Please make sure that your package.json file doesn't contain errors.", - }, " ") -} - func getDependencyNotFoundErrorDocumentation(matches [][]string) string { dependency := "" if len(matches) > 0 && len(matches[0]) > 1 { diff --git a/internal/resolution/pm/yarn/job_test.go b/internal/resolution/pm/yarn/job_test.go index 783a6f45..a704aea4 100644 --- a/internal/resolution/pm/yarn/job_test.go +++ b/internal/resolution/pm/yarn/job_test.go @@ -32,68 +32,64 @@ func TestInstall(t *testing.T) { func TestRunInstallCmdErr(t *testing.T) { cases := []struct { - cmd string + name string error string doc string }{ { + name: "General error", error: "cmd-error", doc: util.UnknownError, }, { - error: "error SyntaxError: /home/asus/Projects/playground/rpn_js/package.json: Unexpected string in JSON at position 186\n at JSON.parse ()", - doc: "Your package.json file contains invalid JSON: Unexpected string in JSON at position 186.", - }, - { - error: "error package.json: \"name\" is not a string", - doc: "Your package.json file is not valid: \"name\" is not a string. Please make sure it follows the schema.", - }, - { - error: "error TypeError [ERR_INVALID_ARG_TYPE]: The \"path\" argument must be of type string. Received an instance of Array\n at validateString (internal/validators.js:120:11)\n", - doc: "The \"path\" argument must be of type string. Received an instance of Array. Please make sure that your package.json file doesn't contain errors.", - }, - { + name: "Invalid package name", error: "error Error: https://registry.yarnpkg.com/chalke: Not found\n at Request.params.callback [as _callback] (/usr/local/lib/node_modules/yarn/lib/cli.js:66148:18)", doc: "Failed to find package \"https://registry.yarnpkg.com/chalke\" that satisfies the requirement from yarn dependencies. Please check that dependencies are correct in your package.json file. \nIf this is a private dependency, please make sure that the debricked CLI has access to install it or pre-install it before running the debricked CLI.", }, { + name: "Invalid package name (legacy)", error: `error An unexpected error occurred: "https://registry.yarnpkg.com/chalke: Not found".`, doc: "Failed to find package \"https://registry.yarnpkg.com/chalke\" that satisfies the requirement from yarn dependencies. Please check that dependencies are correct in your package.json file. \nIf this is a private dependency, please make sure that the debricked CLI has access to install it or pre-install it before running the debricked CLI.", }, { + name: "Invalid package version", error: "error Couldn't find any versions for \"chalk\" that matches \"^300.0.0\"\ninfo Visit https://yarnpkg.com/en/docs/cli/install for documentation about this command.", doc: "Couldn't find any versions for \"chalk\" that matches \"^300.0.0\". Please check that dependencies are correct in your package.json file.", }, { + name: "No internet connection", error: "error Error: getaddrinfo ENOTFOUND nexus.dev\n at GetAddrInfoReqWrap.onlookup [as oncomplete] (dns.js:66:26)\n", doc: "Package registry \"nexus.dev\" is not available at the moment. There might be a trouble with your network connection.", }, { + name: "Private package", error: "Error: https://registry.npmjs.org/@private/my-private-package/-/my-private-package-0.0.5.tgz: Request failed \"404 Not Found\"", doc: "Failed to find a package that satisfies requirements for yarn dependencies: https://registry.npmjs.org/@private/my-private-package/-/my-private-package-0.0.5.tgz. This could mean that the package or version does not exist or is private.\n If this is a private dependency, please make sure that the debricked CLI has access to install it or pre-install it before running the debricked CLI.", }, } for _, c := range cases { - cmdErr := errors.New(c.error) - cmdFactoryMock := testdata.NewEchoCmdFactory() - cmdFactoryMock.MakeInstallErr = cmdErr - cmd, _ := cmdFactoryMock.MakeInstallCmd("echo", "package.json") + t.Run(c.name, func(t *testing.T) { + cmdErr := errors.New(c.error) + cmdFactoryMock := testdata.NewEchoCmdFactory() + cmdFactoryMock.MakeInstallErr = cmdErr + cmd, _ := cmdFactoryMock.MakeInstallCmd("echo", "package.json") - expectedError := util.NewPMJobError(c.error) - expectedError.SetDocumentation(c.doc) - expectedError.SetStatus("installing dependencies") - expectedError.SetCommand(cmd.String()) + expectedError := util.NewPMJobError(c.error) + expectedError.SetDocumentation(c.doc) + expectedError.SetStatus("installing dependencies") + expectedError.SetCommand(cmd.String()) - j := NewJob("file", true, cmdFactoryMock) + j := NewJob("file", true, cmdFactoryMock) - go jobTestdata.WaitStatus(j) - j.Run() + go jobTestdata.WaitStatus(j) + j.Run() - errors := j.Errors().GetAll() + allErrors := j.Errors().GetAll() - assert.Len(t, j.Errors().GetAll(), 1) - assert.Contains(t, errors, expectedError) + assert.Len(t, j.Errors().GetAll(), 1) + assert.Contains(t, allErrors, expectedError) + }) } } diff --git a/internal/resolution/resolver.go b/internal/resolution/resolver.go index 1a6af47f..061bf7b4 100644 --- a/internal/resolution/resolver.go +++ b/internal/resolution/resolver.go @@ -13,6 +13,7 @@ import ( type IResolver interface { Resolve(paths []string, exclusions []string, verbose bool) (IResolution, error) + SetNpmPreferred(npmPreferred bool) } type Resolver struct { @@ -20,6 +21,7 @@ type Resolver struct { batchFactory resolutionFile.IBatchFactory strategyFactory strategy.IFactory scheduler IScheduler + npmPreferred bool } func NewResolver( @@ -33,9 +35,14 @@ func NewResolver( batchFactory, strategyFactory, scheduler, + false, } } +func (r Resolver) SetNpmPreferred(npmPreferred bool) { + r.batchFactory.SetNpmPreferred(npmPreferred) +} + func (r Resolver) Resolve(paths []string, exclusions []string, verbose bool) (IResolution, error) { files, err := r.refinePaths(paths, exclusions) if err != nil { diff --git a/internal/resolution/strategy/strategy_factory.go b/internal/resolution/strategy/strategy_factory.go index 5274e7b3..5fb1aa0c 100644 --- a/internal/resolution/strategy/strategy_factory.go +++ b/internal/resolution/strategy/strategy_factory.go @@ -8,6 +8,7 @@ import ( "github.com/debricked/cli/internal/resolution/pm/gomod" "github.com/debricked/cli/internal/resolution/pm/gradle" "github.com/debricked/cli/internal/resolution/pm/maven" + "github.com/debricked/cli/internal/resolution/pm/npm" "github.com/debricked/cli/internal/resolution/pm/nuget" "github.com/debricked/cli/internal/resolution/pm/pip" "github.com/debricked/cli/internal/resolution/pm/yarn" @@ -36,6 +37,8 @@ func (sf Factory) Make(pmFileBatch file.IBatch, paths []string) (IStrategy, erro return pip.NewStrategy(pmFileBatch.Files()), nil case yarn.Name: return yarn.NewStrategy(pmFileBatch.Files()), nil + case npm.Name: + return npm.NewStrategy(pmFileBatch.Files()), nil case nuget.Name: return nuget.NewStrategy(pmFileBatch.Files()), nil case composer.Name: diff --git a/internal/resolution/testdata/resolver_mock.go b/internal/resolution/testdata/resolver_mock.go index 6449d35d..392030b0 100644 --- a/internal/resolution/testdata/resolver_mock.go +++ b/internal/resolution/testdata/resolver_mock.go @@ -12,6 +12,9 @@ type ResolverMock struct { files []string } +func (r *ResolverMock) SetNpmPreferred(_ bool) { +} + func (r *ResolverMock) Resolve(_ []string, _ []string, _ bool) (resolution.IResolution, error) { for _, f := range r.files { createdFile, err := os.Create(f) diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index 0be96872..e349d569 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -54,6 +54,7 @@ type DebrickedOptions struct { CommitAuthor string RepositoryUrl string IntegrationName string + NpmPreferred bool PassOnTimeOut bool CallGraphUploadTimeout int CallGraphGenerateTimeout int @@ -133,6 +134,7 @@ func (dScanner *DebrickedScanner) Scan(o IOptions) error { func (dScanner *DebrickedScanner) scanResolve(options DebrickedOptions) error { if options.Resolve { + dScanner.resolver.SetNpmPreferred(options.NpmPreferred) _, resErr := dScanner.resolver.Resolve([]string{options.Path}, options.Exclusions, options.Verbose) if resErr != nil { return resErr diff --git a/test/resolve/resolver_test.go b/test/resolve/resolver_test.go index 9baffda1..52616265 100644 --- a/test/resolve/resolver_test.go +++ b/test/resolve/resolver_test.go @@ -6,7 +6,9 @@ import ( "testing" "github.com/debricked/cli/internal/cmd/resolve" + "github.com/debricked/cli/internal/resolution/pm/npm" "github.com/debricked/cli/internal/wire" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) @@ -24,9 +26,15 @@ func TestResolves(t *testing.T) { packageManager: "composer", }, { - name: "basic package.json", + name: "basic package.json (Yarn)", manifestFile: "testdata/npm/package.json", lockFileName: "yarn.lock", + packageManager: "yarn", + }, + { + name: "basic package.json (NPM)", + manifestFile: "testdata/npm/package.json", + lockFileName: "package-lock.json", packageManager: "npm", }, { @@ -70,6 +78,10 @@ func TestResolves(t *testing.T) { for _, cT := range cases { c := cT t.Run(c.name, func(t *testing.T) { + if c.packageManager == npm.Name { + viper.Set(resolve.NpmPreferredFlag, true) + } + resolveCmd := resolve.NewResolveCmd(wire.GetCliContainer().Resolver()) lockFileDir := filepath.Dir(c.manifestFile) lockFile := filepath.Join(lockFileDir, c.lockFileName)