Skip to content

Commit

Permalink
fix: fix parallelization issues on win and in language-server
Browse files Browse the repository at this point in the history
chore: rollback test

chore: add test and update gaf
  • Loading branch information
PeterSchafer committed Aug 22, 2024
1 parent ee39065 commit 313d94f
Show file tree
Hide file tree
Showing 13 changed files with 294 additions and 199 deletions.
5 changes: 4 additions & 1 deletion cliv2/cmd/cliv2/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -566,7 +566,10 @@ func MainWithErrorCode() int {
}

// cleanup resources in use
basic_workflows.Cleanup()
_, err = globalEngine.Invoke(basic_workflows.WORKFLOWID_GLOBAL_CLEANUP)
if err != nil {
globalLogger.Printf("Failed to cleanup %v", err)
}

return exitCode
}
Expand Down
2 changes: 1 addition & 1 deletion cliv2/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ require (
github.com/snyk/cli-extension-sbom v0.0.0-20240812130014-3f4e892f15ec
github.com/snyk/container-cli v0.0.0-20240322120441-6d9b9482f9b1
github.com/snyk/error-catalog-golang-public v0.0.0-20240809094525-c48d19c27edb
github.com/snyk/go-application-framework v0.0.0-20240816082313-d6797668c3be
github.com/snyk/go-application-framework v0.0.0-20240822160346-edf22a8795be
github.com/snyk/go-httpauth v0.0.0-20240307114523-1f5ea3f55c65
github.com/snyk/snyk-iac-capture v0.6.5
github.com/snyk/snyk-ls v0.0.0-20240814110458-759bec2da65d
Expand Down
4 changes: 2 additions & 2 deletions cliv2/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -744,8 +744,8 @@ github.com/snyk/container-cli v0.0.0-20240322120441-6d9b9482f9b1 h1:9RKY9NdX5DrJ
github.com/snyk/container-cli v0.0.0-20240322120441-6d9b9482f9b1/go.mod h1:38w+dcAQp9eG3P5t2eNS9eG0reut10AeJjLv5lJ5lpM=
github.com/snyk/error-catalog-golang-public v0.0.0-20240809094525-c48d19c27edb h1:w9tJhpTFxWqAhLeraGsMExDjGK9x5Dwj1NRFwb+t+QE=
github.com/snyk/error-catalog-golang-public v0.0.0-20240809094525-c48d19c27edb/go.mod h1:Ytttq7Pw4vOCu9NtRQaOeDU2dhBYUyNBe6kX4+nIIQ4=
github.com/snyk/go-application-framework v0.0.0-20240816082313-d6797668c3be h1:iEcoKC6UeEyLwvu5o2mwCGX1ExQrN+lIbb/r5UGFYdo=
github.com/snyk/go-application-framework v0.0.0-20240816082313-d6797668c3be/go.mod h1:zgYTVG71nX7zTb3ELeRlnwE/uKQxeOyQmAHtg6bC4uU=
github.com/snyk/go-application-framework v0.0.0-20240822160346-edf22a8795be h1:5m55hR2Vmvx4dqeWNOjRwppAOBvmTM/BodJCHAbFoEY=
github.com/snyk/go-application-framework v0.0.0-20240822160346-edf22a8795be/go.mod h1:zgYTVG71nX7zTb3ELeRlnwE/uKQxeOyQmAHtg6bC4uU=
github.com/snyk/go-httpauth v0.0.0-20240307114523-1f5ea3f55c65 h1:CEQuYv0Go6MEyRCD3YjLYM2u3Oxkx8GpCpFBd4rUTUk=
github.com/snyk/go-httpauth v0.0.0-20240307114523-1f5ea3f55c65/go.mod h1:88KbbvGYlmLgee4OcQ19yr0bNpXpOr2kciOthaSzCAg=
github.com/snyk/policy-engine v0.30.11 h1:wUy5LMar2vccMbNM62MSBRdjAQAhAbIm7aNXXO+g2tk=
Expand Down
60 changes: 0 additions & 60 deletions cliv2/internal/cliv2/cliv2.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,9 @@ import (
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"syscall"
"time"

"github.com/gofrs/flock"
Expand Down Expand Up @@ -141,11 +138,6 @@ func (c *CLI) Init() (err error) {

func (c *CLI) ClearCache() error {
err := c.clearVersionFolders()
if err != nil {
return err
}

err = c.clearTemporaryProcessFolders()
return err
}

Expand Down Expand Up @@ -178,58 +170,6 @@ func (c *CLI) clearVersionFolders() error {
return nil
}

func (c *CLI) clearTemporaryProcessFolders() error {
// clean up the tmp dir of the current version
maxConsecutiveDeletes := 5
deleteCount := 0
tempDir := filepath.Dir(c.GetTempDir())
fileInfo, err := os.ReadDir(tempDir)
if err != nil {
return err
}

// cleanup tmp files related to a non-existing process
processTempPattern := regexp.MustCompile("pid([0-9]*)")
for _, file := range fileInfo {
currentPath := path.Join(tempDir, file.Name())
matches := processTempPattern.FindStringSubmatch(file.Name())
if len(matches) == 2 {
processFound := true
pid, localError := strconv.Atoi(matches[1])
if localError != nil {
continue
}

p, localError := os.FindProcess(pid)
if localError != nil {
processFound = false
}

if p != nil {
localError = p.Signal(syscall.Signal(0))
if localError != nil {
processFound = false
}
}

if !processFound {
deleteCount++
err = os.RemoveAll(currentPath)
if err != nil {
c.DebugLogger.Println("Error deleting temporary files: ", currentPath)
}
}
}

// Stop the loop after 5 deletions to not create too much overhead
if deleteCount == maxConsecutiveDeletes {
break
}
}

return nil
}

func (c *CLI) AppendEnvironmentVariables(env []string) {
c.env = append(c.env, env...)
}
Expand Down
7 changes: 0 additions & 7 deletions cliv2/internal/cliv2/cliv2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"sort"
"testing"
Expand Down Expand Up @@ -411,16 +410,11 @@ func Test_clearCache(t *testing.T) {
lockfile := path.Join(cli.CacheDirectory, "v1.914.0.lock")
randomFile := path.Join(versionNoV, "filename")
currentVersion := cli.GetBinaryLocation()
tempDir := filepath.Dir(cli.GetTempDir())
oldProcessTempDir := path.Join(tempDir, "pid123")
oldProcessTempDirFile := path.Join(oldProcessTempDir, "bla.txt")

assert.NoError(t, os.Mkdir(versionWithV, 0755))
assert.NoError(t, os.Mkdir(versionNoV, 0755))
assert.NoError(t, os.Mkdir(oldProcessTempDir, 0755))
assert.NoError(t, os.WriteFile(randomFile, []byte("Writing some strings"), 0666))
assert.NoError(t, os.WriteFile(lockfile, []byte("Writing some strings"), 0666))
assert.NoError(t, os.WriteFile(oldProcessTempDirFile, []byte("Writing some strings"), 0666))

// clear cache
err := cli.ClearCache()
Expand All @@ -430,7 +424,6 @@ func Test_clearCache(t *testing.T) {
assert.NoDirExists(t, versionWithV)
assert.NoDirExists(t, versionNoV)
assert.NoFileExists(t, randomFile)
assert.NoFileExists(t, oldProcessTempDirFile)
// check if directories that need to exist still exist
assert.FileExists(t, currentVersion)
assert.FileExists(t, lockfile)
Expand Down
64 changes: 35 additions & 29 deletions cliv2/internal/proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,17 @@ import (
"github.com/rs/zerolog"
"github.com/snyk/go-application-framework/pkg/configuration"
"github.com/snyk/go-application-framework/pkg/networking"
"github.com/snyk/go-application-framework/pkg/networking/certs"
pkg_utils "github.com/snyk/go-application-framework/pkg/utils"

"github.com/snyk/go-application-framework/pkg/networking/certs"
"github.com/snyk/go-application-framework/pkg/networking/middleware"
"github.com/snyk/go-httpauth/pkg/httpauth"

"github.com/snyk/cli/cliv2/internal/constants"
"github.com/snyk/cli/cliv2/internal/utils"

"github.com/elazarl/goproxy"
"github.com/elazarl/goproxy/ext/auth"

"github.com/snyk/cli/cliv2/internal/constants"
"github.com/snyk/cli/cliv2/internal/utils"
)

type WrapperProxy struct {
Expand Down Expand Up @@ -55,17 +55,17 @@ const (
PROXY_USERNAME = "snykcli"
)

func NewWrapperProxy(config configuration.Configuration, cliVersion string, debugLogger *zerolog.Logger) (*WrapperProxy, error) {
var p WrapperProxy
p.cliVersion = cliVersion
p.addHeaderFunc = func(request *http.Request) error { return nil }
type CaData struct {
CertPool *x509.CertPool
CertFile string
}

func InitCA(config configuration.Configuration, cliVersion string, logger *zerolog.Logger) (*CaData, error) {
cacheDirectory := config.GetString(configuration.CACHE_PATH)
insecureSkipVerify := config.GetBool(configuration.INSECURE_HTTPS)

certName := "snyk-embedded-proxy"
p.DebugLogger = debugLogger
certPEMBlock, keyPEMBlock, err := certs.MakeSelfSignedCert(certName, []string{}, log.New(&pkg_utils.ToZeroLogDebug{Logger: p.DebugLogger}, "", 0))
logWriter := pkg_utils.ToZeroLogDebug{Logger: logger}
certPEMBlock, keyPEMBlock, err := certs.MakeSelfSignedCert(certName, []string{}, log.New(&logWriter, "", 0))
if err != nil {
return nil, err
}
Expand All @@ -77,12 +77,12 @@ func NewWrapperProxy(config configuration.Configuration, cliVersion string, debu
}
certFile, err := os.CreateTemp(tmpDirectory, "snyk-cli-cert-*.crt")
if err != nil {
fmt.Println("failed to create temp cert file")
logger.Println("failed to create temp cert file")
return nil, err
}
defer certFile.Close()

p.CertificateLocation = certFile.Name() // gives full path, not just the name
certificateLocation := certFile.Name() // gives full path, not just the name

rootCAs, err := x509.SystemCertPool()
if err != nil {
Expand All @@ -105,27 +105,42 @@ func NewWrapperProxy(config configuration.Configuration, cliVersion string, debu
}
}

debugLogger.Debug().Msgf("Using additional CAs from file: %v", extraCaCertFile)
logger.Debug().Msgf("Using additional CAs from file: %v", extraCaCertFile)
}
}

debugLogger.Debug().Msgf("Temporary CertificateLocation: %v", p.CertificateLocation)
logger.Debug().Msgf("Temporary CertificateLocation: %v", certificateLocation)
certPEMString := string(certPEMBlock)
err = utils.WriteToFile(p.CertificateLocation, certPEMString)
err = utils.WriteToFile(certificateLocation, certPEMString)
if err != nil {
fmt.Println("failed to write cert to file")
logger.Print("failed to write cert to file")
return nil, err
}

err = setCAFromBytes(certPEMBlock, keyPEMBlock)
err = setGlobalProxyCA(certPEMBlock, keyPEMBlock)
if err != nil {
return nil, err
}

return &CaData{
CertPool: rootCAs,
CertFile: certificateLocation,
}, nil
}

func NewWrapperProxy(config configuration.Configuration, cliVersion string, debugLogger *zerolog.Logger, ca CaData) (*WrapperProxy, error) {
var p WrapperProxy
p.cliVersion = cliVersion
p.addHeaderFunc = func(request *http.Request) error { return nil }
p.DebugLogger = debugLogger
p.CertificateLocation = ca.CertFile

insecureSkipVerify := config.GetBool(configuration.INSECURE_HTTPS)

p.transport = &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: insecureSkipVerify, // goproxy defaults to true
RootCAs: rootCAs,
RootCAs: ca.CertPool,
},
}

Expand Down Expand Up @@ -242,18 +257,9 @@ func (p *WrapperProxy) Stop() {

func (p *WrapperProxy) Close() {
p.Stop()

p.DebugLogger.Print("deleting temp cert file:", p.CertificateLocation)
err := os.Remove(p.CertificateLocation)
if err != nil {
p.DebugLogger.Print("failed to delete cert file")
p.DebugLogger.Print(err)
} else {
p.DebugLogger.Print("deleted temp cert file:", p.CertificateLocation)
}
}

func setCAFromBytes(certPEMBlock []byte, keyPEMBlock []byte) error {
func setGlobalProxyCA(certPEMBlock []byte, keyPEMBlock []byte) error {
goproxyCa, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
if err != nil {
return err
Expand Down
Loading

0 comments on commit 313d94f

Please sign in to comment.