Skip to content

Commit

Permalink
fix: fix parallelization issues on win and in language-server
Browse files Browse the repository at this point in the history
  • Loading branch information
PeterSchafer committed Aug 20, 2024
1 parent bfd3036 commit 9fa49db
Show file tree
Hide file tree
Showing 10 changed files with 233 additions and 204 deletions.
5 changes: 4 additions & 1 deletion cliv2/cmd/cliv2/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -566,7 +566,10 @@ func MainWithErrorCode() int {
}

// cleanup resources in use
basic_workflows.Cleanup()
_, err = globalEngine.Invoke(basic_workflows.WORKFLOWID_GLOBAL_CLEANUP)
if err != nil {
globalLogger.Printf("Failed to cleanup %v", err)
}

return exitCode
}
Expand Down
60 changes: 0 additions & 60 deletions cliv2/internal/cliv2/cliv2.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,9 @@ import (
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"syscall"
"time"

"github.com/gofrs/flock"
Expand Down Expand Up @@ -141,11 +138,6 @@ func (c *CLI) Init() (err error) {

func (c *CLI) ClearCache() error {
err := c.clearVersionFolders()
if err != nil {
return err
}

err = c.clearTemporaryProcessFolders()
return err
}

Expand Down Expand Up @@ -178,58 +170,6 @@ func (c *CLI) clearVersionFolders() error {
return nil
}

func (c *CLI) clearTemporaryProcessFolders() error {
// clean up the tmp dir of the current version
maxConsecutiveDeletes := 5
deleteCount := 0
tempDir := filepath.Dir(c.GetTempDir())
fileInfo, err := os.ReadDir(tempDir)
if err != nil {
return err
}

// cleanup tmp files related to a non-existing process
processTempPattern := regexp.MustCompile("pid([0-9]*)")
for _, file := range fileInfo {
currentPath := path.Join(tempDir, file.Name())
matches := processTempPattern.FindStringSubmatch(file.Name())
if len(matches) == 2 {
processFound := true
pid, localError := strconv.Atoi(matches[1])
if localError != nil {
continue
}

p, localError := os.FindProcess(pid)
if localError != nil {
processFound = false
}

if p != nil {
localError = p.Signal(syscall.Signal(0))
if localError != nil {
processFound = false
}
}

if !processFound {
deleteCount++
err = os.RemoveAll(currentPath)
if err != nil {
c.DebugLogger.Println("Error deleting temporary files: ", currentPath)
}
}
}

// Stop the loop after 5 deletions to not create too much overhead
if deleteCount == maxConsecutiveDeletes {
break
}
}

return nil
}

func (c *CLI) AppendEnvironmentVariables(env []string) {
c.env = append(c.env, env...)
}
Expand Down
7 changes: 0 additions & 7 deletions cliv2/internal/cliv2/cliv2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"sort"
"testing"
Expand Down Expand Up @@ -411,16 +410,11 @@ func Test_clearCache(t *testing.T) {
lockfile := path.Join(cli.CacheDirectory, "v1.914.0.lock")
randomFile := path.Join(versionNoV, "filename")
currentVersion := cli.GetBinaryLocation()
tempDir := filepath.Dir(cli.GetTempDir())
oldProcessTempDir := path.Join(tempDir, "pid123")
oldProcessTempDirFile := path.Join(oldProcessTempDir, "bla.txt")

assert.NoError(t, os.Mkdir(versionWithV, 0755))
assert.NoError(t, os.Mkdir(versionNoV, 0755))
assert.NoError(t, os.Mkdir(oldProcessTempDir, 0755))
assert.NoError(t, os.WriteFile(randomFile, []byte("Writing some strings"), 0666))
assert.NoError(t, os.WriteFile(lockfile, []byte("Writing some strings"), 0666))
assert.NoError(t, os.WriteFile(oldProcessTempDirFile, []byte("Writing some strings"), 0666))

// clear cache
err := cli.ClearCache()
Expand All @@ -430,7 +424,6 @@ func Test_clearCache(t *testing.T) {
assert.NoDirExists(t, versionWithV)
assert.NoDirExists(t, versionNoV)
assert.NoFileExists(t, randomFile)
assert.NoFileExists(t, oldProcessTempDirFile)
// check if directories that need to exist still exist
assert.FileExists(t, currentVersion)
assert.FileExists(t, lockfile)
Expand Down
64 changes: 35 additions & 29 deletions cliv2/internal/proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,17 @@ import (
"github.com/rs/zerolog"
"github.com/snyk/go-application-framework/pkg/configuration"
"github.com/snyk/go-application-framework/pkg/networking"
"github.com/snyk/go-application-framework/pkg/networking/certs"
pkg_utils "github.com/snyk/go-application-framework/pkg/utils"

"github.com/snyk/go-application-framework/pkg/networking/certs"
"github.com/snyk/go-application-framework/pkg/networking/middleware"
"github.com/snyk/go-httpauth/pkg/httpauth"

"github.com/snyk/cli/cliv2/internal/constants"
"github.com/snyk/cli/cliv2/internal/utils"

"github.com/elazarl/goproxy"
"github.com/elazarl/goproxy/ext/auth"

"github.com/snyk/cli/cliv2/internal/constants"
"github.com/snyk/cli/cliv2/internal/utils"
)

type WrapperProxy struct {
Expand Down Expand Up @@ -55,17 +55,17 @@ const (
PROXY_USERNAME = "snykcli"
)

func NewWrapperProxy(config configuration.Configuration, cliVersion string, debugLogger *zerolog.Logger) (*WrapperProxy, error) {
var p WrapperProxy
p.cliVersion = cliVersion
p.addHeaderFunc = func(request *http.Request) error { return nil }
type CaData struct {
CertPool *x509.CertPool
CertFile string
}

func InitCA(config configuration.Configuration, cliVersion string, logger *zerolog.Logger) (*CaData, error) {
cacheDirectory := config.GetString(configuration.CACHE_PATH)
insecureSkipVerify := config.GetBool(configuration.INSECURE_HTTPS)

certName := "snyk-embedded-proxy"
p.DebugLogger = debugLogger
certPEMBlock, keyPEMBlock, err := certs.MakeSelfSignedCert(certName, []string{}, log.New(&pkg_utils.ToZeroLogDebug{Logger: p.DebugLogger}, "", 0))
logWriter := pkg_utils.ToZeroLogDebug{Logger: logger}
certPEMBlock, keyPEMBlock, err := certs.MakeSelfSignedCert(certName, []string{}, log.New(&logWriter, "", 0))
if err != nil {
return nil, err
}
Expand All @@ -77,12 +77,12 @@ func NewWrapperProxy(config configuration.Configuration, cliVersion string, debu
}
certFile, err := os.CreateTemp(tmpDirectory, "snyk-cli-cert-*.crt")
if err != nil {
fmt.Println("failed to create temp cert file")
logger.Println("failed to create temp cert file")
return nil, err
}
defer certFile.Close()

p.CertificateLocation = certFile.Name() // gives full path, not just the name
certificateLocation := certFile.Name() // gives full path, not just the name

rootCAs, err := x509.SystemCertPool()
if err != nil {
Expand All @@ -105,27 +105,42 @@ func NewWrapperProxy(config configuration.Configuration, cliVersion string, debu
}
}

debugLogger.Debug().Msgf("Using additional CAs from file: %v", extraCaCertFile)
logger.Debug().Msgf("Using additional CAs from file: %v", extraCaCertFile)
}
}

debugLogger.Debug().Msgf("Temporary CertificateLocation: %v", p.CertificateLocation)
logger.Debug().Msgf("Temporary CertificateLocation: %v", certificateLocation)
certPEMString := string(certPEMBlock)
err = utils.WriteToFile(p.CertificateLocation, certPEMString)
err = utils.WriteToFile(certificateLocation, certPEMString)
if err != nil {
fmt.Println("failed to write cert to file")
logger.Print("failed to write cert to file")
return nil, err
}

err = setCAFromBytes(certPEMBlock, keyPEMBlock)
err = setGlobalProxyCA(certPEMBlock, keyPEMBlock)
if err != nil {
return nil, err
}

return &CaData{
CertPool: rootCAs,
CertFile: certificateLocation,
}, nil
}

func NewWrapperProxy(config configuration.Configuration, cliVersion string, debugLogger *zerolog.Logger, ca CaData) (*WrapperProxy, error) {
var p WrapperProxy
p.cliVersion = cliVersion
p.addHeaderFunc = func(request *http.Request) error { return nil }
p.DebugLogger = debugLogger
p.CertificateLocation = ca.CertFile

insecureSkipVerify := config.GetBool(configuration.INSECURE_HTTPS)

p.transport = &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: insecureSkipVerify, // goproxy defaults to true
RootCAs: rootCAs,
RootCAs: ca.CertPool,
},
}

Expand Down Expand Up @@ -242,18 +257,9 @@ func (p *WrapperProxy) Stop() {

func (p *WrapperProxy) Close() {
p.Stop()

p.DebugLogger.Print("deleting temp cert file:", p.CertificateLocation)
err := os.Remove(p.CertificateLocation)
if err != nil {
p.DebugLogger.Print("failed to delete cert file")
p.DebugLogger.Print(err)
} else {
p.DebugLogger.Print("deleted temp cert file:", p.CertificateLocation)
}
}

func setCAFromBytes(certPEMBlock []byte, keyPEMBlock []byte) error {
func setGlobalProxyCA(certPEMBlock []byte, keyPEMBlock []byte) error {
goproxyCa, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
if err != nil {
return err
Expand Down
Loading

0 comments on commit 9fa49db

Please sign in to comment.