From fa66aedbe28e1f369b30eb3e523fe2c2f48e657f Mon Sep 17 00:00:00 2001 From: filip Date: Thu, 7 Mar 2024 11:18:59 +0100 Subject: [PATCH 1/6] Add version consolidation flag (naming & doc work in progress) --- internal/cmd/scan/scan.go | 14 ++++++++++++++ internal/scan/scanner.go | 1 + 2 files changed, 15 insertions(+) diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index 736fd846..aee6c8b9 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -23,6 +23,7 @@ var jsonFilePath string var exclusions = file.Exclusions() var verbose bool var regenerate int +var versionConsolidation int var noResolve bool var noFingerprint bool var passOnDowntime bool @@ -40,6 +41,7 @@ const ( IntegrationFlag = "integration" ExclusionFlag = "exclusion" VerboseFlag = "verbose" + VersionConsolidationFlag = "version-consolidation" RegenerateFlag = "regenerate" NoResolveFlag = "no-resolve" FingerprintFlag = "fingerprint" @@ -110,6 +112,17 @@ $ debricked scan . `+exampleFlags) "\nExample:\n$ debricked resolve . --regenerate=1", }, "\n") cmd.Flags().IntVar(®enerate, RegenerateFlag, 0, regenerateDoc) + versionConsolidationDoc := strings.Join( + []string{ + "Toggles version consolidation of manifest and manifestless resolition with 3 modes:\n", + "Consolidation Level | Meaning", + "------------------- | -------", + "0 | No consolidation", + "1 (default) | Use manifest version hints", + "2 (not implemented) | Combine version and package hints", + "\nExample:\n$ debricked scan . --version-consolidation=1", + }, "\n") + cmd.Flags().IntVar(&versionConsolidation, VersionConsolidationFlag, 1, versionConsolidationDoc) verboseDoc := strings.Join( []string{ "This flag allows you to reduce error output for resolution.", @@ -156,6 +169,7 @@ func RunE(s *scan.IScanner) func(_ *cobra.Command, args []string) error { Exclusions: viper.GetStringSlice(ExclusionFlag), Verbose: viper.GetBool(VerboseFlag), Regenerate: viper.GetInt(RegenerateFlag), + VersionConsolidation: viper.GetInt(VersionConsolidationFlag), RepositoryName: viper.GetString(RepositoryFlag), CommitName: viper.GetString(CommitFlag), BranchName: viper.GetString(BranchFlag), diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index e6555dba..67d952fb 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -50,6 +50,7 @@ type DebrickedOptions struct { Exclusions []string Verbose bool Regenerate int + VersionConsolidation int RepositoryName string CommitName string BranchName string From 4072825a0669950d58c19d962e6b8e2963787542 Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 11 Mar 2024 10:31:30 +0100 Subject: [PATCH 2/6] Add version consolidation to file upload (flag passtrough) --- internal/scan/scanner.go | 1 + internal/upload/batch.go | 18 ++++++++++-------- internal/upload/batch_test.go | 10 +++++----- internal/upload/uploader.go | 3 ++- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index 67d952fb..cf79ac1f 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -207,6 +207,7 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g GitMetaObject: gitMetaObject, IntegrationsName: options.IntegrationName, CallGraphUploadTimeout: options.CallGraphUploadTimeout, + VersionConsolidation: options.VersionConsolidation, } result, err := (*dScanner.uploader).Upload(uploaderOptions) if err != nil { diff --git a/internal/upload/batch.go b/internal/upload/batch.go index 2663e47b..f1af9d3d 100644 --- a/internal/upload/batch.go +++ b/internal/upload/batch.go @@ -32,16 +32,17 @@ var ( const callgraphName = "debricked-call-graph" type uploadBatch struct { - client *client.IDebClient - fileGroups file.Groups - gitMetaObject *git.MetaObject - integrationName string - ciUploadId int - callGraphTimeout int + client *client.IDebClient + fileGroups file.Groups + gitMetaObject *git.MetaObject + integrationName string + ciUploadId int + callGraphTimeout int + versionConsolidation int } -func newUploadBatch(client *client.IDebClient, fileGroups file.Groups, gitMetaObject *git.MetaObject, integrationName string, callGraphTimeout int) *uploadBatch { - return &uploadBatch{client: client, fileGroups: fileGroups, gitMetaObject: gitMetaObject, integrationName: integrationName, ciUploadId: 0, callGraphTimeout: callGraphTimeout} +func newUploadBatch(client *client.IDebClient, fileGroups file.Groups, gitMetaObject *git.MetaObject, integrationName string, callGraphTimeout int, versionConsolidation int) *uploadBatch { + return &uploadBatch{client: client, fileGroups: fileGroups, gitMetaObject: gitMetaObject, integrationName: integrationName, ciUploadId: 0, callGraphTimeout: callGraphTimeout, versionConsolidation: versionConsolidation} } // upload concurrently posts all file groups to Debricked @@ -123,6 +124,7 @@ func (uploadBatch *uploadBatch) uploadFile(filePath string, timeout int) error { _ = writer.WriteField("commitName", uploadBatch.gitMetaObject.CommitName) _ = writer.WriteField("repositoryUrl", uploadBatch.gitMetaObject.RepositoryUrl) _ = writer.WriteField("branchName", uploadBatch.gitMetaObject.BranchName) + _ = writer.WriteField("versionConsolidation", strconv.Itoa(uploadBatch.versionConsolidation)) if uploadBatch.initialized() { _ = writer.WriteField("ciUploadId", strconv.Itoa(uploadBatch.ciUploadId)) } diff --git a/internal/upload/batch_test.go b/internal/upload/batch_test.go index 62af0fc8..730e9b5a 100644 --- a/internal/upload/batch_test.go +++ b/internal/upload/batch_test.go @@ -36,7 +36,7 @@ func TestUploadWithBadFiles(t *testing.T) { clientMock.AddMockResponse(mockRes) clientMock.AddMockResponse(mockRes) c = clientMock - batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60) + batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, 1) var buf bytes.Buffer log.SetOutput(&buf) err = batch.upload() @@ -48,7 +48,7 @@ func TestUploadWithBadFiles(t *testing.T) { } func TestInitAnalysisWithoutAnyFiles(t *testing.T) { - batch := newUploadBatch(nil, file.Groups{}, nil, "CLI", 10*60) + batch := newUploadBatch(nil, file.Groups{}, nil, "CLI", 10*60, 1) err := batch.initAnalysis() assert.ErrorContains(t, err, "failed to find dependency files") @@ -71,7 +71,7 @@ func TestWaitWithPollingTerminatedError(t *testing.T) { } clientMock.AddMockResponse(mockRes) c = clientMock - batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60) + batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, 1) uploadResult, err := batch.wait() @@ -96,7 +96,7 @@ func TestInitUploadBadFile(t *testing.T) { clientMock.AddMockResponse(mockRes) var c client.IDebClient = clientMock - batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60) + batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, 1) files, err := batch.initUpload() @@ -123,7 +123,7 @@ func TestInitUpload(t *testing.T) { clientMock.AddMockResponse(mockRes) var c client.IDebClient = clientMock - batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60) + batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, 1) files, err := batch.initUpload() diff --git a/internal/upload/uploader.go b/internal/upload/uploader.go index 90203944..0eb865c0 100644 --- a/internal/upload/uploader.go +++ b/internal/upload/uploader.go @@ -15,6 +15,7 @@ type DebrickedOptions struct { GitMetaObject git.MetaObject IntegrationsName string CallGraphUploadTimeout int + VersionConsolidation int } type IUploader interface { @@ -35,7 +36,7 @@ func NewUploader(c client.IDebClient) (*Uploader, error) { func (uploader *Uploader) Upload(o IOptions) (*UploadResult, error) { dOptions := o.(DebrickedOptions) - batch := newUploadBatch(uploader.client, dOptions.FileGroups, &dOptions.GitMetaObject, dOptions.IntegrationsName, dOptions.CallGraphUploadTimeout) + batch := newUploadBatch(uploader.client, dOptions.FileGroups, &dOptions.GitMetaObject, dOptions.IntegrationsName, dOptions.CallGraphUploadTimeout, dOptions.VersionConsolidation) err := batch.upload() if err != nil { From 5fc6ab232250cf26aaa18b0ac2ce614cc846b198 Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 11 Mar 2024 14:33:21 +0100 Subject: [PATCH 3/6] Change flag to boolean --- internal/cmd/scan/scan.go | 19 +++++++------------ internal/scan/scanner.go | 4 ++-- internal/upload/batch.go | 23 ++++++++++------------- internal/upload/batch_test.go | 10 +++++----- internal/upload/uploader.go | 4 ++-- 5 files changed, 26 insertions(+), 34 deletions(-) diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index aee6c8b9..7772fe5e 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -23,7 +23,7 @@ var jsonFilePath string var exclusions = file.Exclusions() var verbose bool var regenerate int -var versionConsolidation int +var versionHint bool var noResolve bool var noFingerprint bool var passOnDowntime bool @@ -41,7 +41,7 @@ const ( IntegrationFlag = "integration" ExclusionFlag = "exclusion" VerboseFlag = "verbose" - VersionConsolidationFlag = "version-consolidation" + VersionHintFlag = "version-hint" RegenerateFlag = "regenerate" NoResolveFlag = "no-resolve" FingerprintFlag = "fingerprint" @@ -112,17 +112,12 @@ $ debricked scan . `+exampleFlags) "\nExample:\n$ debricked resolve . --regenerate=1", }, "\n") cmd.Flags().IntVar(®enerate, RegenerateFlag, 0, regenerateDoc) - versionConsolidationDoc := strings.Join( + versionHintDoc := strings.Join( []string{ - "Toggles version consolidation of manifest and manifestless resolition with 3 modes:\n", - "Consolidation Level | Meaning", - "------------------- | -------", - "0 | No consolidation", - "1 (default) | Use manifest version hints", - "2 (not implemented) | Combine version and package hints", - "\nExample:\n$ debricked scan . --version-consolidation=1", + "Toggles version hinting, i.e using manifest versions to help manifestless resolution.\n", + "\nExample:\n$ debricked scan . --version-hint=False", }, "\n") - cmd.Flags().IntVar(&versionConsolidation, VersionConsolidationFlag, 1, versionConsolidationDoc) + cmd.Flags().BoolVar(&versionHint, VersionHintFlag, true, versionHintDoc) verboseDoc := strings.Join( []string{ "This flag allows you to reduce error output for resolution.", @@ -169,7 +164,7 @@ func RunE(s *scan.IScanner) func(_ *cobra.Command, args []string) error { Exclusions: viper.GetStringSlice(ExclusionFlag), Verbose: viper.GetBool(VerboseFlag), Regenerate: viper.GetInt(RegenerateFlag), - VersionConsolidation: viper.GetInt(VersionConsolidationFlag), + VersionHint: viper.GetBool(VersionHintFlag), RepositoryName: viper.GetString(RepositoryFlag), CommitName: viper.GetString(CommitFlag), BranchName: viper.GetString(BranchFlag), diff --git a/internal/scan/scanner.go b/internal/scan/scanner.go index cf79ac1f..c649a002 100644 --- a/internal/scan/scanner.go +++ b/internal/scan/scanner.go @@ -50,7 +50,7 @@ type DebrickedOptions struct { Exclusions []string Verbose bool Regenerate int - VersionConsolidation int + VersionHint bool RepositoryName string CommitName string BranchName string @@ -207,7 +207,7 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g GitMetaObject: gitMetaObject, IntegrationsName: options.IntegrationName, CallGraphUploadTimeout: options.CallGraphUploadTimeout, - VersionConsolidation: options.VersionConsolidation, + VersionHint: options.VersionHint, } result, err := (*dScanner.uploader).Upload(uploaderOptions) if err != nil { diff --git a/internal/upload/batch.go b/internal/upload/batch.go index f1af9d3d..641cbdd7 100644 --- a/internal/upload/batch.go +++ b/internal/upload/batch.go @@ -32,17 +32,17 @@ var ( const callgraphName = "debricked-call-graph" type uploadBatch struct { - client *client.IDebClient - fileGroups file.Groups - gitMetaObject *git.MetaObject - integrationName string - ciUploadId int - callGraphTimeout int - versionConsolidation int + client *client.IDebClient + fileGroups file.Groups + gitMetaObject *git.MetaObject + integrationName string + ciUploadId int + callGraphTimeout int + versionHint bool } -func newUploadBatch(client *client.IDebClient, fileGroups file.Groups, gitMetaObject *git.MetaObject, integrationName string, callGraphTimeout int, versionConsolidation int) *uploadBatch { - return &uploadBatch{client: client, fileGroups: fileGroups, gitMetaObject: gitMetaObject, integrationName: integrationName, ciUploadId: 0, callGraphTimeout: callGraphTimeout, versionConsolidation: versionConsolidation} +func newUploadBatch(client *client.IDebClient, fileGroups file.Groups, gitMetaObject *git.MetaObject, integrationName string, callGraphTimeout int, versionHint bool) *uploadBatch { + return &uploadBatch{client: client, fileGroups: fileGroups, gitMetaObject: gitMetaObject, integrationName: integrationName, ciUploadId: 0, callGraphTimeout: callGraphTimeout, versionHint: versionHint} } // upload concurrently posts all file groups to Debricked @@ -124,11 +124,10 @@ func (uploadBatch *uploadBatch) uploadFile(filePath string, timeout int) error { _ = writer.WriteField("commitName", uploadBatch.gitMetaObject.CommitName) _ = writer.WriteField("repositoryUrl", uploadBatch.gitMetaObject.RepositoryUrl) _ = writer.WriteField("branchName", uploadBatch.gitMetaObject.BranchName) - _ = writer.WriteField("versionConsolidation", strconv.Itoa(uploadBatch.versionConsolidation)) + _ = writer.WriteField("versionHint", strconv.FormatBool(uploadBatch.versionHint)) if uploadBatch.initialized() { _ = writer.WriteField("ciUploadId", strconv.Itoa(uploadBatch.ciUploadId)) } - response, err := (*uploadBatch.client).Post( "/api/1.0/open/uploads/dependencies/files", writer.FormDataContentType(), @@ -138,11 +137,9 @@ func (uploadBatch *uploadBatch) uploadFile(filePath string, timeout int) error { if err != nil { return err } - if !uploadBatch.initialized() { data, _ := io.ReadAll(response.Body) defer response.Body.Close() - uFile := uploadedFile{} _ = json.Unmarshal(data, &uFile) if uFile.CiUploadId == 0 { diff --git a/internal/upload/batch_test.go b/internal/upload/batch_test.go index 730e9b5a..3d8f1174 100644 --- a/internal/upload/batch_test.go +++ b/internal/upload/batch_test.go @@ -36,7 +36,7 @@ func TestUploadWithBadFiles(t *testing.T) { clientMock.AddMockResponse(mockRes) clientMock.AddMockResponse(mockRes) c = clientMock - batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, 1) + batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true) var buf bytes.Buffer log.SetOutput(&buf) err = batch.upload() @@ -48,7 +48,7 @@ func TestUploadWithBadFiles(t *testing.T) { } func TestInitAnalysisWithoutAnyFiles(t *testing.T) { - batch := newUploadBatch(nil, file.Groups{}, nil, "CLI", 10*60, 1) + batch := newUploadBatch(nil, file.Groups{}, nil, "CLI", 10*60, true) err := batch.initAnalysis() assert.ErrorContains(t, err, "failed to find dependency files") @@ -71,7 +71,7 @@ func TestWaitWithPollingTerminatedError(t *testing.T) { } clientMock.AddMockResponse(mockRes) c = clientMock - batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, 1) + batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true) uploadResult, err := batch.wait() @@ -96,7 +96,7 @@ func TestInitUploadBadFile(t *testing.T) { clientMock.AddMockResponse(mockRes) var c client.IDebClient = clientMock - batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, 1) + batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true) files, err := batch.initUpload() @@ -123,7 +123,7 @@ func TestInitUpload(t *testing.T) { clientMock.AddMockResponse(mockRes) var c client.IDebClient = clientMock - batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, 1) + batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true) files, err := batch.initUpload() diff --git a/internal/upload/uploader.go b/internal/upload/uploader.go index 0eb865c0..60745f88 100644 --- a/internal/upload/uploader.go +++ b/internal/upload/uploader.go @@ -15,7 +15,7 @@ type DebrickedOptions struct { GitMetaObject git.MetaObject IntegrationsName string CallGraphUploadTimeout int - VersionConsolidation int + VersionHint bool } type IUploader interface { @@ -36,7 +36,7 @@ func NewUploader(c client.IDebClient) (*Uploader, error) { func (uploader *Uploader) Upload(o IOptions) (*UploadResult, error) { dOptions := o.(DebrickedOptions) - batch := newUploadBatch(uploader.client, dOptions.FileGroups, &dOptions.GitMetaObject, dOptions.IntegrationsName, dOptions.CallGraphUploadTimeout, dOptions.VersionConsolidation) + batch := newUploadBatch(uploader.client, dOptions.FileGroups, &dOptions.GitMetaObject, dOptions.IntegrationsName, dOptions.CallGraphUploadTimeout, dOptions.VersionHint) err := batch.upload() if err != nil { From bee87e29fbf3ad3e3bc71be757169269feda4590 Mon Sep 17 00:00:00 2001 From: filip Date: Wed, 13 Mar 2024 14:30:03 +0100 Subject: [PATCH 4/6] Change to upload endpoint --- internal/upload/batch.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/upload/batch.go b/internal/upload/batch.go index 641cbdd7..3810664c 100644 --- a/internal/upload/batch.go +++ b/internal/upload/batch.go @@ -124,7 +124,6 @@ func (uploadBatch *uploadBatch) uploadFile(filePath string, timeout int) error { _ = writer.WriteField("commitName", uploadBatch.gitMetaObject.CommitName) _ = writer.WriteField("repositoryUrl", uploadBatch.gitMetaObject.RepositoryUrl) _ = writer.WriteField("branchName", uploadBatch.gitMetaObject.BranchName) - _ = writer.WriteField("versionHint", strconv.FormatBool(uploadBatch.versionHint)) if uploadBatch.initialized() { _ = writer.WriteField("ciUploadId", strconv.Itoa(uploadBatch.ciUploadId)) } @@ -162,6 +161,7 @@ func (uploadBatch *uploadBatch) initAnalysis() error { IntegrationName: uploadBatch.integrationName, CommitName: uploadBatch.gitMetaObject.CommitName, Author: uploadBatch.gitMetaObject.Author, + VersionHint: strconv.FormatBool(uploadBatch.versionHint), DebrickedIntegration: "cli", }) @@ -276,6 +276,7 @@ type uploadFinish struct { CommitName string `json:"commitName"` Author string `json:"author"` DebrickedIntegration string `json:"debrickedIntegration"` + VersionHint string `json:"versionHint"` } func getRelativeFilePath(filePath string) string { From 1abe11ae4f0f1b6b84dd23c29fae9bc0c235b58e Mon Sep 17 00:00:00 2001 From: filip Date: Thu, 21 Mar 2024 14:21:38 +0100 Subject: [PATCH 5/6] Normalize bool hint --- internal/cmd/scan/scan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/scan/scan.go b/internal/cmd/scan/scan.go index 7772fe5e..722e9c2f 100644 --- a/internal/cmd/scan/scan.go +++ b/internal/cmd/scan/scan.go @@ -115,7 +115,7 @@ $ debricked scan . `+exampleFlags) versionHintDoc := strings.Join( []string{ "Toggles version hinting, i.e using manifest versions to help manifestless resolution.\n", - "\nExample:\n$ debricked scan . --version-hint=False", + "\nExample:\n$ debricked scan . --version-hint=false", }, "\n") cmd.Flags().BoolVar(&versionHint, VersionHintFlag, true, versionHintDoc) verboseDoc := strings.Join( From ab95bba506e379cde83034c028c82acd6931dadc Mon Sep 17 00:00:00 2001 From: filip Date: Thu, 21 Mar 2024 15:04:21 +0100 Subject: [PATCH 6/6] Make flag actual json boolean --- internal/upload/batch.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/upload/batch.go b/internal/upload/batch.go index 3810664c..ca17b7e2 100644 --- a/internal/upload/batch.go +++ b/internal/upload/batch.go @@ -161,7 +161,7 @@ func (uploadBatch *uploadBatch) initAnalysis() error { IntegrationName: uploadBatch.integrationName, CommitName: uploadBatch.gitMetaObject.CommitName, Author: uploadBatch.gitMetaObject.Author, - VersionHint: strconv.FormatBool(uploadBatch.versionHint), + VersionHint: uploadBatch.versionHint, DebrickedIntegration: "cli", }) @@ -276,7 +276,7 @@ type uploadFinish struct { CommitName string `json:"commitName"` Author string `json:"author"` DebrickedIntegration string `json:"debrickedIntegration"` - VersionHint string `json:"versionHint"` + VersionHint bool `json:"versionHint"` } func getRelativeFilePath(filePath string) string {