From 36c555ea61240892ee4d230117c5bccaff4bcbfc Mon Sep 17 00:00:00 2001 From: Ivan Nikiforov Date: Wed, 16 Oct 2024 13:45:23 +0200 Subject: [PATCH 01/28] fix: too long string written to cpe git.commitMessage (#5147) * fix too long CPE string written to git/commitMessage * Add debug log * Fix debug log * Truncate long git commit message title * Add tests for truncateString * Fix test * Fix tests * Fix tests --------- Co-authored-by: Ivan Nikiforov --- cmd/artifactPrepareVersion.go | 17 ++++++++++++++++- cmd/artifactPrepareVersion_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/cmd/artifactPrepareVersion.go b/cmd/artifactPrepareVersion.go index 173a693dac..4c7f77320d 100644 --- a/cmd/artifactPrepareVersion.go +++ b/cmd/artifactPrepareVersion.go @@ -237,7 +237,13 @@ func runArtifactPrepareVersion(config *artifactPrepareVersionOptions, telemetryD commonPipelineEnvironment.git.commitID = gitCommitID // this commitID changes and is not necessarily the HEAD commitID commonPipelineEnvironment.artifactVersion = newVersion commonPipelineEnvironment.originalArtifactVersion = version - commonPipelineEnvironment.git.commitMessage = gitCommitMessage + + gitCommitMessages := strings.Split(gitCommitMessage, "\n") + commitMessage := truncateString(gitCommitMessages[0], 50) // Github recommends to keep commit message title less than 50 chars + + commonPipelineEnvironment.git.commitMessage = commitMessage + + log.Entry().Debug("CPE git commitMessage:", commitMessage) // we may replace GetVersion() above with GetCoordinates() at some point ... coordinates, err := artifact.GetCoordinates() @@ -254,6 +260,15 @@ func runArtifactPrepareVersion(config *artifactPrepareVersionOptions, telemetryD return nil } +func truncateString(str string, maxLength int) string { + chars := []rune(str) + + if len(chars) > maxLength { + return string(chars[:maxLength]) + "..." + } + return str +} + func openGit() (gitRepository, error) { workdir, _ := os.Getwd() return gitUtils.PlainOpen(workdir) diff --git a/cmd/artifactPrepareVersion_test.go b/cmd/artifactPrepareVersion_test.go index 9d42f44dd5..9199865822 100644 --- a/cmd/artifactPrepareVersion_test.go +++ b/cmd/artifactPrepareVersion_test.go @@ -887,3 +887,33 @@ func TestPropagateVersion(t *testing.T) { assert.Contains(t, fmt.Sprint(err), "failed to retrieve artifact") }) } + +func TestTruncateString(t *testing.T) { + t.Run("input string longer than maxLength - truncate", func(t *testing.T) { + inputStr := "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor" + expected := "Lorem ipsum dolor sit amet, consectetur adipiscing..." + + outputStr := truncateString(inputStr, 50) + assert.Equal(t, outputStr, expected) + }) + + t.Run("input string shorter than maxLength - return as is", func(t *testing.T) { + inputStr := "Lorem ipsum dolor sit amet" + outputStr := truncateString(inputStr, 50) + + assert.Equal(t, outputStr, inputStr) + }) + + t.Run("input string contains unicode chars", func(t *testing.T) { + inputStr := "パイパーは素晴らしい図書館です" + expected := "パイパーは..." + + outputStr := truncateString(inputStr, 5) + assert.Equal(t, outputStr, expected) + }) + + t.Run("input string is empty", func(t *testing.T) { + outputStr := truncateString("", 5) + assert.Equal(t, outputStr, "") + }) +} From e1563e023717650696c7abb601ca6b3c050edec2 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Wed, 16 Oct 2024 14:10:52 +0200 Subject: [PATCH 02/28] Add additional info to coordinates (#5149) --- pkg/versioning/versioning.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/versioning/versioning.go b/pkg/versioning/versioning.go index a716cb192f..9ae5dfd79b 100644 --- a/pkg/versioning/versioning.go +++ b/pkg/versioning/versioning.go @@ -13,13 +13,13 @@ import ( // Coordinates to address the artifact coordinates like groupId, artifactId, version and packaging type Coordinates struct { - GroupID string - ArtifactID string - Version string - Packaging string - BuildPath string - URL string - PURL string + GroupID string `json:"groupId"` + ArtifactID string `json:"artifactId"` + Version string `json:"version"` + Packaging string `json:"packaging"` + BuildPath string `json:"buildPath"` + URL string `json:"url"` + PURL string `json:"purl"` } // Artifact defines the versioning operations for various build tools From 34a60daad1d241ffc6c05c8003468e7e2c57275b Mon Sep 17 00:00:00 2001 From: Ivan Nikiforov Date: Wed, 16 Oct 2024 16:09:51 +0200 Subject: [PATCH 03/28] Update documentation for scriptArguments param (#5128) * Update documentation for scriptArguments param * Update resources/metadata/shellExecute.yaml Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * Update doc with new examples * Fix typo --------- Co-authored-by: Ivan Nikiforov Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> --- cmd/shellExecute_generated.go | 2 +- resources/metadata/shellExecute.yaml | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/shellExecute_generated.go b/cmd/shellExecute_generated.go index a69334c40f..3d0eeacb1a 100644 --- a/cmd/shellExecute_generated.go +++ b/cmd/shellExecute_generated.go @@ -142,7 +142,7 @@ func ShellExecuteCommand() *cobra.Command { func addShellExecuteFlags(cmd *cobra.Command, stepConfig *shellExecuteOptions) { cmd.Flags().StringSliceVar(&stepConfig.Sources, "sources", []string{}, "Scripts paths that must be present in the current workspace or https links to scripts. Only https urls from github are allowed and must be in the format :https://{githubBaseurl}/api/v3/repos/{owner}/{repository}/contents/{path to script} Authentication for the download is only supported via the 'githubToken' param. Make sure the script has the necessary execute permissions.") cmd.Flags().StringVar(&stepConfig.GithubToken, "githubToken", os.Getenv("PIPER_githubToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line") - cmd.Flags().StringSliceVar(&stepConfig.ScriptArguments, "scriptArguments", []string{}, "scriptArguments that are needed to be passed to scripts. the scriptArguments list is a flat list and has a positional relationship to the `sources` param. For e.g. The scriptArguments string at position 1 will be considered as the argument(s) for script at position 1 in `sources` list. For multiple arguments for a script please add them as a comma seperated string.") + cmd.Flags().StringSliceVar(&stepConfig.ScriptArguments, "scriptArguments", []string{}, "scriptArguments that need to be passed to the scripts.") } diff --git a/resources/metadata/shellExecute.yaml b/resources/metadata/shellExecute.yaml index c5a2a24605..32dafb73b3 100644 --- a/resources/metadata/shellExecute.yaml +++ b/resources/metadata/shellExecute.yaml @@ -42,8 +42,18 @@ spec: - PARAMETERS - STAGES - STEPS - description: scriptArguments that are needed to be passed to scripts. the scriptArguments list is a flat list and has a positional relationship to the `sources` param. - For e.g. The scriptArguments string at position 1 will be considered as the argument(s) for script at position 1 in `sources` list. For multiple arguments for a script please add them as a comma seperated string. + description: "scriptArguments that need to be passed to the scripts." + longDescription: |- + The scriptArguments list is a flat list and has a positional relationship to the `sources` parameter. + + For example, the `scriptArguments` string at position 0 will be considered as the argument(s) for script at position 0 in `sources` list. + `--sources ".pipeline/firstScript.sh" --sources ".pipeline/secondScript.sh" --scriptArguments "$(first_script_arg)" --scriptArguments "$(second_script_arg)"` + + For multiple arguments for a particular script, please add them as a comma-separated string enclosed in additional quotes, e.g.: + `--sources ".pipeline/yourScript.sh" --scriptArguments "\"$(first_arg),$(second_arg)\""` + + For multiple scripts with multiple arguments per each script your command would look like: + `--sources ".pipeline/firstScript.sh" --sources ".pipeline/secondScript.sh" --scriptArguments "\"$(first_script_arg1),$(first_script_arg2)\"" --scriptArguments "\"$(second_script_arg1),$(second_script_arg2)\""` mandatory: false containers: - name: shell From b317b1d1ebccca3e5702d1b6451c9f74f88eb720 Mon Sep 17 00:00:00 2001 From: Adrien <99400874+hubadr@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:16:14 +0200 Subject: [PATCH 04/28] Add debug log for zip file content (#5152) Co-authored-by: thtri --- cmd/checkmarxOneExecuteScan.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/checkmarxOneExecuteScan.go b/cmd/checkmarxOneExecuteScan.go index d809304290..c457372c65 100644 --- a/cmd/checkmarxOneExecuteScan.go +++ b/cmd/checkmarxOneExecuteScan.go @@ -943,12 +943,15 @@ func (c *checkmarxOneExecuteScanHelper) zipFolder(source string, zipFile io.Writ return nil } + fileName := strings.TrimPrefix(path, baseDir) noMatch, err := c.isFileNotMatchingPattern(patterns, path, info, utils) if err != nil || noMatch { + if noMatch { + log.Entry().Debugf("Excluded %s", fileName) + } return err } - fileName := strings.TrimPrefix(path, baseDir) writer, err := archive.Create(fileName) if err != nil { return err @@ -960,6 +963,9 @@ func (c *checkmarxOneExecuteScanHelper) zipFolder(source string, zipFile io.Writ } defer file.Close() _, err = io.Copy(writer, file) + if err == nil { + log.Entry().Debugf("Zipped %s", fileName) + } fileCount++ return err }) From 769067e9960dde0021ecfa5bad099fbed48be1cf Mon Sep 17 00:00:00 2001 From: Dmitrii Pavlukhin Date: Fri, 18 Oct 2024 17:06:41 +0300 Subject: [PATCH 05/28] added-option-to-use-global-config-file (#5145) --- cmd/whitesourceExecuteScan.go | 1 + cmd/whitesourceExecuteScan_generated.go | 11 +++++++++++ pkg/whitesource/scanOptions.go | 7 ++++--- pkg/whitesource/scanUA.go | 7 +++++++ resources/metadata/whitesourceExecuteScan.yaml | 9 +++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/cmd/whitesourceExecuteScan.go b/cmd/whitesourceExecuteScan.go index 1987466d13..14772ec8f6 100644 --- a/cmd/whitesourceExecuteScan.go +++ b/cmd/whitesourceExecuteScan.go @@ -490,6 +490,7 @@ func wsScanOptions(config *ScanOptions) *ws.ScanOptions { AgentDownloadURL: config.AgentDownloadURL, AgentFileName: config.AgentFileName, ConfigFilePath: config.ConfigFilePath, + UseGlobalConfiguration: config.UseGlobalConfiguration, Includes: config.Includes, Excludes: config.Excludes, JreDownloadURL: config.JreDownloadURL, diff --git a/cmd/whitesourceExecuteScan_generated.go b/cmd/whitesourceExecuteScan_generated.go index 30f29a67f5..5456231fe9 100644 --- a/cmd/whitesourceExecuteScan_generated.go +++ b/cmd/whitesourceExecuteScan_generated.go @@ -33,6 +33,7 @@ type whitesourceExecuteScanOptions struct { BuildDescriptorFile string `json:"buildDescriptorFile,omitempty"` BuildTool string `json:"buildTool,omitempty"` ConfigFilePath string `json:"configFilePath,omitempty"` + UseGlobalConfiguration bool `json:"useGlobalConfiguration,omitempty"` ContainerRegistryPassword string `json:"containerRegistryPassword,omitempty"` ContainerRegistryUser string `json:"containerRegistryUser,omitempty"` CreateProductFromPipeline bool `json:"createProductFromPipeline,omitempty"` @@ -352,6 +353,7 @@ func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceE cmd.Flags().StringVar(&stepConfig.BuildDescriptorFile, "buildDescriptorFile", os.Getenv("PIPER_buildDescriptorFile"), "Explicit path to the build descriptor file.") cmd.Flags().StringVar(&stepConfig.BuildTool, "buildTool", os.Getenv("PIPER_buildTool"), "Defines the tool which is used for building the artifact.") cmd.Flags().StringVar(&stepConfig.ConfigFilePath, "configFilePath", `./wss-unified-agent.config`, "Explicit path to the WhiteSource Unified Agent configuration file.") + cmd.Flags().BoolVar(&stepConfig.UseGlobalConfiguration, "useGlobalConfiguration", false, "The parameter is applicable for multi-module mend projects. If set to true, the configuration file will be used for all modules. Otherwise each module will require its own configuration file in the module folder.") cmd.Flags().StringVar(&stepConfig.ContainerRegistryPassword, "containerRegistryPassword", os.Getenv("PIPER_containerRegistryPassword"), "For `buildTool: docker`: Password for container registry access - typically provided by the CI/CD environment.") cmd.Flags().StringVar(&stepConfig.ContainerRegistryUser, "containerRegistryUser", os.Getenv("PIPER_containerRegistryUser"), "For `buildTool: docker`: Username for container registry access - typically provided by the CI/CD environment.") cmd.Flags().BoolVar(&stepConfig.CreateProductFromPipeline, "createProductFromPipeline", true, "Whether to create the related WhiteSource product on the fly based on the supplied pipeline configuration.") @@ -527,6 +529,15 @@ func whitesourceExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: `./wss-unified-agent.config`, }, + { + Name: "useGlobalConfiguration", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "GENERAL", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: false, + }, { Name: "containerRegistryPassword", ResourceRef: []config.ResourceReference{ diff --git a/pkg/whitesource/scanOptions.go b/pkg/whitesource/scanOptions.go index 7a3bcb6645..beb6038d9c 100644 --- a/pkg/whitesource/scanOptions.go +++ b/pkg/whitesource/scanOptions.go @@ -31,9 +31,10 @@ type ScanOptions struct { DefaultNpmRegistry string NpmIncludeDevDependencies bool - AgentDownloadURL string - AgentFileName string - ConfigFilePath string + AgentDownloadURL string + AgentFileName string + ConfigFilePath string + UseGlobalConfiguration bool JreDownloadURL string diff --git a/pkg/whitesource/scanUA.go b/pkg/whitesource/scanUA.go index 54e84ddfb3..1e3e3fe01e 100644 --- a/pkg/whitesource/scanUA.go +++ b/pkg/whitesource/scanUA.go @@ -127,6 +127,13 @@ func (s *Scan) ExecuteUAScanInPath(config *ScanOptions, utils Utils, scanPath st } } + if config.UseGlobalConfiguration { + config.ConfigFilePath, err = filepath.Abs(config.ConfigFilePath) + if err != nil { + return err + } + } + configPath, err := config.RewriteUAConfigurationFile(utils, s.AggregateProjectName, config.Verbose) if err != nil { return err diff --git a/resources/metadata/whitesourceExecuteScan.yaml b/resources/metadata/whitesourceExecuteScan.yaml index 95adcdd7fc..9b030ac5b4 100644 --- a/resources/metadata/whitesourceExecuteScan.yaml +++ b/resources/metadata/whitesourceExecuteScan.yaml @@ -128,6 +128,15 @@ spec: - STAGES - STEPS default: ./wss-unified-agent.config + - name: useGlobalConfiguration + type: bool + description: "The parameter is applicable for multi-module mend projects. If set to true, the configuration file will be used for all modules. Otherwise each module will require its own configuration file in the module folder." + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + default: false - name: containerRegistryPassword description: "For `buildTool: docker`: Password for container registry access - typically provided by the CI/CD environment." type: string From 4eb1756b54472c34cad9f4ecf389d74f826f36c2 Mon Sep 17 00:00:00 2001 From: maxcask Date: Mon, 21 Oct 2024 12:23:50 +0400 Subject: [PATCH 06/28] fix(Central Build): Fix handling legacy stage name for Jenkins pipelines (#5151) * add handle stageName * some improvements --------- Co-authored-by: maxcask Co-authored-by: Googlom Co-authored-by: Googlom <36107508+Googlom@users.noreply.github.com> --- pkg/config/evaluation.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/config/evaluation.go b/pkg/config/evaluation.go index 1775181a51..4482a2d296 100644 --- a/pkg/config/evaluation.go +++ b/pkg/config/evaluation.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" + "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/orchestrator" "github.com/SAP/jenkins-library/pkg/piperutils" ) @@ -39,6 +40,9 @@ func (r *RunConfigV1) evaluateConditionsV1(config *Config, utils piperutils.File // to also consider using the technical name. stageName := stage.DisplayName + // Central Build in Jenkins was renamed to Build. + handleLegacyStageNaming(config, currentOrchestrator, stageName) + // Check #1: Apply explicit activation/deactivation from config file (if any) // and then evaluate stepActive conditions runStep := make(map[string]bool, len(stage.Steps)) @@ -305,3 +309,23 @@ func anyOtherStepIsActive(targetStep string, runSteps map[string]bool) bool { return false } + +func handleLegacyStageNaming(c *Config, orchestrator, stageName string) { + if orchestrator == "Jenkins" && stageName == "Build" { + _, buildExists := c.Stages["Build"] + centralBuildStageConfig, centralBuildExists := c.Stages["Central Build"] + if buildExists && centralBuildExists { + log.Entry().Warnf("You have 2 entries for build stage in config.yml. " + + "Parameters defined under 'Central Build' are ignored. " + + "Please use only 'Build'") + return + } + + if centralBuildExists { + c.Stages["Build"] = centralBuildStageConfig + log.Entry().Warnf("You are using 'Central Build' stage in config.yml. " + + "Please move parameters under the 'Build' stage, " + + "since 'Central Build' will be removed in future releases") + } + } +} From 5c47be3f8fe0a483f9b1fd5b3b6a8684eb355aa6 Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:29:34 +0500 Subject: [PATCH 07/28] refactor(vault): Refactor vault package (#5148) * move to old package * go mod * remove old * refactor done * Update pkg/vault/oidc.go Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * commit suggestions Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * commit suggestions Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * commit suggestions --------- Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> --- cmd/vaultRotateSecretId.go | 8 +- go.mod | 13 +- go.sum | 46 +- integration/integration_vault_test.go | 33 +- pkg/config/vault.go | 13 +- pkg/vault/client.go | 480 +++++--------------- pkg/vault/helpers.go | 23 + pkg/vault/oidc.go | 54 +-- pkg/vault/oidc_test.go | 6 +- pkg/vault/vault.go | 286 ++++++++++++ pkg/vault/{client_test.go => vault_test.go} | 57 +-- 11 files changed, 541 insertions(+), 478 deletions(-) create mode 100644 pkg/vault/helpers.go create mode 100644 pkg/vault/vault.go rename pkg/vault/{client_test.go => vault_test.go} (89%) diff --git a/cmd/vaultRotateSecretId.go b/cmd/vaultRotateSecretId.go index c7d6602d93..0532350cf4 100644 --- a/cmd/vaultRotateSecretId.go +++ b/cmd/vaultRotateSecretId.go @@ -41,20 +41,22 @@ func (v vaultRotateSecretIDUtilsBundle) UpdateSecretInStore(config *vaultRotateS func vaultRotateSecretId(config vaultRotateSecretIdOptions, telemetryData *telemetry.CustomData) { - vaultConfig := &vault.Config{ + vaultConfig := &vault.ClientConfig{ Config: &api.Config{ Address: config.VaultServerURL, }, Namespace: config.VaultNamespace, + RoleID: GeneralConfig.VaultRoleID, + SecretID: GeneralConfig.VaultRoleSecretID, } - client, err := vault.NewClientWithAppRole(vaultConfig, GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID) + client, err := vault.NewClient(vaultConfig) if err != nil { log.Entry().WithError(err).Fatal("could not create Vault client") } defer client.MustRevokeToken() utils := vaultRotateSecretIDUtilsBundle{ - Client: &client, + Client: client, config: &config, updateFunc: writeVaultSecretIDToStore, } diff --git a/go.mod b/go.mod index 3fecf3bf95..35fa8e764a 100644 --- a/go.mod +++ b/go.mod @@ -32,8 +32,9 @@ require ( github.com/google/go-containerregistry v0.19.0 github.com/google/go-github/v45 v45.2.0 github.com/google/uuid v1.6.0 - github.com/hashicorp/go-retryablehttp v0.7.2 - github.com/hashicorp/vault/api v1.9.2 + github.com/hashicorp/go-retryablehttp v0.7.7 + github.com/hashicorp/vault/api v1.15.0 + github.com/hashicorp/vault/api/auth/approle v0.8.0 github.com/iancoleman/orderedmap v0.2.0 github.com/imdario/mergo v0.3.15 github.com/influxdata/influxdb-client-go/v2 v2.13.0 @@ -76,7 +77,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecr v1.32.2 // indirect github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.4 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -87,7 +88,7 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/go-jose/go-jose/v3 v3.0.3 // indirect + github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/gnostic-models v0.6.8 // indirect @@ -152,7 +153,6 @@ require ( github.com/aws/smithy-go v1.20.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buildpacks/imgutil v0.0.0-20230919143643-4ec9360d5f02 // indirect - github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/containerd v1.7.20 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect @@ -164,7 +164,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-logr/logr v1.4.1 // indirect @@ -190,7 +190,6 @@ require ( github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect diff --git a/go.sum b/go.sum index c95413fbec..3df2dfff02 100644 --- a/go.sum +++ b/go.sum @@ -196,10 +196,8 @@ github.com/buildpacks/imgutil v0.0.0-20230919143643-4ec9360d5f02/go.mod h1:Ade+4 github.com/buildpacks/lifecycle v0.18.5 h1:lfoUX8jYCUZ2/Tr2AopaRjinqDivkNkcTChzysQTo00= github.com/buildpacks/lifecycle v0.18.5/go.mod h1:Kvuu9IWABPLXc6yHCMtbdmgrGEi7QEiVzi5GGtcAkW0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= -github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -287,9 +285,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= @@ -321,8 +318,8 @@ github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lK github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= -github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U= +github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -565,14 +562,13 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= -github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs= @@ -591,8 +587,10 @@ github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvH github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hashicorp/vault/api v1.9.2 h1:YjkZLJ7K3inKgMZ0wzCU9OHqc+UqMQyXsPXnf3Cl2as= -github.com/hashicorp/vault/api v1.9.2/go.mod h1:jo5Y/ET+hNyz+JnKDt8XLAdKs+AM0G5W0Vp1IrFI8N8= +github.com/hashicorp/vault/api v1.15.0 h1:O24FYQCWwhwKnF7CuSqP30S51rTV7vz1iACXE/pj5DA= +github.com/hashicorp/vault/api v1.15.0/go.mod h1:+5YTO09JGn0u+b6ySD/LLVf8WkJCPLAL2Vkmrn2+CM8= +github.com/hashicorp/vault/api/auth/approle v0.8.0 h1:FuVtWZ0xD6+wz1x0l5s0b4852RmVXQNEiKhVXt6lfQY= +github.com/hashicorp/vault/api/auth/approle v0.8.0/go.mod h1:NV7O9r5JUtNdVnqVZeMHva81AIdpG0WoIQohNt1VCPM= github.com/heroku/color v0.0.6 h1:UTFFMrmMLFcL3OweqP1lAdp8i1y/9oHqkeHjQ/b/Ny0= github.com/heroku/color v0.0.6/go.mod h1:ZBvOcx7cTF2QKOv4LbmoBtNl5uB17qWxGuzZrsi1wLU= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -678,15 +676,11 @@ github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kN github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -889,7 +883,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -1015,7 +1008,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1102,7 +1094,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1158,7 +1149,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1181,10 +1171,7 @@ golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1196,7 +1183,6 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1206,8 +1192,6 @@ golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1222,8 +1206,6 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/integration/integration_vault_test.go b/integration/integration_vault_test.go index 0d29329a68..1be34d67c3 100644 --- a/integration/integration_vault_test.go +++ b/integration/integration_vault_test.go @@ -47,7 +47,7 @@ func TestVaultIntegrationGetSecret(t *testing.T) { assert.NoError(t, err) port, err := vaultContainer.MappedPort(ctx, "8200") host := fmt.Sprintf("http://%s:%s", ip, port.Port()) - config := &vault.Config{Config: &api.Config{Address: host}} + config := &vault.ClientConfig{Config: &api.Config{Address: host}} // setup vault for testing secretData := SecretData{ "key1": "value1", @@ -55,7 +55,7 @@ func TestVaultIntegrationGetSecret(t *testing.T) { } setupVault(t, config, testToken, secretData) - client, err := vault.NewClient(config, testToken) + client, err := vault.NewClientWithToken(config, testToken) assert.NoError(t, err) secret, err := client.GetKvSecret("secret/test") assert.NoError(t, err) @@ -93,14 +93,14 @@ func TestVaultIntegrationWriteSecret(t *testing.T) { assert.NoError(t, err) port, err := vaultContainer.MappedPort(ctx, "8200") host := fmt.Sprintf("http://%s:%s", ip, port.Port()) - config := &vault.Config{Config: &api.Config{Address: host}} + config := &vault.ClientConfig{Config: &api.Config{Address: host}} // setup vault for testing secretData := map[string]string{ "key1": "value1", "key2": "value2", } - client, err := vault.NewClient(config, testToken) + client, err := vault.NewClientWithToken(config, testToken) assert.NoError(t, err) err = client.WriteKvSecret("secret/test", secretData) @@ -159,16 +159,17 @@ func TestVaultIntegrationAppRole(t *testing.T) { assert.NoError(t, err) port, err := vaultContainer.MappedPort(ctx, "8200") host := fmt.Sprintf("http://%s:%s", ip, port.Port()) - config := &vault.Config{Config: &api.Config{Address: host}} + config := &vault.ClientConfig{Config: &api.Config{Address: host}} secretIDMetadata := map[string]interface{}{ "field1": "value1", } roleID, secretID := setupVaultAppRole(t, config, testToken, appRolePath, secretIDMetadata) - + config.RoleID = roleID + config.SecretID = secretID t.Run("Test Vault AppRole login", func(t *testing.T) { - client, err := vault.NewClientWithAppRole(config, roleID, secretID) + client, err := vault.NewClient(config) assert.NoError(t, err) secret, err := client.GetSecret("auth/token/lookup-self") meta := secret.Data["meta"].(SecretData) @@ -178,7 +179,7 @@ func TestVaultIntegrationAppRole(t *testing.T) { }) t.Run("Test Vault AppRoleTTL Fetch", func(t *testing.T) { - client, err := vault.NewClient(config, testToken) + client, err := vault.NewClientWithToken(config, testToken) assert.NoError(t, err) ttl, err := client.GetAppRoleSecretIDTtl(secretID, appRoleName) assert.NoError(t, err) @@ -186,7 +187,7 @@ func TestVaultIntegrationAppRole(t *testing.T) { }) t.Run("Test Vault AppRole Rotation", func(t *testing.T) { - client, err := vault.NewClient(config, testToken) + client, err := vault.NewClientWithToken(config, testToken) assert.NoError(t, err) newSecretID, err := client.GenerateNewAppRoleSecret(secretID, appRoleName) assert.NoError(t, err) @@ -194,7 +195,7 @@ func TestVaultIntegrationAppRole(t *testing.T) { assert.NotEqual(t, secretID, newSecretID) // verify metadata is not broken - client, err = vault.NewClientWithAppRole(config, roleID, newSecretID) + client, err = vault.NewClient(config) assert.NoError(t, err) secret, err := client.GetSecret("auth/token/lookup-self") meta := secret.Data["meta"].(SecretData) @@ -204,7 +205,7 @@ func TestVaultIntegrationAppRole(t *testing.T) { }) t.Run("Test Fetching RoleName from vault", func(t *testing.T) { - client, err := vault.NewClientWithAppRole(config, roleID, secretID) + client, err := vault.NewClient(config) assert.NoError(t, err) fetchedRoleName, err := client.GetAppRoleName() assert.NoError(t, err) @@ -238,16 +239,18 @@ func TestVaultIntegrationTokenRevocation(t *testing.T) { assert.NoError(t, err) port, err := vaultContainer.MappedPort(ctx, "8200") host := fmt.Sprintf("http://%s:%s", ip, port.Port()) - config := &vault.Config{Config: &api.Config{Address: host}} + config := &vault.ClientConfig{Config: &api.Config{Address: host}} secretIDMetadata := map[string]interface{}{ "field1": "value1", } roleID, secretID := setupVaultAppRole(t, config, testToken, appRolePath, secretIDMetadata) + config.RoleID = roleID + config.SecretID = secretID t.Run("Test Revocation works", func(t *testing.T) { - client, err := vault.NewClientWithAppRole(config, roleID, secretID) + client, err := vault.NewClient(config) assert.NoError(t, err) secret, err := client.GetSecret("auth/token/lookup-self") meta := secret.Data["meta"].(SecretData) @@ -263,7 +266,7 @@ func TestVaultIntegrationTokenRevocation(t *testing.T) { }) } -func setupVaultAppRole(t *testing.T, config *vault.Config, token, appRolePath string, metadata map[string]interface{}) (string, string) { +func setupVaultAppRole(t *testing.T, config *vault.ClientConfig, token, appRolePath string, metadata map[string]interface{}) (string, string) { t.Helper() client, err := api.NewClient(config.Config) assert.NoError(t, err) @@ -302,7 +305,7 @@ func setupVaultAppRole(t *testing.T, config *vault.Config, token, appRolePath st return roleID.(string), secretID.(string) } -func setupVault(t *testing.T, config *vault.Config, token string, secret SecretData) { +func setupVault(t *testing.T, config *vault.ClientConfig, token string, secret SecretData) { t.Helper() client, err := api.NewClient(config.Config) assert.NoError(t, err) diff --git a/pkg/config/vault.go b/pkg/config/vault.go index db53761039..c85173d351 100644 --- a/pkg/config/vault.go +++ b/pkg/config/vault.go @@ -87,6 +87,7 @@ var globalVaultClient *vault.Client func GlobalVaultClient() VaultClient { // an interface containing a nil pointer is considered non-nil in Go + // It is nil if Vault is not configured if globalVaultClient == nil { return nil } @@ -122,15 +123,17 @@ func GetVaultClientFromConfig(config map[string]interface{}, creds VaultCredenti namespace = config["vaultNamespace"].(string) log.Entry().Debugf(" with namespace %s", namespace) } - var client vault.Client + var client *vault.Client var err error - clientConfig := &vault.Config{Config: &api.Config{Address: address}, Namespace: namespace} + clientConfig := &vault.ClientConfig{Config: &api.Config{Address: address}, Namespace: namespace} if creds.VaultToken != "" { log.Entry().Debugf(" with Token authentication") - client, err = vault.NewClient(clientConfig, creds.VaultToken) + client, err = vault.NewClientWithToken(clientConfig, creds.VaultToken) } else { log.Entry().Debugf(" with AppRole authentication") - client, err = vault.NewClientWithAppRole(clientConfig, creds.AppRoleID, creds.AppRoleSecretID) + clientConfig.RoleID = creds.AppRoleID + clientConfig.SecretID = creds.AppRoleSecretID + client, err = vault.NewClient(clientConfig) } if err != nil { log.Entry().Info(" failed") @@ -138,7 +141,7 @@ func GetVaultClientFromConfig(config map[string]interface{}, creds VaultCredenti } // Set global vault client for usage in steps - globalVaultClient = &client + globalVaultClient = client log.Entry().Info(" succeeded") return client, nil diff --git a/pkg/vault/client.go b/pkg/vault/client.go index ba1e14e929..659d94c205 100644 --- a/pkg/vault/client.go +++ b/pkg/vault/client.go @@ -2,423 +2,195 @@ package vault import ( "context" - "encoding/json" "fmt" + "github.com/SAP/jenkins-library/pkg/log" + vaultAPI "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/api/auth/approle" + "github.com/pkg/errors" "io" "net/http" - "path" - "strconv" "strings" "time" - - "github.com/SAP/jenkins-library/pkg/log" - "github.com/hashicorp/vault/api" ) // Client handles communication with Vault type Client struct { - lClient logicalClient - config *Config + vaultApiClient *vaultAPI.Client + logical logicalClient + cfg *ClientConfig } -// Config contains the vault client configuration -type Config struct { - *api.Config - AppRoleMountPoint string +type ClientConfig struct { + *vaultAPI.Config Namespace string + AppRoleMountPoint string + RoleID string + SecretID string } // logicalClient interface for mocking type logicalClient interface { - Read(string) (*api.Secret, error) - Write(string, map[string]interface{}) (*api.Secret, error) + Read(string) (*vaultAPI.Secret, error) + Write(string, map[string]interface{}) (*vaultAPI.Secret, error) } -type VaultCredentials struct { - AppRoleID string - AppRoleSecretID string - VaultToken string -} - -// NewClient instantiates a Client and sets the specified token -func NewClient(config *Config, token string) (Client, error) { - if config == nil { - config = &Config{Config: api.DefaultConfig()} - } - client, err := api.NewClient(config.Config) - if err != nil { - return Client{}, err - } - if config.Namespace != "" { - client.SetNamespace(config.Namespace) +func newClient(cfg *ClientConfig) (*Client, error) { + if cfg == nil { + cfg = &ClientConfig{Config: vaultAPI.DefaultConfig()} } - client.SetToken(token) - return Client{client.Logical(), config}, nil -} -// NewClientWithAppRole instantiates a new client and obtains a token via the AppRole auth method -func NewClientWithAppRole(config *Config, roleID, secretID string) (Client, error) { - if config == nil { - config = &Config{Config: api.DefaultConfig()} - } - if config.AppRoleMountPoint == "" { - config.AppRoleMountPoint = "auth/approle" - } - client, err := api.NewClient(config.Config) + var err error + c := &Client{cfg: cfg} + c.vaultApiClient, err = vaultAPI.NewClient(cfg.Config) if err != nil { - return Client{}, err - } - - client.SetMinRetryWait(time.Second * 5) - client.SetMaxRetryWait(time.Second * 90) - client.SetMaxRetries(3) - client.SetCheckRetry(func(ctx context.Context, resp *http.Response, err error) (bool, error) { - if resp != nil { - log.Entry().Debugln("Vault response: ", resp.Status, resp.StatusCode, err) - } else { - log.Entry().Debugln("Vault response: ", err) - } - - isEOF := false - if err != nil && strings.Contains(err.Error(), "EOF") { - log.Entry().Infoln("isEOF is true") - isEOF = true - } - - if err == io.EOF { - log.Entry().Infoln("err = io.EOF is true") - } - - retry, err := api.DefaultRetryPolicy(ctx, resp, err) - - if err != nil || err == io.EOF || isEOF || retry { - log.Entry().Infoln("Retrying vault request...") - return true, nil - } - return false, nil - }) - - if config.Namespace != "" { - client.SetNamespace(config.Namespace) - } - - result, err := client.Logical().Write(path.Join(config.AppRoleMountPoint, "/login"), map[string]interface{}{ - "role_id": roleID, - "secret_id": secretID, - }) - if err != nil { - return Client{}, err + return nil, err } - - authInfo := result.Auth - if authInfo == nil || authInfo.ClientToken == "" { - return Client{}, fmt.Errorf("Could not obtain token from approle with role_id %s", roleID) + c.logical = c.vaultApiClient.Logical() + if cfg.Namespace != "" { + c.vaultApiClient.SetNamespace(cfg.Namespace) } - return NewClient(config, authInfo.ClientToken) + return c, nil } -// GetSecret uses the given path to fetch a secret from vault -func (v Client) GetSecret(path string) (*api.Secret, error) { - path = sanitizePath(path) - c := v.lClient - - secret, err := c.Read(path) +func NewClient(cfg *ClientConfig) (*Client, error) { + c, err := newClient(cfg) if err != nil { - return nil, err + return nil, errors.Wrap(err, "vault client initialization failed") } + applyApiClientRetryConfiguration(c.vaultApiClient) + + initialLoginDone := make(chan struct{}) + go c.startTokenLifecycleManager(initialLoginDone) // this goroutine ends with main goroutine + // wait for initial login or a failure + <-initialLoginDone - return secret, nil + // In case of a failure, the function returns an unauthorized client, which will cause subsequent requests to fail. + return c, nil } -// GetKvSecret reads secret from the KV engine. -// It Automatically transforms the logical path to the HTTP API Path for the corresponding KV Engine version -func (v Client) GetKvSecret(path string) (map[string]string, error) { - path = sanitizePath(path) - mountpath, version, err := v.getKvInfo(path) +func NewClientWithToken(cfg *ClientConfig, token string) (*Client, error) { + c, err := newClient(cfg) if err != nil { - return nil, err - } - if version == 2 { - path = addPrefixToKvPath(path, mountpath, "data") - } else if version != 1 { - return nil, fmt.Errorf("KV Engine in version %d is currently not supported", version) + return nil, errors.Wrap(err, "vault client initialization failed") } - secret, err := v.GetSecret(path) - if secret == nil || err != nil { - return nil, err + c.vaultApiClient.SetToken(token) + return c, nil +} - } - var rawData interface{} - switch version { - case 1: - rawData = secret.Data - case 2: - var ok bool - rawData, ok = secret.Data["data"] - if !ok { - return nil, fmt.Errorf("Missing 'data' field in response: %v", rawData) +func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { + defer func() { + // make sure to close channel to avoid blocking of the caller + _, open := <-initialLoginDone + if open { + close(initialLoginDone) } - } - - data, ok := rawData.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("Excpected 'data' field to be a map[string]interface{} but got %T instead", rawData) - } - - secretData := make(map[string]string, len(data)) - for k, v := range data { - switch t := v.(type) { - case string: - secretData[k] = t - case int: - secretData[k] = fmt.Sprintf("%d", t) - default: - jsonBytes, err := json.Marshal(t) - if err != nil { - log.Entry().Warnf("failed to parse Vault secret key %q, error: %s", k, err.Error()) - continue - } - - secretData[k] = string(jsonBytes) + }() + + initialLoginSucceed := false + for i := 0; i < c.vaultApiClient.MaxRetries(); i++ { + vaultLoginResp, err := c.login() + if err != nil { + log.Entry().Errorf("unable to authenticate to Vault: %v", err) + continue + } + if !initialLoginSucceed { + initialLoginDone <- struct{}{} + close(initialLoginDone) + initialLoginSucceed = true } - } - return secretData, nil -} -// WriteKvSecret writes secret to kv engine -func (v Client) WriteKvSecret(path string, newSecret map[string]string) error { - oldSecret, err := v.GetKvSecret(path) - if err != nil { - return err - } - secret := make(map[string]interface{}, len(oldSecret)) - for k, v := range oldSecret { - secret[k] = v - } - for k, v := range newSecret { - secret[k] = v - } - path = sanitizePath(path) - mountpath, version, err := v.getKvInfo(path) - if err != nil { - return err - } - if version == 2 { - path = addPrefixToKvPath(path, mountpath, "data") - secret = map[string]interface{}{"data": secret} - } else if version != 1 { - return fmt.Errorf("KV Engine in version %d is currently not supported", version) + tokenErr := c.manageTokenLifecycle(vaultLoginResp) + if tokenErr != nil { + log.Entry().Errorf("unable to start managing token lifecycle: %v", err) + continue + } } - - _, err = v.lClient.Write(path, secret) - return err } -// GenerateNewAppRoleSecret creates a new secret-id -func (v *Client) GenerateNewAppRoleSecret(secretID, appRoleName string) (string, error) { - appRolePath := v.getAppRolePath(appRoleName) - secretIDData, err := v.lookupSecretID(secretID, appRolePath) - if err != nil { - return "", err +// Starts token lifecycle management. Returns only fatal errors as errors, +// otherwise returns nil, so we can attempt login again. +func (c *Client) manageTokenLifecycle(authResp *vaultAPI.Secret) error { + if !authResp.Auth.Renewable { + log.Entry().Debugf("Token is not configured to be renewable. Re-attempting login.") + return nil } - reqPath := sanitizePath(path.Join(appRolePath, "/secret-id")) - - // we preserve metadata which was attached to the secret-id - json, err := json.Marshal(secretIDData["metadata"]) + watcher, err := c.vaultApiClient.NewLifetimeWatcher(&vaultAPI.LifetimeWatcherInput{Secret: authResp}) if err != nil { - return "", err + return fmt.Errorf("unable to initialize new lifetime watcher for renewing auth token: %w", err) } - secret, err := v.lClient.Write(reqPath, map[string]interface{}{ - "metadata": string(json), - }) - if err != nil { - return "", err - } - - if secret == nil || secret.Data == nil { - return "", fmt.Errorf("Could not generate new approle secret-id for approle path %s", reqPath) - } + go watcher.Start() + defer watcher.Stop() - secretIDRaw, ok := secret.Data["secret_id"] - if !ok { - return "", fmt.Errorf("Vault response for path %s did not contain a new secret-id", reqPath) - } + for { + select { + // `DoneCh` will return if renewal fails, or if the remaining lease + // duration is under a built-in threshold and either renewing is not + // extending it or renewing is disabled. In any case, the caller + // needs to attempt to log in again. + case err := <-watcher.DoneCh(): + if err != nil { + log.Entry().Printf("Failed to renew Vault token: %v. Re-attempting login.", err) + return nil + } + // This occurs once the token has reached max TTL. + log.Entry().Printf("Token can no longer be renewed. Re-attempting login.") + return nil - newSecretID, ok := secretIDRaw.(string) - if !ok { - return "", fmt.Errorf("New secret-id from approle path %s has an unexpected type %T expected 'string'", reqPath, secretIDRaw) + // Successfully completed renewal + case <-watcher.RenewCh(): + log.Entry().Printf("Vault token successfully renewed") + } } - - return newSecretID, nil } -// GetAppRoleSecretIDTtl returns the remaining time until the given secret-id expires -func (v *Client) GetAppRoleSecretIDTtl(secretID, roleName string) (time.Duration, error) { - appRolePath := v.getAppRolePath(roleName) - data, err := v.lookupSecretID(secretID, appRolePath) - if err != nil { - return 0, err - } - - if data == nil || data["expiration_time"] == nil { - return 0, fmt.Errorf("Could not load secret-id information from path %s", appRolePath) - } - - expiration, ok := data["expiration_time"].(string) - if !ok || expiration == "" { - return 0, fmt.Errorf("Could not handle get expiration time for secret-id from path %s", appRolePath) - } - - expirationDate, err := time.Parse(time.RFC3339, expiration) - +func (c *Client) login() (*vaultAPI.Secret, error) { + appRoleAuth, err := approle.NewAppRoleAuth(c.cfg.RoleID, &approle.SecretID{FromString: c.cfg.SecretID}) if err != nil { - return 0, err - } - - ttl := expirationDate.Sub(time.Now()) - if ttl < 0 { - return 0, nil - } - - return ttl, nil -} - -// RevokeToken revokes the token which is currently used. -// The client can't be used anymore after this function was called. -func (v Client) RevokeToken() error { - _, err := v.lClient.Write("auth/token/revoke-self", map[string]interface{}{}) - return err -} - -// MustRevokeToken same as RevokeToken but the programm is terminated with an error if this fails. -// Should be used in defer statements only. -func (v Client) MustRevokeToken() { - if err := v.RevokeToken(); err != nil { - log.Entry().WithError(err).Fatal("Could not revoke token") + return nil, fmt.Errorf("unable to initialize appRole auth method: %w", err) } -} -// GetAppRoleName returns the AppRole role name which was used to authenticate. -// Returns "" when AppRole authentication wasn't used -func (v *Client) GetAppRoleName() (string, error) { - const lookupPath = "auth/token/lookup-self" - secret, err := v.GetSecret(lookupPath) + authInfo, err := c.vaultApiClient.Auth().Login(context.Background(), appRoleAuth) if err != nil { - return "", err - } - - if secret.Data == nil { - return "", fmt.Errorf("Could not lookup token information: %s", lookupPath) - } - - meta, ok := secret.Data["meta"] - - if !ok { - return "", fmt.Errorf("Token info did not contain metadata %s", lookupPath) - } - - metaMap, ok := meta.(map[string]interface{}) - - if !ok { - return "", fmt.Errorf("Token info field 'meta' is not a map: %s", lookupPath) - } - - roleName := metaMap["role_name"] - - if roleName == nil { - return "", nil + return nil, fmt.Errorf("unable to login to appRole auth method: %w", err) } - - roleNameStr, ok := roleName.(string) - if !ok { - // when approle authentication is not used vault admins can use the role_name field with other type - // so no error in this case - return "", nil - } - - return roleNameStr, nil -} - -// SetAppRoleMountPoint sets the path under which the approle auth backend is mounted -func (v *Client) SetAppRoleMountPoint(appRoleMountpoint string) { - v.config.AppRoleMountPoint = appRoleMountpoint -} - -func (v *Client) getAppRolePath(roleName string) string { - appRoleMountPoint := v.config.AppRoleMountPoint - if appRoleMountPoint == "" { - appRoleMountPoint = "auth/approle" + if authInfo == nil { + return nil, fmt.Errorf("no auth info was returned after login") } - return path.Join(appRoleMountPoint, "role", roleName) -} -func sanitizePath(path string) string { - path = strings.TrimSpace(path) - path = strings.TrimPrefix(path, "/") - path = strings.TrimSuffix(path, "/") - return path + return authInfo, nil } -func addPrefixToKvPath(p, mountPath, apiPrefix string) string { - switch { - case p == mountPath, p == strings.TrimSuffix(mountPath, "/"): - return path.Join(mountPath, apiPrefix) - default: - p = strings.TrimPrefix(p, mountPath) - return path.Join(mountPath, apiPrefix, p) - } -} - -func (v *Client) getKvInfo(path string) (string, int, error) { - secret, err := v.GetSecret("sys/internal/ui/mounts/" + path) - if err != nil { - return "", 0, err - } - - if secret == nil { - return "", 0, fmt.Errorf("Failed to get version and engine mountpoint for path: %s", path) - } - - var mountPath string - if mountPathRaw, ok := secret.Data["path"]; ok { - mountPath = mountPathRaw.(string) - } - - options := secret.Data["options"] - if options == nil { - return mountPath, 1, nil - } - - versionRaw := options.(map[string]interface{})["version"] - if versionRaw == nil { - return mountPath, 1, nil - } +func applyApiClientRetryConfiguration(vaultApiClient *vaultAPI.Client) { + vaultApiClient.SetMinRetryWait(time.Second * 5) + vaultApiClient.SetMaxRetryWait(time.Second * 90) + vaultApiClient.SetMaxRetries(3) + vaultApiClient.SetCheckRetry(func(ctx context.Context, resp *http.Response, err error) (bool, error) { + if resp != nil { + log.Entry().Debugln("Vault response: ", resp.Status, resp.StatusCode, err) + } else { + log.Entry().Debugln("Vault response: ", err) + } - version := versionRaw.(string) - if version == "" { - return mountPath, 1, nil - } + isEOF := false + if err != nil && strings.Contains(err.Error(), "EOF") { + log.Entry().Infoln("isEOF is true") + isEOF = true + } - vNumber, err := strconv.Atoi(version) - if err != nil { - return mountPath, 0, err - } + if err == io.EOF { + log.Entry().Infoln("err = io.EOF is true") + } - return mountPath, vNumber, nil -} + retry, err := vaultAPI.DefaultRetryPolicy(ctx, resp, err) -func (v *Client) lookupSecretID(secretID, appRolePath string) (map[string]interface{}, error) { - reqPath := sanitizePath(path.Join(appRolePath, "/secret-id/lookup")) - secret, err := v.lClient.Write(reqPath, map[string]interface{}{ - "secret_id": secretID, + if err != nil || err == io.EOF || isEOF || retry { + log.Entry().Infoln("Retrying vault request...") + return true, nil + } + return false, nil }) - if err != nil { - return nil, err - } - - return secret.Data, nil } diff --git a/pkg/vault/helpers.go b/pkg/vault/helpers.go new file mode 100644 index 0000000000..1c9db72cbb --- /dev/null +++ b/pkg/vault/helpers.go @@ -0,0 +1,23 @@ +package vault + +import ( + "path" + "strings" +) + +func sanitizePath(path string) string { + path = strings.TrimSpace(path) + path = strings.TrimPrefix(path, "/") + path = strings.TrimSuffix(path, "/") + return path +} + +func addPrefixToKvPath(p, mountPath, apiPrefix string) string { + switch { + case p == mountPath, p == strings.TrimSuffix(mountPath, "/"): + return path.Join(mountPath, apiPrefix) + default: + p = strings.TrimPrefix(p, mountPath) + return path.Join(mountPath, apiPrefix, p) + } +} diff --git a/pkg/vault/oidc.go b/pkg/vault/oidc.go index 493050d04d..972084b59b 100644 --- a/pkg/vault/oidc.go +++ b/pkg/vault/oidc.go @@ -13,39 +13,47 @@ import ( "github.com/pkg/errors" ) -type JwtPayload struct { +type jwtPayload struct { Expire int64 `json:"exp"` } // getOIDCToken returns the generated OIDC token and sets it in the env -func (v Client) getOIDCToken(roleID string) (string, error) { +func (c *Client) getOIDCToken(roleID string) (string, error) { oidcPath := sanitizePath(path.Join("identity/oidc/token/", roleID)) - c := v.lClient - jwt, err := c.Read(oidcPath) + jwt, err := c.logical.Read(oidcPath) if err != nil { return "", err } token := jwt.Data["token"].(string) + if token == "" { + return "", fmt.Errorf("received an empty token") + } + log.RegisterSecret(token) os.Setenv("PIPER_OIDCIdentityToken", token) return token, nil } -// getJWTTokenPayload returns the payload of the JWT token using base64 decoding -func getJWTTokenPayload(token string) ([]byte, error) { +// getJWTPayload returns the payload of the JWT token using base64 decoding +func getJWTPayload(token string) (*jwtPayload, error) { parts := strings.Split(token, ".") - if len(parts) >= 2 { - substr := parts[1] - decodedBytes, err := base64.RawStdEncoding.DecodeString(substr) - if err != nil { - return nil, errors.Wrap(err, "JWT payload couldn't be decoded: %s") - } - return decodedBytes, nil + if len(parts) < 2 { + return nil, fmt.Errorf("not a valid JWT token") + } + + decodedBytes, err := base64.RawStdEncoding.DecodeString(parts[1]) + if err != nil { + return nil, errors.Wrap(err, "JWT payload couldn't be decoded: %s") + } + + var payload jwtPayload + if err = json.Unmarshal(decodedBytes, &payload); err != nil { + return nil, errors.Wrap(err, "JWT unmarshal failed") } - return nil, fmt.Errorf("Not a valid JWT token") + return &payload, nil } func oidcTokenIsValid(token string) bool { @@ -53,34 +61,28 @@ func oidcTokenIsValid(token string) bool { return false } - payload, err := getJWTTokenPayload(token) + jwtTokenPayload, err := getJWTPayload(token) if err != nil { log.Entry().Debugf("OIDC token couldn't be validated: %s", err) return false } - var jwtPayload JwtPayload - err = json.Unmarshal(payload, &jwtPayload) - if err != nil { - log.Entry().Debugf("OIDC token couldn't be validated: %s", err) - return false - } - - expiryTime := time.Unix(jwtPayload.Expire, 0) + expiryTime := time.Unix(jwtTokenPayload.Expire, 0) currentTime := time.Now() return expiryTime.After(currentTime) } // GetOIDCTokenByValidation returns the token if token is expired then get a new token else return old token -func (v Client) GetOIDCTokenByValidation(roleID string) (string, error) { +func (c *Client) GetOIDCTokenByValidation(roleID string) (string, error) { token := os.Getenv("PIPER_OIDCIdentityToken") if oidcTokenIsValid(token) { return token, nil } - token, err := v.getOIDCToken(roleID) - if token == "" || err != nil { + log.Entry().Debug("obtaining new OIDC token") + token, err := c.getOIDCToken(roleID) + if err != nil { return "", err } diff --git a/pkg/vault/oidc_test.go b/pkg/vault/oidc_test.go index 2e1a973305..7547a54ee8 100644 --- a/pkg/vault/oidc_test.go +++ b/pkg/vault/oidc_test.go @@ -28,7 +28,7 @@ func TestOIDC(t *testing.T) { // init vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", oidcPath).Return(mockJwt, nil) // run @@ -50,7 +50,7 @@ func TestOIDC(t *testing.T) { t.Setenv("PIPER_OIDCIdentityToken", token) vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", oidcPath).Return(mockJwt, nil) // run @@ -72,7 +72,7 @@ func TestOIDC(t *testing.T) { t.Setenv("PIPER_OIDCIdentityToken", token) vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", oidcPath).Return(mockJwt, nil) // run diff --git a/pkg/vault/vault.go b/pkg/vault/vault.go new file mode 100644 index 0000000000..5943b31c19 --- /dev/null +++ b/pkg/vault/vault.go @@ -0,0 +1,286 @@ +package vault + +import ( + "encoding/json" + "fmt" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/hashicorp/vault/api" + "path" + "strconv" + "time" +) + +// GetSecret uses the given path to fetch a secret from vault +func (c *Client) GetSecret(path string) (*api.Secret, error) { + return c.logical.Read(sanitizePath(path)) +} + +// GetKvSecret reads secret from the KV engine. +// It Automatically transforms the logical path to the HTTP API Path for the corresponding KV Engine version +func (c *Client) GetKvSecret(path string) (map[string]string, error) { + path = sanitizePath(path) + mountPath, version, err := c.getKvInfo(path) + if err != nil { + return nil, err + } + if version == 2 { + path = addPrefixToKvPath(path, mountPath, "data") + } else if version != 1 { + return nil, fmt.Errorf("KV Engine in version %d is currently not supported", version) + } + + secret, err := c.GetSecret(path) + if secret == nil || err != nil { + return nil, err + + } + var rawData interface{} + switch version { + case 1: + rawData = secret.Data + case 2: + var ok bool + rawData, ok = secret.Data["data"] + if !ok { + return nil, fmt.Errorf("missing 'data' field in response: %v", rawData) + } + } + + data, ok := rawData.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("excpected 'data' field to be a map[string]interface{} but got %T instead", rawData) + } + + secretData := make(map[string]string, len(data)) + for k, v := range data { + switch t := v.(type) { + case string: + secretData[k] = t + case int: + secretData[k] = fmt.Sprintf("%d", t) + default: + jsonBytes, err := json.Marshal(t) + if err != nil { + log.Entry().Warnf("failed to parse Vault secret key %q, error: %s", k, err.Error()) + continue + } + + secretData[k] = string(jsonBytes) + } + } + return secretData, nil +} + +// WriteKvSecret writes secret to kv engine +func (c *Client) WriteKvSecret(path string, newSecret map[string]string) error { + oldSecret, err := c.GetKvSecret(path) + if err != nil { + return err + } + secret := make(map[string]interface{}, len(oldSecret)) + for k, v := range oldSecret { + secret[k] = v + } + for k, v := range newSecret { + secret[k] = v + } + path = sanitizePath(path) + mountPath, version, err := c.getKvInfo(path) + if err != nil { + return err + } + if version == 2 { + path = addPrefixToKvPath(path, mountPath, "data") + secret = map[string]interface{}{"data": secret} + } else if version != 1 { + return fmt.Errorf("KV Engine in version %d is currently not supported", version) + } + + _, err = c.logical.Write(path, secret) + return err +} + +// GenerateNewAppRoleSecret creates a new secret-id +func (c *Client) GenerateNewAppRoleSecret(secretID, appRoleName string) (string, error) { + appRolePath := c.getAppRolePath(appRoleName) + secretIDData, err := c.lookupSecretID(secretID, appRolePath) + if err != nil { + return "", err + } + + reqPath := sanitizePath(path.Join(appRolePath, "/secret-id")) + + // we preserve metadata which was attached to the secret-id + jsonBytes, err := json.Marshal(secretIDData["metadata"]) + if err != nil { + return "", err + } + + secret, err := c.logical.Write(reqPath, map[string]interface{}{ + "metadata": string(jsonBytes), + }) + if err != nil { + return "", err + } + + if secret == nil || secret.Data == nil { + return "", fmt.Errorf("could not generate new approle secret-id for approle path %s", reqPath) + } + + secretIDRaw, ok := secret.Data["secret_id"] + if !ok { + return "", fmt.Errorf("Vault response for path %s did not contain a new secret-id", reqPath) + } + + newSecretID, ok := secretIDRaw.(string) + if !ok { + return "", fmt.Errorf("new secret-id from approle path %s has an unexpected type %T expected 'string'", reqPath, secretIDRaw) + } + + return newSecretID, nil +} + +// GetAppRoleSecretIDTtl returns the remaining time until the given secret-id expires +func (c *Client) GetAppRoleSecretIDTtl(secretID, roleName string) (time.Duration, error) { + appRolePath := c.getAppRolePath(roleName) + data, err := c.lookupSecretID(secretID, appRolePath) + if err != nil { + return 0, err + } + + if data == nil || data["expiration_time"] == nil { + return 0, fmt.Errorf("could not load secret-id information from path %s", appRolePath) + } + + expiration, ok := data["expiration_time"].(string) + if !ok || expiration == "" { + return 0, fmt.Errorf("could not handle get expiration time for secret-id from path %s", appRolePath) + } + + expirationDate, err := time.Parse(time.RFC3339, expiration) + + if err != nil { + return 0, err + } + + ttl := expirationDate.Sub(time.Now()) + if ttl < 0 { + return 0, nil + } + + return ttl, nil +} + +// RevokeToken revokes the token which is currently used. +// The client can't be used anymore after this function was called. +func (c *Client) RevokeToken() error { + _, err := c.logical.Write("auth/token/revoke-self", map[string]interface{}{}) + return err +} + +// MustRevokeToken same as RevokeToken but the program is terminated with an error if this fails. +// Should be used in defer statements only. +func (c *Client) MustRevokeToken() { + if err := c.RevokeToken(); err != nil { + log.Entry().WithError(err).Fatal("Could not revoke token") + } +} + +// GetAppRoleName returns the AppRole role name which was used to authenticate. +// Returns "" when AppRole authentication wasn't used +func (c *Client) GetAppRoleName() (string, error) { + const lookupPath = "auth/token/lookup-self" + secret, err := c.GetSecret(lookupPath) + if err != nil { + return "", err + } + + if secret.Data == nil { + return "", fmt.Errorf("could not lookup token information: %s", lookupPath) + } + + meta, ok := secret.Data["meta"] + + if !ok { + return "", fmt.Errorf("token info did not contain metadata %s", lookupPath) + } + + metaMap, ok := meta.(map[string]interface{}) + + if !ok { + return "", fmt.Errorf("token info field 'meta' is not a map: %s", lookupPath) + } + + roleName := metaMap["role_name"] + + if roleName == nil { + return "", nil + } + + roleNameStr, ok := roleName.(string) + if !ok { + // when AppRole authentication is not used vault admins can use the role_name field with other type + // so no error in this case + return "", nil + } + + return roleNameStr, nil +} + +func (c *Client) getAppRolePath(roleName string) string { + appRoleMountPoint := c.cfg.AppRoleMountPoint + if appRoleMountPoint == "" { + appRoleMountPoint = "auth/approle" + } + return path.Join(appRoleMountPoint, "role", roleName) +} + +func (c *Client) getKvInfo(path string) (string, int, error) { + secret, err := c.GetSecret("sys/internal/ui/mounts/" + path) + if err != nil { + return "", 0, err + } + + if secret == nil { + return "", 0, fmt.Errorf("failed to get version and engine mountpoint for path: %s", path) + } + + var mountPath string + if mountPathRaw, ok := secret.Data["path"]; ok { + mountPath = mountPathRaw.(string) + } + + options := secret.Data["options"] + if options == nil { + return mountPath, 1, nil + } + + versionRaw := options.(map[string]interface{})["version"] + if versionRaw == nil { + return mountPath, 1, nil + } + + version := versionRaw.(string) + if version == "" { + return mountPath, 1, nil + } + + vNumber, err := strconv.Atoi(version) + if err != nil { + return mountPath, 0, err + } + + return mountPath, vNumber, nil +} + +func (c *Client) lookupSecretID(secretID, appRolePath string) (map[string]interface{}, error) { + reqPath := sanitizePath(path.Join(appRolePath, "/secret-id/lookup")) + secret, err := c.logical.Write(reqPath, map[string]interface{}{ + "secret_id": secretID, + }) + if err != nil { + return nil, err + } + + return secret.Data, nil +} diff --git a/pkg/vault/client_test.go b/pkg/vault/vault_test.go similarity index 89% rename from pkg/vault/client_test.go rename to pkg/vault/vault_test.go index 90a86b35bb..a71bfe9ea0 100644 --- a/pkg/vault/client_test.go +++ b/pkg/vault/vault_test.go @@ -30,7 +30,7 @@ const ( func TestGetKV2Secret(t *testing.T) { t.Run("Test missing secret", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} setupMockKvV2(vaultMock) vaultMock.On("Read", "secret/data/notexist").Return(nil, nil) secret, err := client.GetKvSecret("secret/notexist") @@ -45,7 +45,7 @@ func TestGetKV2Secret(t *testing.T) { t.Run("Getting secret from KV engine (v2)", func(t *testing.T) { vaultMock := &mocks.VaultMock{} setupMockKvV2(vaultMock) - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", secretAPIPath).Return(kv2Secret(SecretData{"key1": "value1", "key2": map[string]any{"subkey1": "subvalue2"}, "key3": 3, "key4": []string{"foo", "bar"}}), nil) secret, err := client.GetKvSecret(secretName) assert.NoError(t, err, "Expect GetKvSecret to succeed") @@ -58,7 +58,7 @@ func TestGetKV2Secret(t *testing.T) { t.Run("error is thrown when data field is missing", func(t *testing.T) { vaultMock := &mocks.VaultMock{} setupMockKvV2(vaultMock) - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", secretAPIPath).Return(kv1Secret(SecretData{"key1": "value1"}), nil) secret, err := client.GetKvSecret(secretName) assert.Error(t, err, "Expected to fail since 'data' field is missing") @@ -74,7 +74,7 @@ func TestGetKV1Secret(t *testing.T) { t.Run("Test missing secret", func(t *testing.T) { vaultMock := &mocks.VaultMock{} setupMockKvV1(vaultMock) - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", mock.AnythingOfType("string")).Return(nil, nil) secret, err := client.GetKvSecret("secret/notexist") @@ -85,7 +85,7 @@ func TestGetKV1Secret(t *testing.T) { t.Run("Test parsing KV1 secrets", func(t *testing.T) { vaultMock := &mocks.VaultMock{} setupMockKvV1(vaultMock) - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", secretName).Return(kv1Secret(SecretData{"key1": "value1", "key2": 5}), nil) secret, err := client.GetKvSecret(secretName) @@ -103,7 +103,7 @@ func TestSecretIDGeneration(t *testing.T) { t.Run("Test generating new secret-id", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(5 * time.Hour).Format(time.RFC3339) metadata := map[string]interface{}{ @@ -128,7 +128,7 @@ func TestSecretIDGeneration(t *testing.T) { t.Run("Test with no secret-id returned", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(5 * time.Hour).Format(time.RFC3339) metadata := map[string]interface{}{ @@ -151,7 +151,7 @@ func TestSecretIDGeneration(t *testing.T) { t.Run("Test with no new secret-id returned", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(5 * time.Hour).Format(time.RFC3339) metadata := map[string]interface{}{ @@ -168,7 +168,7 @@ func TestSecretIDGeneration(t *testing.T) { vaultMock.On("Write", path.Join(appRolePath, "/secret-id"), mapWith("metadata", string(metadataJSON))).Return(kv1Secret(nil), nil) newSecretID, err := client.GenerateNewAppRoleSecret(secretID, appRoleName) - assert.EqualError(t, err, fmt.Sprintf("Could not generate new approle secret-id for approle path %s", path.Join(appRolePath, "secret-id"))) + assert.EqualError(t, err, fmt.Sprintf("could not generate new approle secret-id for approle path %s", path.Join(appRolePath, "secret-id"))) assert.Equal(t, newSecretID, "") }) } @@ -181,7 +181,7 @@ func TestSecretIDTtl(t *testing.T) { t.Run("Test fetching secreID TTL", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(5 * time.Hour).Format(time.RFC3339) vaultMock.On("Write", path.Join(appRolePath, "secret-id/lookup"), mapWith("secret_id", secretID)).Return(kv1Secret(SecretData{ @@ -195,16 +195,16 @@ func TestSecretIDTtl(t *testing.T) { t.Run("Test with no expiration time", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Write", path.Join(appRolePath, "secret-id/lookup"), mapWith("secret_id", secretID)).Return(kv1Secret(SecretData{}), nil) ttl, err := client.GetAppRoleSecretIDTtl(secretID, appRoleName) - assert.EqualError(t, err, fmt.Sprintf("Could not load secret-id information from path %s", appRolePath)) + assert.EqualError(t, err, fmt.Sprintf("could not load secret-id information from path %s", appRolePath)) assert.Equal(t, time.Duration(0), ttl) }) t.Run("Test with wrong date format", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Write", path.Join(appRolePath, "secret-id/lookup"), mapWith("secret_id", secretID)).Return(kv1Secret(SecretData{ "expiration_time": time.Now().String(), }), nil) @@ -215,7 +215,7 @@ func TestSecretIDTtl(t *testing.T) { t.Run("Test with expired secret-id", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(-5 * time.Hour).Format(time.RFC3339) vaultMock.On("Write", path.Join(appRolePath, "secret-id/lookup"), mapWith("secret_id", secretID)).Return(kv1Secret(SecretData{ @@ -234,7 +234,7 @@ func TestGetAppRoleName(t *testing.T) { t.Run("Test that correct role name is returned", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(SecretData{ "meta": SecretData{ "role_name": "test", @@ -248,27 +248,27 @@ func TestGetAppRoleName(t *testing.T) { t.Run("Test without secret data", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(nil), nil) appRoleName, err := client.GetAppRoleName() - assert.EqualError(t, err, "Could not lookup token information: auth/token/lookup-self") + assert.EqualError(t, err, "could not lookup token information: auth/token/lookup-self") assert.Empty(t, appRoleName) }) t.Run("Test without metadata data", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(SecretData{}), nil) appRoleName, err := client.GetAppRoleName() - assert.EqualError(t, err, "Token info did not contain metadata auth/token/lookup-self") + assert.EqualError(t, err, "token info did not contain metadata auth/token/lookup-self") assert.Empty(t, appRoleName) }) t.Run("Test without role name in metadata", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(SecretData{ "meta": SecretData{}, }), nil) @@ -280,7 +280,7 @@ func TestGetAppRoleName(t *testing.T) { t.Run("Test that different role_name types are ignored", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(SecretData{ "meta": SecretData{ "role_name": 5, @@ -297,7 +297,7 @@ func TestTokenRevocation(t *testing.T) { t.Parallel() t.Run("Test that revocation error is returned", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Write", "auth/token/revoke-self", mock.IsType(map[string]interface{}{})).Return(nil, errors.New("Test")) @@ -308,7 +308,7 @@ func TestTokenRevocation(t *testing.T) { t.Run("Test that revocation endpoint is called", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Write", "auth/token/revoke-self", mock.IsType(map[string]interface{}{})).Return(nil, nil) @@ -319,7 +319,7 @@ func TestTokenRevocation(t *testing.T) { func TestUnknownKvVersion(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "sys/internal/ui/mounts/secret/secret").Return(&api.Secret{ Data: map[string]interface{}{ @@ -335,15 +335,6 @@ func TestUnknownKvVersion(t *testing.T) { } -func TestSetAppRoleMountPont(t *testing.T) { - client := Client{nil, &Config{}} - const newMountpoint = "auth/test" - - client.SetAppRoleMountPoint("auth/test") - - assert.Equal(t, newMountpoint, client.config.AppRoleMountPoint) -} - func setupMockKvV2(vaultMock *mocks.VaultMock) { vaultMock.On("Read", mock.MatchedBy(func(path string) bool { return strings.HasPrefix(path, sysLookupPath) From 3ad2628095bf5e162807e428bc84a3601c8a300c Mon Sep 17 00:00:00 2001 From: Anil Keshav Date: Tue, 22 Oct 2024 10:50:32 +0200 Subject: [PATCH 08/28] feat(vault): not allowing batch token revoke (#4918) * not allowing batch token revoke * chaging values to hold variable name * error message when identifying service token * refactor --------- Co-authored-by: Googlom --- pkg/vault/vault.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pkg/vault/vault.go b/pkg/vault/vault.go index 5943b31c19..4d916230b3 100644 --- a/pkg/vault/vault.go +++ b/pkg/vault/vault.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/vault/api" "path" "strconv" + "strings" "time" ) @@ -181,7 +182,27 @@ func (c *Client) RevokeToken() error { // MustRevokeToken same as RevokeToken but the program is terminated with an error if this fails. // Should be used in defer statements only. func (c *Client) MustRevokeToken() { - if err := c.RevokeToken(); err != nil { + lookupPath := "auth/token/lookup-self" + const serviceTokenPrefix = "hvs." + + secret, err := c.GetSecret(lookupPath) + if err != nil { + log.Entry().Warnf("Could not lookup token at %s, not continuing to revoke: %v", lookupPath, err) + return + } + + tokenID, ok := secret.Data["id"].(string) + if !ok { + log.Entry().Warnf("Could not lookup token.Data.id at %s, not continuing to revoke", lookupPath) + return + } + + if !strings.HasPrefix(tokenID, serviceTokenPrefix) { + log.Entry().Warnf("Service token not identified at %s, not continuing to revoke", lookupPath) + return + } + + if err = c.RevokeToken(); err != nil { log.Entry().WithError(err).Fatal("Could not revoke token") } } From 7b08d47bcb3be7b1e95b3ffee0c1d81b61367703 Mon Sep 17 00:00:00 2001 From: Akramdzhon Azamov <900658008.akram@gmail.com> Date: Wed, 23 Oct 2024 14:09:15 +0500 Subject: [PATCH 09/28] Detect script version 9 (#5154) * detect script v9 as default and detect script v8 as optional for blackduck * unit test fix --------- Co-authored-by: Dmitrii Pavlukhin --- cmd/detectExecuteScan.go | 7 ++++--- cmd/detectExecuteScan_generated.go | 8 ++++---- cmd/detectExecuteScan_test.go | 2 +- resources/metadata/detectExecuteScan.yaml | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cmd/detectExecuteScan.go b/cmd/detectExecuteScan.go index e0f6a68444..5fc5561a10 100644 --- a/cmd/detectExecuteScan.go +++ b/cmd/detectExecuteScan.go @@ -427,10 +427,11 @@ func getDetectScript(config detectExecuteScanOptions, utils detectUtils) error { log.Entry().Infof("Downloading Detect Script") downloadScript := func() error { - if config.UseDetect9 { - return utils.DownloadFile("https://detect.synopsys.com/detect9.sh", "detect.sh", nil, nil) + if config.UseDetect8 { + return utils.DownloadFile("https://detect.synopsys.com/detect8.sh", "detect.sh", nil, nil) } - return utils.DownloadFile("https://detect.synopsys.com/detect8.sh", "detect.sh", nil, nil) + return utils.DownloadFile("https://detect.synopsys.com/detect9.sh", "detect.sh", nil, nil) + } if err := downloadScript(); err != nil { diff --git a/cmd/detectExecuteScan_generated.go b/cmd/detectExecuteScan_generated.go index 2da821f465..bd771a332c 100644 --- a/cmd/detectExecuteScan_generated.go +++ b/cmd/detectExecuteScan_generated.go @@ -77,7 +77,7 @@ type detectExecuteScanOptions struct { RegistryURL string `json:"registryUrl,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` RepositoryUsername string `json:"repositoryUsername,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` RepositoryPassword string `json:"repositoryPassword,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` - UseDetect9 bool `json:"useDetect9,omitempty"` + UseDetect8 bool `json:"useDetect8,omitempty"` } type detectExecuteScanInflux struct { @@ -354,7 +354,7 @@ func addDetectExecuteScanFlags(cmd *cobra.Command, stepConfig *detectExecuteScan cmd.Flags().StringVar(&stepConfig.RegistryURL, "registryUrl", os.Getenv("PIPER_registryUrl"), "Used accessing for the images to be scanned (typically filled by CPE)") cmd.Flags().StringVar(&stepConfig.RepositoryUsername, "repositoryUsername", os.Getenv("PIPER_repositoryUsername"), "Used accessing for the images to be scanned (typically filled by CPE)") cmd.Flags().StringVar(&stepConfig.RepositoryPassword, "repositoryPassword", os.Getenv("PIPER_repositoryPassword"), "Used accessing for the images to be scanned (typically filled by CPE)") - cmd.Flags().BoolVar(&stepConfig.UseDetect9, "useDetect9", false, "This flag enables the use of the supported version 9 of the Detect Script instead of v8") + cmd.Flags().BoolVar(&stepConfig.UseDetect8, "useDetect8", false, "This flag enables the use of the supported version 8 of the Detect Script instead of v9") cmd.MarkFlagRequired("token") cmd.MarkFlagRequired("projectName") @@ -947,12 +947,12 @@ func detectExecuteScanMetadata() config.StepData { Default: os.Getenv("PIPER_repositoryPassword"), }, { - Name: "useDetect9", + Name: "useDetect8", ResourceRef: []config.ResourceReference{}, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "bool", Mandatory: false, - Aliases: []config.Alias{{Name: "detect/useDetect9"}}, + Aliases: []config.Alias{{Name: "detect/useDetect8"}}, Default: false, }, }, diff --git a/cmd/detectExecuteScan_test.go b/cmd/detectExecuteScan_test.go index 7f02817729..8b4ac8d1f8 100644 --- a/cmd/detectExecuteScan_test.go +++ b/cmd/detectExecuteScan_test.go @@ -310,7 +310,7 @@ func TestRunDetect(t *testing.T) { utilsMock.AddFile("detect.sh", []byte("")) err := runDetect(ctx, detectExecuteScanOptions{}, utilsMock, &detectExecuteScanInflux{}) - assert.Equal(t, utilsMock.downloadedFiles["https://detect.synopsys.com/detect8.sh"], "detect.sh") + assert.Equal(t, utilsMock.downloadedFiles["https://detect.synopsys.com/detect9.sh"], "detect.sh") assert.True(t, utilsMock.HasRemovedFile("detect.sh")) assert.NoError(t, err) assert.Equal(t, ".", utilsMock.Dir, "Wrong execution directory used") diff --git a/resources/metadata/detectExecuteScan.yaml b/resources/metadata/detectExecuteScan.yaml index e098c32c40..138a1e5e19 100644 --- a/resources/metadata/detectExecuteScan.yaml +++ b/resources/metadata/detectExecuteScan.yaml @@ -653,11 +653,11 @@ spec: resourceRef: - name: commonPipelineEnvironment param: container/repositoryPassword - - name: useDetect9 + - name: useDetect8 description: - "This flag enables the use of the supported version 9 of the Detect Script instead of v8" + "This flag enables the use of the supported version 8 of the Detect Script instead of v9" aliases: - - name: detect/useDetect9 + - name: detect/useDetect8 type: bool scope: - PARAMETERS From 7b7ba7743607e1840f3e80057374b0f94b22a74c Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:32:37 +0500 Subject: [PATCH 10/28] modify logging (#5158) --- pkg/vault/client.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/vault/client.go b/pkg/vault/client.go index 659d94c205..4e1fc842ab 100644 --- a/pkg/vault/client.go +++ b/pkg/vault/client.go @@ -90,9 +90,13 @@ func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { initialLoginSucceed := false for i := 0; i < c.vaultApiClient.MaxRetries(); i++ { + if i != 0 { + log.Entry().Infof("Retrying Vault login. Attempt %d of %d", i, c.vaultApiClient.MaxRetries()) + } + vaultLoginResp, err := c.login() if err != nil { - log.Entry().Errorf("unable to authenticate to Vault: %v", err) + log.Entry().Warnf("unable to authenticate to Vault: %v", err) continue } if !initialLoginSucceed { @@ -103,7 +107,7 @@ func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { tokenErr := c.manageTokenLifecycle(vaultLoginResp) if tokenErr != nil { - log.Entry().Errorf("unable to start managing token lifecycle: %v", err) + log.Entry().Warnf("unable to start managing token lifecycle: %v", err) continue } } From 4990b2d0ba9670dc54ac7d08ce701da4221ff87d Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Thu, 24 Oct 2024 12:47:32 +0500 Subject: [PATCH 11/28] fix(sonar): allign groovy part of Sonar step with common piperExecuteBin (#5157) * add some logging to Vault login * allign groovy part of the sonar step * Revert "add some logging to Vault login" This reverts commit d1738c124d2c1fbfb5becaad2a28dafcef4574fc. --- vars/sonarExecuteScan.groovy | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/vars/sonarExecuteScan.groovy b/vars/sonarExecuteScan.groovy index 7265f4bbac..94d4c4fff6 100644 --- a/vars/sonarExecuteScan.groovy +++ b/vars/sonarExecuteScan.groovy @@ -65,17 +65,19 @@ void call(Map parameters = [:]) { writePipelineEnv(script: script, piperGoPath: piperGoPath) withEnv(environment) { influxWrapper(script) { - piperExecuteBin.credentialWrapper(config, credentialInfo) { - if (stepConfig.instance) { - withSonarQubeEnv(stepConfig.instance) { - echo "Instance is deprecated - please use serverUrl parameter to set URL to the Sonar backend." + try { + piperExecuteBin.credentialWrapper(config, credentialInfo) { + if (stepConfig.instance) { + withSonarQubeEnv(stepConfig.instance) { + echo "Instance is deprecated - please use serverUrl parameter to set URL to the Sonar backend." + sh "${piperGoPath} ${STEP_NAME}${customDefaultConfig}${customConfigArg}" + } + } else { sh "${piperGoPath} ${STEP_NAME}${customDefaultConfig}${customConfigArg}" - jenkinsUtils.handleStepResults(STEP_NAME, false, false) - readPipelineEnv(script: script, piperGoPath: piperGoPath) } - } else { - sh "${piperGoPath} ${STEP_NAME}${customDefaultConfig}${customConfigArg}" } + } finally { + jenkinsUtils.handleStepResults(STEP_NAME, false, false) } } } From 0c41f9c14156a263d9f94f538796515a50182038 Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:08:09 +0500 Subject: [PATCH 12/28] fix deadlock and add more logging (#5160) --- pkg/vault/client.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/vault/client.go b/pkg/vault/client.go index 4e1fc842ab..4d5875be9b 100644 --- a/pkg/vault/client.go +++ b/pkg/vault/client.go @@ -82,16 +82,18 @@ func NewClientWithToken(cfg *ClientConfig, token string) (*Client, error) { func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { defer func() { // make sure to close channel to avoid blocking of the caller - _, open := <-initialLoginDone - if open { - close(initialLoginDone) - } + log.Entry().Debugf("exiting Vault token lifecycle manager") + initialLoginDone <- struct{}{} + close(initialLoginDone) }() initialLoginSucceed := false - for i := 0; i < c.vaultApiClient.MaxRetries(); i++ { + retryAttemptDuration := c.vaultApiClient.MinRetryWait() + for i := 0; i <= c.vaultApiClient.MaxRetries(); i++ { if i != 0 { - log.Entry().Infof("Retrying Vault login. Attempt %d of %d", i, c.vaultApiClient.MaxRetries()) + log.Entry().Infof("Retrying Vault login in %.0f seconds. Attempt %d of %d", + retryAttemptDuration.Seconds(), i, c.vaultApiClient.MaxRetries()) + time.Sleep(retryAttemptDuration) } vaultLoginResp, err := c.login() @@ -101,7 +103,6 @@ func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { } if !initialLoginSucceed { initialLoginDone <- struct{}{} - close(initialLoginDone) initialLoginSucceed = true } From 8b4109bf85db2807f7eceb347df3e9c4a90e558d Mon Sep 17 00:00:00 2001 From: Holger Partsch Date: Thu, 24 Oct 2024 17:29:40 +0200 Subject: [PATCH 13/28] fix: command injection vulnerability (#5161) due to missing quoting, command injection was possible via pipeline configuration. This is now fixed using a quoting and escaping utility. Co-authored-by: Oliver Feldmann --- test/groovy/TestsPublishResultsTest.groovy | 17 +++++++++++++++++ vars/testsPublishResults.groovy | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/test/groovy/TestsPublishResultsTest.groovy b/test/groovy/TestsPublishResultsTest.groovy index f851577880..94dd1c21c5 100644 --- a/test/groovy/TestsPublishResultsTest.groovy +++ b/test/groovy/TestsPublishResultsTest.groovy @@ -1,3 +1,4 @@ +import com.sap.piper.BashUtils import org.junit.After import org.junit.Before import org.junit.Ignore @@ -9,8 +10,12 @@ import org.junit.rules.ExpectedException import util.BasePiperTest import util.JenkinsReadYamlRule import util.JenkinsStepRule + +import static org.hamcrest.Matchers.not import static org.junit.Assert.assertEquals +import static org.junit.Assert.assertThat import static org.junit.Assert.assertTrue +import static org.hamcrest.Matchers.containsString import com.sap.piper.Utils @@ -230,4 +235,16 @@ class TestsPublishResultsTest extends BasePiperTest { stepRule.step.testsPublishResults(script: nullScript, failOnError: true) } + + @Test + void testPublishUnitTestsWithUpdateResultsDoesNotAllowCommandExecution() throws Exception { + def injectString = "' -exec touch {} ; rm -rf / # –" + helper.registerAllowedMethod('sh', [String], { String cmd -> + assertThat(cmd, containsString(BashUtils.quoteAndEscape(injectString))) + }) + + stepRule.step.testsPublishResults(script: nullScript, junit: [pattern: injectString, archive: true, active: true, updateResults: true]) + + + } } diff --git a/vars/testsPublishResults.groovy b/vars/testsPublishResults.groovy index 00a0685e55..e86521726d 100644 --- a/vars/testsPublishResults.groovy +++ b/vars/testsPublishResults.groovy @@ -1,4 +1,5 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.sap.piper.GenerateDocumentation import com.sap.piper.ConfigurationHelper @@ -193,7 +194,7 @@ void touchFiles(pattern){ echo "[${STEP_NAME}] update test results" def patternArray = pattern.split(',') for(def i = 0; i < patternArray.length; i++){ - sh "find . -wholename '${patternArray[i].trim()}' -exec touch {} \\;" + sh "find . -wholename ${q(patternArray[i].trim())} -exec touch {} \\;" } } From f9dc47e47e278991725a10c46ff70049edc78f5b Mon Sep 17 00:00:00 2001 From: Srinikitha Kondreddy Date: Fri, 25 Oct 2024 09:53:55 +0200 Subject: [PATCH 14/28] Fix: validate app name (#5155) --- pkg/transportrequest/cts/upload.go | 4 ++-- pkg/transportrequest/cts/upload_test.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/transportrequest/cts/upload.go b/pkg/transportrequest/cts/upload.go index 6f959d7463..2cf141c378 100644 --- a/pkg/transportrequest/cts/upload.go +++ b/pkg/transportrequest/cts/upload.go @@ -62,7 +62,7 @@ const ( abapUserKey = "ABAP_USER" abapPasswordKey = "ABAP_PASSWORD" defaultConfigFileName = "ui5-deploy.yaml" - pattern = "^[a-zA-Z0-9_]+$" + pattern = "^(/[A-Za-z0-9_]{3,8}/)?[A-Za-z0-9_]+$" ) // WithConnection ... @@ -194,7 +194,7 @@ func getFioriDeployStatement( if len(app.Name) > 0 { re := regexp.MustCompile(pattern) if !re.MatchString(app.Name) { - return "", fmt.Errorf("application name '%s' contains spaces or special characters and is not according to the regex '%s'.", app.Name, pattern) + return "", fmt.Errorf("application name '%s' contains spaces or special characters or invalid namespace prefix and is not according to the regex '%s'.", app.Name, pattern) } log.Entry().Debugf("application name '%s' used from piper config", app.Name) cmd = append(cmd, "--name", app.Name) diff --git a/pkg/transportrequest/cts/upload_test.go b/pkg/transportrequest/cts/upload_test.go index 852d4908f2..f48b7da945 100644 --- a/pkg/transportrequest/cts/upload_test.go +++ b/pkg/transportrequest/cts/upload_test.go @@ -52,7 +52,7 @@ func TestUploadCTS(t *testing.T) { cmd := mock.ShellMockRunner{} action := UploadAction{ Connection: Connection{Endpoint: "https://example.org:8080/cts", Client: "001", User: "me", Password: "******"}, - Application: Application{Pack: "abapPackage", Name: "appName", Desc: "the Desc"}, + Application: Application{Pack: "abapPackage", Name: "/0ABCD/appName", Desc: "the Desc"}, Node: Node{ DeployDependencies: []string{}, InstallOpts: []string{}, @@ -66,7 +66,7 @@ func TestUploadCTS(t *testing.T) { if assert.NoError(t, err) { assert.Regexp( t, - "(?m)^fiori deploy --failfast --yes --username ABAP_USER --password ABAP_PASSWORD --description \"the Desc\" --noConfig --url https://example.org:8080/cts --client 001 --transport 12345678 --package abapPackage --name appName$", + "(?m)^fiori deploy --failfast --yes --username ABAP_USER --password ABAP_PASSWORD --description \"the Desc\" --noConfig --url https://example.org:8080/cts --client 001 --transport 12345678 --package abapPackage --name /0ABCD/appName", cmd.Calls[0], "Expected fiori deploy command not found", ) @@ -101,11 +101,11 @@ func TestUploadCTS(t *testing.T) { } }) - t.Run("fail in case of invalid app name", func(t *testing.T) { + t.Run("fail in case of invalid app name", func(t *testing.T) { cmd := mock.ShellMockRunner{} action := UploadAction{ Connection: Connection{Endpoint: "https://example.org:8080/cts", Client: "001", User: "me", Password: "******"}, - Application: Application{Pack: "abapPackage", Name: "app Name", Desc: "the Desc"}, + Application: Application{Pack: "abapPackage", Name: "/AB/app1", Desc: "the Desc"}, Node: Node{ DeployDependencies: []string{}, InstallOpts: []string{}, @@ -116,7 +116,7 @@ func TestUploadCTS(t *testing.T) { } err := action.Perform(&cmd) - expectedErrorMessge := "application name 'app Name' contains spaces or special characters and is not according to the regex '^[a-zA-Z0-9_]+$'." + expectedErrorMessge := "application name '/AB/app1' contains spaces or special characters or invalid namespace prefix and is not according to the regex '^(/[A-Za-z0-9_]{3,8}/)?[A-Za-z0-9_]+$'." assert.EqualErrorf(t, err, expectedErrorMessge, "invalid app name") }) From d6aaf43fae2621413e1cd01604c0d6c9325475ed Mon Sep 17 00:00:00 2001 From: Vyacheslav Starostin <32613074+vstarostin@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:07:28 +0500 Subject: [PATCH 15/28] Update version of org.cyclonedx:cyclonedx-maven-plugin (#5156) * Update version of org.cyclonedx:cyclonedx-maven-plugin * Update version of org.cyclonedx:cyclonedx-maven-plugin --------- Co-authored-by: Googlom <36107508+Googlom@users.noreply.github.com> --- cmd/mavenBuild.go | 6 +++--- cmd/mavenBuild_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index 559e5c2188..a3cb22d86d 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -66,7 +66,7 @@ func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { } defines = append(defines, createBOMConfig...) - goals := []string{"org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeBom"} + goals := []string{"org.cyclonedx:cyclonedx-maven-plugin:2.7.9:makeBom"} if config.Flatten { goals = append(goals, "flatten:flatten") @@ -88,7 +88,7 @@ func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { return err } -func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomData, utils maven.Utils, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) error { +func runMavenBuild(config *mavenBuildOptions, _ *telemetry.CustomData, utils maven.Utils, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) error { var flags = []string{"-update-snapshots", "--batch-mode"} @@ -111,7 +111,7 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat if config.CreateBOM { // Append the makeAggregateBOM goal to the rest of the goals - goals = append(goals, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") + goals = append(goals, "org.cyclonedx:cyclonedx-maven-plugin:2.7.9:makeAggregateBom") createBOMConfig := []string{ "-DschemaVersion=1.4", "-DincludeBomSerialNumber=true", diff --git a/cmd/mavenBuild_test.go b/cmd/mavenBuild_test.go index ea5e9abdbc..36647bb3ba 100644 --- a/cmd/mavenBuild_test.go +++ b/cmd/mavenBuild_test.go @@ -55,7 +55,7 @@ func TestMavenBuild(t *testing.T) { assert.Nil(t, err) if assert.Equal(t, 2, len(mockedUtils.Calls), "Expected two Maven invocations (default + makeAggregateBom)") { assert.Equal(t, "mvn", mockedUtils.Calls[1].Exec) - assert.Contains(t, mockedUtils.Calls[0].Params, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") + assert.Contains(t, mockedUtils.Calls[0].Params, "org.cyclonedx:cyclonedx-maven-plugin:2.7.9:makeAggregateBom") assert.Contains(t, mockedUtils.Calls[0].Params, "-DoutputName=bom-maven") } }) From 183004a80d2b240b642fa23bdd0c7631058d2a58 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 29 Oct 2024 08:39:34 +0100 Subject: [PATCH 16/28] Add identifier data to create uuid in events (#5165) --- cmd/gcpPublishEvent.go | 2 +- pkg/events/events.go | 16 ++++++++++++++-- pkg/events/events_test.go | 21 ++++++++++++++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/cmd/gcpPublishEvent.go b/cmd/gcpPublishEvent.go index 83baab8cf2..95a5493e93 100644 --- a/cmd/gcpPublishEvent.go +++ b/cmd/gcpPublishEvent.go @@ -68,7 +68,7 @@ func runGcpPublishEvent(utils gcpPublishEventUtils) error { } func createNewEvent(config *gcpPublishEventOptions) ([]byte, error) { - event, err := events.NewEvent(config.EventType, config.EventSource).CreateWithJSONData(config.EventData) + event, err := events.NewEvent(config.EventType, config.EventSource, "").CreateWithJSONData(config.EventData) if err != nil { return []byte{}, errors.Wrap(err, "failed to create new event") } diff --git a/pkg/events/events.go b/pkg/events/events.go index 700337e9f4..ed95f10268 100644 --- a/pkg/events/events.go +++ b/pkg/events/events.go @@ -21,12 +21,14 @@ type Event struct { cloudEvent cloudevents.Event eventType string eventSource string + uuidData string } -func NewEvent(eventType, eventSource string) Event { +func NewEvent(eventType, eventSource string, uuidString string) Event { return Event{ eventType: eventType, eventSource: eventSource, + uuidData: uuidString, } } @@ -45,8 +47,14 @@ func (e Event) CreateWithJSONData(data string, opts ...Option) (Event, error) { func (e Event) Create(data any, opts ...Option) Event { e.cloudEvent = cloudevents.NewEvent("1.0") + + if e.uuidData != "" { + e.cloudEvent.SetID(GetUUID(e.uuidData)) + } else { + e.cloudEvent.SetID(uuid.New().String()) + } + // set default values - e.cloudEvent.SetID(uuid.New().String()) e.cloudEvent.SetType(e.eventType) e.cloudEvent.SetTime(time.Now()) e.cloudEvent.SetSource(e.eventSource) @@ -58,6 +66,10 @@ func (e Event) Create(data any, opts ...Option) Event { return e } +func GetUUID(pipelineIdentifier string) string { + return uuid.NewMD5(uuid.NameSpaceOID, []byte(pipelineIdentifier)).String() +} + func (e Event) ToBytes() ([]byte, error) { data, err := json.Marshal(e.cloudEvent) if err != nil { diff --git a/pkg/events/events_test.go b/pkg/events/events_test.go index 87e4742ef8..4ab9a40bbe 100644 --- a/pkg/events/events_test.go +++ b/pkg/events/events_test.go @@ -11,7 +11,7 @@ func TestEventCreation(t *testing.T) { t.Run("success", func(t *testing.T) { // init // test - event := NewEvent(mock.Anything, mock.Anything).Create(nil) + event := NewEvent(mock.Anything, mock.Anything, "").Create(nil) // asserts assert.Equal(t, mock.Anything, event.cloudEvent.Type()) assert.Equal(t, mock.Anything, event.cloudEvent.Source()) @@ -21,7 +21,7 @@ func TestEventCreation(t *testing.T) { // init testData := `{"testKey":"testValue"}` // test - event, err := NewEvent(mock.Anything, mock.Anything).CreateWithJSONData(testData) + event, err := NewEvent(mock.Anything, mock.Anything, "").CreateWithJSONData(testData) // asserts assert.NoError(t, err) assert.Equal(t, string(event.cloudEvent.Data()), testData) @@ -32,10 +32,25 @@ func TestEventCreation(t *testing.T) { testData := `{"testKey": "testValue"}` additionalData := `{"additionalKey": "additionalValue"}` // test - event, err := NewEvent(mock.Anything, mock.Anything).CreateWithJSONData(testData) + event, err := NewEvent(mock.Anything, mock.Anything, "").CreateWithJSONData(testData) event.AddToCloudEventData(additionalData) // asserts assert.NoError(t, err) assert.Equal(t, string(event.cloudEvent.Data()), `{"additionalKey":"additionalValue","testKey":"testValue"}`) }) } + +func TestGetUUID(t *testing.T) { + pipelineIdentifier := "pipelineIdentifier" + uuid := GetUUID(pipelineIdentifier) + + if uuid == "" { + t.Fatalf("expected a UUID but got none") + } + + uuid2 := GetUUID(pipelineIdentifier) + if uuid != uuid2 { + t.Fatalf("expected the same UUID but got different ones") + } + +} From da609e1536e3ed1faf4bebd200f17ccb03681fdb Mon Sep 17 00:00:00 2001 From: Holger Partsch Date: Wed, 30 Oct 2024 11:59:47 +0100 Subject: [PATCH 17/28] Fix more potential command injection via quoting (#5164) * fix: make quoting null safe * fix: apply quoting in artifact set version * fix: add quoting to more shell step * refactor: use import alias * fix: further quoting --------- Co-authored-by: Oliver Feldmann --- src/com/sap/piper/BashUtils.groovy | 3 ++ .../piper/tools/neo/NeoCommandHelper.groovy | 28 ++++++++++--------- test/groovy/ArtifactSetVersionTest.groovy | 18 ++++++------ .../groovy/ContainerPushToRegistryTest.groovy | 8 +++--- test/groovy/HealthExecuteCheckTest.groovy | 4 +-- test/groovy/NeoDeployTest.groovy | 23 ++++++++------- vars/artifactSetVersion.groovy | 15 +++++----- vars/containerPushToRegistry.groovy | 3 +- vars/dockerExecute.groovy | 3 +- vars/healthExecuteCheck.groovy | 3 +- vars/mailSendNotification.groovy | 9 +++--- vars/neoDeploy.groovy | 7 +++-- 12 files changed, 67 insertions(+), 57 deletions(-) diff --git a/src/com/sap/piper/BashUtils.groovy b/src/com/sap/piper/BashUtils.groovy index 2387898e02..15817b89dc 100644 --- a/src/com/sap/piper/BashUtils.groovy +++ b/src/com/sap/piper/BashUtils.groovy @@ -8,6 +8,9 @@ class BashUtils implements Serializable { * Put string in single quotes and escape contained single quotes by putting them into a double quoted string */ static String quoteAndEscape(String str) { + if(str == null) { + return 'null' + } def escapedString = str.replace("'", ESCAPED_SINGLE_QUOTE) return "'${escapedString}'" } diff --git a/src/com/sap/piper/tools/neo/NeoCommandHelper.groovy b/src/com/sap/piper/tools/neo/NeoCommandHelper.groovy index d81a3e385e..56250344de 100644 --- a/src/com/sap/piper/tools/neo/NeoCommandHelper.groovy +++ b/src/com/sap/piper/tools/neo/NeoCommandHelper.groovy @@ -1,8 +1,10 @@ package com.sap.piper.tools.neo -import com.sap.piper.BashUtils +import static com.sap.piper.BashUtils.quoteAndEscape as q + import com.sap.piper.StepAssertions + class NeoCommandHelper { private Script step @@ -87,7 +89,7 @@ class NeoCommandHelper { private String source() { StepAssertions.assertFileExists(step, source) - return "--source ${BashUtils.quoteAndEscape(source)}" + return "--source ${q(source)}" } private String extensions() { @@ -96,19 +98,19 @@ class NeoCommandHelper { } private String mainArgs() { - String usernamePassword = "--user ${BashUtils.quoteAndEscape(user)} --password ${BashUtils.quoteAndEscape(password)}" + String usernamePassword = "--user ${q(user)} --password ${q(password)}" if (deployMode == DeployMode.WAR_PROPERTIES_FILE) { StepAssertions.assertFileIsConfiguredAndExists(step, deploymentConfiguration, 'propertiesFile') return "${deploymentConfiguration.propertiesFile} ${usernamePassword}" } - String targetArgs = "--host ${BashUtils.quoteAndEscape(deploymentConfiguration.host)}" - targetArgs += " --account ${BashUtils.quoteAndEscape(deploymentConfiguration.account)}" + String targetArgs = "--host ${q(deploymentConfiguration.host)}" + targetArgs += " --account ${q(deploymentConfiguration.account)}" if (deployMode == DeployMode.WAR_PARAMS) { - targetArgs += " --application ${BashUtils.quoteAndEscape(deploymentConfiguration.application)}" + targetArgs += " --application ${q(deploymentConfiguration.application)}" } return "${targetArgs} ${usernamePassword}" @@ -120,11 +122,11 @@ class NeoCommandHelper { } String args = "" - args += " --runtime ${BashUtils.quoteAndEscape(deploymentConfiguration.runtime)}" - args += " --runtime-version ${BashUtils.quoteAndEscape(deploymentConfiguration.runtimeVersion)}" + args += " --runtime ${q(deploymentConfiguration.runtime)}" + args += " --runtime-version ${q(deploymentConfiguration.runtimeVersion)}" if (deploymentConfiguration.size) { - args += " --size ${BashUtils.quoteAndEscape(deploymentConfiguration.size)}" + args += " --size ${q(deploymentConfiguration.size)}" } if (deploymentConfiguration.containsKey('environment')) { @@ -139,17 +141,17 @@ class NeoCommandHelper { for (int i = 0; i < keys.size(); i++) { def key = keys[i] def value = environment.get(keys[i]) - args += " --ev ${BashUtils.quoteAndEscape(key)}=${BashUtils.quoteAndEscape(value)}" + args += " --ev ${q(key)}=${q(value)}" } } if (deploymentConfiguration.containsKey('vmArguments')) { - args += " --vm-arguments ${BashUtils.quoteAndEscape(deploymentConfiguration.vmArguments)}" + args += " --vm-arguments ${q(deploymentConfiguration.vmArguments)}" } - + if (deploymentConfiguration.containsKey('azDistribution')) { - args += " --az-distribution ${BashUtils.quoteAndEscape(deploymentConfiguration.azDistribution)}" + args += " --az-distribution ${q(deploymentConfiguration.azDistribution)}" } return args diff --git a/test/groovy/ArtifactSetVersionTest.groovy b/test/groovy/ArtifactSetVersionTest.groovy index f9b145337c..edba0169f6 100644 --- a/test/groovy/ArtifactSetVersionTest.groovy +++ b/test/groovy/ArtifactSetVersionTest.groovy @@ -130,8 +130,8 @@ class ArtifactSetVersionTest extends BasePiperTest { assertThat(shellRule.shell.join(), stringContainsInOrder([ "git add .", "git commit -m 'update version 1.2.3-20180101010203_testCommitId'", - 'git tag build_1.2.3-20180101010203_testCommitId', - 'git push myGitSshUrl build_1.2.3-20180101010203_testCommitId', + "git tag 'build_1.2.3-20180101010203_testCommitId'", + "git push 'myGitSshUrl' 'build_1.2.3-20180101010203_testCommitId'", ] )) } @@ -173,8 +173,8 @@ class ArtifactSetVersionTest extends BasePiperTest { assertThat(((Iterable)shellRule.shell).join(), stringContainsInOrder([ "git add .", "git commit -m 'update version 1.2.3-20180101010203_testCommitId'", - 'git tag build_1.2.3-20180101010203_testCommitId', - 'git push https://me:topSecret@example.org/myGitRepo build_1.2.3-20180101010203_testCommitId', + "git tag 'build_1.2.3-20180101010203_testCommitId'", + "git push https://me:topSecret@example.org/myGitRepo 'build_1.2.3-20180101010203_testCommitId'", ] )) } @@ -246,8 +246,8 @@ class ArtifactSetVersionTest extends BasePiperTest { assertThat(((Iterable)shellRule.shell).join(), stringContainsInOrder([ "git add .", "git commit -m 'update version 1.2.3-20180101010203_testCommitId'", - 'git tag build_1.2.3-20180101010203_testCommitId', - '#!/bin/bash -e git push --quiet https://me:top%40Secret@example.org/myGitRepo build_1.2.3-20180101010203_testCommitId &>/dev/null', + "git tag 'build_1.2.3-20180101010203_testCommitId'", + "#!/bin/bash -e git push --quiet https://me:top%40Secret@example.org/myGitRepo 'build_1.2.3-20180101010203_testCommitId' &>/dev/null", ] )) } @@ -278,8 +278,8 @@ class ArtifactSetVersionTest extends BasePiperTest { assertThat(((Iterable)shellRule.shell).join(), stringContainsInOrder([ "git add .", "git commit -m 'update version 1.2.3-20180101010203_testCommitId'", - 'git tag build_1.2.3-20180101010203_testCommitId', - '#!/bin/bash -e git push --quiet https://me:top%40Secret@example.org/myGitRepo build_1.2.3-20180101010203_testCommitId &>/dev/null', + "git tag 'build_1.2.3-20180101010203_testCommitId'", + "#!/bin/bash -e git push --quiet https://me:top%40Secret@example.org/myGitRepo 'build_1.2.3-20180101010203_testCommitId' &>/dev/null", ] )) } @@ -301,7 +301,7 @@ class ArtifactSetVersionTest extends BasePiperTest { void testVersioningCustomGitUserAndEMail() { stepRule.step.artifactSetVersion(script: stepRule.step, juStabGitUtils: gitUtils, buildTool: 'maven', gitSshUrl: 'myGitSshUrl', gitUserEMail: 'test@test.com', gitUserName: 'test') - assertThat(shellRule.shell, hasItem(containsString("git -c user.email=\"test@test.com\" -c user.name=\"test\" commit -m 'update version 1.2.3-20180101010203_testCommitId'"))) + assertThat(shellRule.shell, hasItem(containsString("git -c user.email='test@test.com' -c user.name='test' commit -m 'update version 1.2.3-20180101010203_testCommitId'"))) } @Test diff --git a/test/groovy/ContainerPushToRegistryTest.groovy b/test/groovy/ContainerPushToRegistryTest.groovy index 474ac86990..65fe14f973 100644 --- a/test/groovy/ContainerPushToRegistryTest.groovy +++ b/test/groovy/ContainerPushToRegistryTest.groovy @@ -221,7 +221,7 @@ class ContainerPushToRegistryTest extends BasePiperTest { assertThat(dockerMock.targetRegistry.credentials, is('testCredentialsId')) assertThat(dockerMock.targetRegistry.isAnonymous, is(false)) assertThat(dockerMock.image, is('path/testImage:tag')) - assertThat(shellCallRule.shell, hasItem('docker tag testRegistry:55555/path/testImage:tag path/testImage:tag')) + assertThat(shellCallRule.shell, hasItem("docker tag 'testRegistry:55555'/'path/testImage:tag' 'path/testImage:tag'")) assertThat(dockerMockPull, is(true)) } @@ -238,7 +238,7 @@ class ContainerPushToRegistryTest extends BasePiperTest { assertThat(dockerMock.sourceRegistry.url, is('http://testSourceRegistry')) assertThat(dockerMock.image, is('testSourceName:testSourceTag')) assertThat(dockerMock.sourceRegistry.isAnonymous, is(true)) - assertThat(shellCallRule.shell, hasItem('docker tag testSourceRegistry/testSourceName:testSourceTag testSourceName:testSourceTag')) + assertThat(shellCallRule.shell, hasItem("docker tag 'testSourceRegistry'/'testSourceName:testSourceTag' 'testSourceName:testSourceTag'")) assertThat(dockerMockPull, is(true)) } @@ -256,7 +256,7 @@ class ContainerPushToRegistryTest extends BasePiperTest { assertThat(dockerMock.sourceRegistry.url, is('http://testSourceRegistry')) assertThat(dockerMock.sourceRegistry.isAnonymous, is(true)) assertThat(dockerMock.image, is('testSourceName:testSourceTag')) - assertThat(shellCallRule.shell, hasItem('docker tag testSourceRegistry/testSourceName:testSourceTag testImage:tag')) + assertThat(shellCallRule.shell, hasItem("docker tag 'testSourceRegistry'/'testSourceName:testSourceTag' 'testImage:tag'")) assertThat(dockerMockPull, is(true)) } @@ -277,7 +277,7 @@ class ContainerPushToRegistryTest extends BasePiperTest { assertThat(dockerMock.targetRegistry.url, is('https://testRegistry')) assertThat(dockerMock.targetRegistry.credentials, is('testCredentialsId')) assertThat(dockerMock.image, is('testSourceName:testSourceTag')) - assertThat(shellCallRule.shell, hasItem('docker tag testSourceRegistry/testSourceName:testSourceTag testImage:tag')) + assertThat(shellCallRule.shell, hasItem("docker tag 'testSourceRegistry'/'testSourceName:testSourceTag' 'testImage:tag'")) assertThat(dockerMockPull, is(true)) } diff --git a/test/groovy/HealthExecuteCheckTest.groovy b/test/groovy/HealthExecuteCheckTest.groovy index 0c0762012d..5cb4c1bd2c 100644 --- a/test/groovy/HealthExecuteCheckTest.groovy +++ b/test/groovy/HealthExecuteCheckTest.groovy @@ -32,8 +32,8 @@ class HealthExecuteCheckTest extends BasePiperTest { @Before void init() throws Exception { // register Jenkins commands with mock values - def command1 = "curl -so /dev/null -w '%{response_code}' http://testserver" - def command2 = "curl -so /dev/null -w '%{response_code}' http://testserver/endpoint" + def command1 = "curl -so /dev/null -w '%{response_code}' 'http://testserver'" + def command2 = "curl -so /dev/null -w '%{response_code}' 'http://testserver/endpoint'" helper.registerAllowedMethod('sh', [Map.class], {map -> return map.script == command1 || map.script == command2 ? "200" : "404" }) diff --git a/test/groovy/NeoDeployTest.groovy b/test/groovy/NeoDeployTest.groovy index 72993b6de9..d4f6d15123 100644 --- a/test/groovy/NeoDeployTest.groovy +++ b/test/groovy/NeoDeployTest.groovy @@ -298,25 +298,24 @@ class NeoDeployTest extends BasePiperTest { nullScript.commonPipelineEnvironment.setMtarFilePath('archive.mtar') - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "https:\\/\\/api\\.test\\.com\\/oauth2\\/apitoken\\/v1", "{\"access_token\":\"xxx\"}") - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "https:\\/\\/slservice\\.test\\.host\\.com\\/slservice\\/v1\\/oauth\\/accounts\\/testUser123\\/mtars", "{\"id\":123}") - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "https:\\/\\/slservice\\.test\\.host\\.com\\/slservice\\/v1\\/oauth\\/accounts\\/testUser123\\/mtars", "{\"state\":\"DONE\"}") - + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "-XPOST.*/apitoken", "{\"access_token\":\"xxx\"}") + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "-XPOST.*https://slservice", "{\"id\":123}") + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "-XGET.*https://slservice", "{\"state\":\"DONE\"}") stepRule.step.neoDeploy( script: nullScript, source: archiveName, deployMode: 'mta', - neo: [ - host: 'test.host.com', - account: 'testUser123', - credentialsId: 'OauthDataFileId', - credentialType: 'SecretFile' - ], + neo: [ + host : 'test.host.com', + account : 'testUser123', + credentialsId : 'OauthDataFileId', + credentialType: 'SecretFile' + ], ) - Assert.assertThat(shellRule.shell[0], containsString("#!/bin/bash curl --fail --silent --show-error --retry 12 -XPOST -u \"abc123:testclientsecret123\" \"https://api.test.com/oauth2/apitoken/v1?grant_type=client_credentials\"")) - Assert.assertThat(shellRule.shell[1], containsString("#!/bin/bash curl --fail --silent --show-error --retry 12 -XPOST -H \"Authorization: Bearer xxx\" -F file=@\"archive.mtar\" \"https://slservice.test.host.com/slservice/v1/oauth/accounts/testUser123/mtars\"")) + Assert.assertThat(shellRule.shell[0], containsString("#!/bin/bash curl --fail --silent --show-error --retry 12 -XPOST -u 'abc123':'testclientsecret123' 'https://api.test.com/oauth2'/apitoken/v1?grant_type=client_credentials")) + Assert.assertThat(shellRule.shell[1], containsString("#!/bin/bash curl --fail --silent --show-error --retry 12 -XPOST -H \"Authorization: Bearer xxx\" -F file=@'archive.mtar' https://slservice.'test.host.com'/slservice/v1/oauth/accounts/'testUser123'/mtars")) } @Test diff --git a/vars/artifactSetVersion.groovy b/vars/artifactSetVersion.groovy index e46ea77845..2faf9390a2 100644 --- a/vars/artifactSetVersion.groovy +++ b/vars/artifactSetVersion.groovy @@ -1,4 +1,5 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.sap.piper.GenerateDocumentation import com.sap.piper.ConfigurationHelper @@ -175,7 +176,7 @@ void call(Map parameters = [:], Closure body = null) { def gitConfig = [] if(config.gitUserEMail) { - gitConfig.add("-c user.email=\"${config.gitUserEMail}\"") + gitConfig.add("-c user.email=${q(config.gitUserEMail)}") } else { // in case there is no user.email configured on project level we might still // be able to work in case there is a configuration available on plain git level. @@ -184,7 +185,7 @@ void call(Map parameters = [:], Closure body = null) { } } if(config.gitUserName) { - gitConfig.add("-c user.name=\"${config.gitUserName}\"") + gitConfig.add("-c user.name=${q(config.gitUserName)}") } else { // in case there is no user.name configured on project level we might still // be able to work in case there is a configuration available on plain git level. @@ -199,7 +200,7 @@ void call(Map parameters = [:], Closure body = null) { set -e git add . --update git ${gitConfig} commit -m 'update version ${newVersion}' - git tag ${config.tagPrefix}${newVersion}""" + git tag ${q(config.tagPrefix+newVersion)}""" config.gitCommitId = gitUtils.getGitCommitIdOrNull() } catch (e) { error "[${STEP_NAME}]git commit and tag failed: ${e}" @@ -215,7 +216,7 @@ void call(Map parameters = [:], Closure body = null) { .use() sshagent([config.gitSshKeyCredentialsId]) { - sh "git push ${config.gitSshUrl} ${config.tagPrefix}${newVersion}" + sh "git push ${q(config.gitSshUrl)} ${q(config.tagPrefix+newVersion)}" } } else if(gitPushMode == GitPushMode.HTTPS) { @@ -259,7 +260,7 @@ void call(Map parameters = [:], Closure body = null) { gitConfig = [] if(config.gitHttpProxy) { - gitConfig.add("-c http.proxy=\"${config.gitHttpProxy}\"") + gitConfig.add("-c http.proxy=${q(config.gitHttpProxy)}") } if(config.gitDisableSslVerification) { @@ -288,7 +289,7 @@ void call(Map parameters = [:], Closure body = null) { gitPushFlags = gitPushFlags.join(' ') sh script: """|#!/bin/bash ${hashbangFlags} - |${gitDebug}git ${gitConfig} push ${gitPushFlags} ${gitUrlWithCredentials} ${config.tagPrefix}${newVersion} ${streamhandling}""".stripMargin() + |${gitDebug}git ${gitConfig} push ${gitPushFlags} ${gitUrlWithCredentials} ${q(config.tagPrefix+newVersion)} ${streamhandling}""".stripMargin() } } else { echo "Git push mode: ${gitPushMode.toString()}. Git push to remote has been skipped." @@ -313,5 +314,5 @@ def isAppContainer(config){ } def getTimestamp(pattern){ - return sh(returnStdout: true, script: "date --utc +'${pattern}'").trim() + return sh(returnStdout: true, script: "date --utc +${q(pattern)}").trim() } diff --git a/vars/containerPushToRegistry.groovy b/vars/containerPushToRegistry.groovy index ca29e6694e..3707eafd8c 100644 --- a/vars/containerPushToRegistry.groovy +++ b/vars/containerPushToRegistry.groovy @@ -5,6 +5,7 @@ import com.sap.piper.DockerUtils import groovy.transform.Field import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q @Field String STEP_NAME = getClass().getName() @Field Set GENERAL_CONFIG_KEYS = [ @@ -101,7 +102,7 @@ void call(Map parameters = [:]) { ) { sourceBuildImage.pull() } - sh "docker tag ${config.sourceRegistry}/${config.sourceImage} ${config.dockerImage}" + sh "docker tag ${q(config.sourceRegistry)}/${q(config.sourceImage)} ${q(config.dockerImage)}" } docker.withRegistry( diff --git a/vars/dockerExecute.groovy b/vars/dockerExecute.groovy index cb9034a711..9ce9901cc7 100644 --- a/vars/dockerExecute.groovy +++ b/vars/dockerExecute.groovy @@ -1,6 +1,7 @@ import com.sap.piper.SidecarUtils import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.cloudbees.groovy.cps.NonCPS import com.sap.piper.ConfigurationHelper @@ -242,7 +243,7 @@ void call(Map parameters = [:], body) { } } else { def networkName = "sidecar-${UUID.randomUUID()}" - sh "docker network create ${networkName}" + sh "docker network create ${q(networkName)}" try { def sidecarImage = docker.image(config.sidecarImage) pullWrapper(config.sidecarPullImage, sidecarImage, config.sidecarRegistryUrl, config.sidecarRegistryCredentialsId) { diff --git a/vars/healthExecuteCheck.groovy b/vars/healthExecuteCheck.groovy index 05b2997c0f..2992959475 100644 --- a/vars/healthExecuteCheck.groovy +++ b/vars/healthExecuteCheck.groovy @@ -1,4 +1,5 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.sap.piper.GenerateDocumentation import com.sap.piper.ConfigurationHelper @@ -70,6 +71,6 @@ void call(Map parameters = [:]) { def curl(url){ return sh( returnStdout: true, - script: "curl -so /dev/null -w '%{response_code}' ${url}" + script: "curl -so /dev/null -w '%{response_code}' ${q(url)}" ).trim() } diff --git a/vars/mailSendNotification.groovy b/vars/mailSendNotification.groovy index df39091662..b53425853d 100644 --- a/vars/mailSendNotification.groovy +++ b/vars/mailSendNotification.groovy @@ -1,4 +1,5 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.sap.piper.ConfigurationHelper import com.sap.piper.GenerateDocumentation @@ -198,8 +199,8 @@ def getCulprits(config, branch, numberOfCommits) { def pullRequestID = branch.replaceAll('PR-', '') def localBranchName = "pr" + pullRequestID sh """git init - git fetch ${config.gitUrl} pull/${pullRequestID}/head:${localBranchName} > /dev/null 2>&1 - git checkout -f ${localBranchName} > /dev/null 2>&1 + git fetch ${q(config.gitUrl)} pull/${q(pullRequestID)}/head:${q(localBranchName)} > /dev/null 2>&1 + git checkout -f ${q(localBranchName)} > /dev/null 2>&1 """ } } else { @@ -210,8 +211,8 @@ def getCulprits(config, branch, numberOfCommits) { credentials: [config.gitSshKeyCredentialsId], ignoreMissing: true ) { - sh """git clone ${config.gitUrl} . - git checkout ${config.gitCommitId} > /dev/null 2>&1""" + sh """git clone ${q(config.gitUrl)} . + git checkout ${q(config.gitCommitId)} > /dev/null 2>&1""" } } else { def retCode = sh(returnStatus: true, script: 'git log > /dev/null 2>&1') diff --git a/vars/neoDeploy.groovy b/vars/neoDeploy.groovy index b83b28f8b8..baaeb79111 100644 --- a/vars/neoDeploy.groovy +++ b/vars/neoDeploy.groovy @@ -10,6 +10,7 @@ import com.sap.piper.tools.neo.WarAction import groovy.transform.Field import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q @Field String STEP_NAME = getClass().getName() @@ -413,7 +414,7 @@ private deployWithBearerToken(def credentialFilePath, Map configuration, Script def myCurl = "curl --fail --silent --show-error --retry 12" def token_json = sh( script: """#!/bin/bash - ${myCurl} -XPOST -u \"${oauthClientId}:${oauthClientSecret}\" \"${oauthUrl}/apitoken/v1?grant_type=client_credentials" + ${myCurl} -XPOST -u ${q(oauthClientId)}:${q(oauthClientSecret)} ${q(oauthUrl)}/apitoken/v1?grant_type=client_credentials """, returnStdout: true ) @@ -424,7 +425,7 @@ private deployWithBearerToken(def credentialFilePath, Map configuration, Script def deploymentContentResponse = sh( script: """#!/bin/bash - ${myCurl} -XPOST -H \"Authorization: Bearer ${token}\" -F file=@\"${deployArchive}\" \"https://slservice.${host}/slservice/v1/oauth/accounts/${account}/mtars\" + ${myCurl} -XPOST -H "Authorization: Bearer ${token}" -F file=@${q(deployArchive)} https://slservice.${q(host)}/slservice/v1/oauth/accounts/${q(account)}/mtars """, returnStdout: true ) @@ -434,7 +435,7 @@ private deployWithBearerToken(def credentialFilePath, Map configuration, Script echo "[${STEP_NAME}] Deployment Id is '${deploymentId}'." def statusPollScript = """#!/bin/bash - ${myCurl} -XGET -H \"Authorization: Bearer ${token}\" \"https://slservice.${host}/slservice/v1/oauth/accounts/${account}/mtars/${deploymentId}\" + ${myCurl} -XGET -H "Authorization: Bearer ${token}" https://slservice.${q(host)}/slservice/v1/oauth/accounts/${q(account)}/mtars/${deploymentId} """ def statusResponse = sh(script: statusPollScript, returnStdout: true) def statusJson = readJSON text: statusResponse From 6988f43f7f244d69a0d971e7dd1e7b10d7098dc9 Mon Sep 17 00:00:00 2001 From: phgermanov Date: Mon, 4 Nov 2024 12:30:39 +0200 Subject: [PATCH 18/28] feat: add build artifacts metadata for mtaBuild (#5166) --- cmd/mavenBuild.go | 29 +--- cmd/mavenBuild_test.go | 54 ------- cmd/mtaBuild.go | 194 +++++++++++++---------- cmd/mtaBuild_generated.go | 14 ++ cmd/mtaBuild_test.go | 235 +++++++++++++++++++++------- pkg/npm/publish.go | 18 +-- pkg/npm/publish_test.go | 49 +----- pkg/piperutils/cyclonedxBom.go | 12 +- pkg/piperutils/cyclonedxbom_test.go | 81 ++++++++-- resources/metadata/mtaBuild.yaml | 9 ++ 10 files changed, 395 insertions(+), 300 deletions(-) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index a3cb22d86d..5f8a4b0a96 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -41,7 +41,7 @@ func mavenBuild(config mavenBuildOptions, telemetryData *telemetry.CustomData, c } func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { - var flags = []string{"-update-snapshots", "--batch-mode"} + flags := []string{"-update-snapshots", "--batch-mode"} if len(config.Profiles) > 0 { flags = append(flags, "--activate-profiles", strings.Join(config.Profiles, ",")) } @@ -89,8 +89,7 @@ func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { } func runMavenBuild(config *mavenBuildOptions, _ *telemetry.CustomData, utils maven.Utils, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) error { - - var flags = []string{"-update-snapshots", "--batch-mode"} + flags := []string{"-update-snapshots", "--batch-mode"} if len(config.Profiles) > 0 { flags = append(flags, "--activate-profiles", strings.Join(config.Profiles, ",")) @@ -255,7 +254,7 @@ func createBuildArtifactsMetadata(config *mavenBuildOptions, commonPipelineEnvir } else { coordinate.BuildPath = filepath.Dir(match) coordinate.URL = config.AltDeploymentRepositoryURL - coordinate.PURL = getPurlForThePom(match) + coordinate.PURL = piperutils.GetPurl(filepath.Join(filepath.Dir(match), "/target/"+mvnSimpleBomFilename+".xml")) buildCoordinates = append(buildCoordinates, coordinate) } } @@ -274,25 +273,6 @@ func createBuildArtifactsMetadata(config *mavenBuildOptions, commonPipelineEnvir return nil, false } -func getPurlForThePom(pomFilePath string) string { - bomPath := filepath.Join(filepath.Dir(pomFilePath) + "/target/" + mvnSimpleBomFilename + ".xml") - exists, _ := piperutils.FileExists(bomPath) - if !exists { - log.Entry().Debugf("bom file doesn't exist and hence no pURL info: %v", bomPath) - return "" - } - bom, err := piperutils.GetBom(bomPath) - if err != nil { - log.Entry().Warnf("failed to get bom file %s: %v", bomPath, err) - return "" - } - - log.Entry().Debugf("Found purl: %s for the bomPath: %s", bom.Metadata.Component.Purl, bomPath) - purl := bom.Metadata.Component.Purl - - return purl -} - func createOrUpdateProjectSettingsXML(projectSettingsFile string, altDeploymentRepositoryID string, altDeploymentRepositoryUser string, altDeploymentRepositoryPassword string, utils maven.Utils) (string, error) { if len(projectSettingsFile) > 0 { projectSettingsFilePath, err := maven.UpdateProjectSettingsXML(projectSettingsFile, altDeploymentRepositoryID, altDeploymentRepositoryUser, altDeploymentRepositoryPassword, utils) @@ -310,7 +290,7 @@ func createOrUpdateProjectSettingsXML(projectSettingsFile string, altDeploymentR } func loadRemoteRepoCertificates(certificateList []string, client piperhttp.Downloader, flags *[]string, runner command.ExecRunner, fileUtils piperutils.FileUtils, javaCaCertFilePath string) error { - //TODO: make use of java/keytool package + // TODO: make use of java/keytool package existingJavaCaCerts := filepath.Join(os.Getenv("JAVA_HOME"), "jre", "lib", "security", "cacerts") if len(javaCaCertFilePath) > 0 { @@ -318,7 +298,6 @@ func loadRemoteRepoCertificates(certificateList []string, client piperhttp.Downl } exists, err := fileUtils.FileExists(existingJavaCaCerts) - if err != nil { return errors.Wrap(err, "Could not find the existing java cacerts") } diff --git a/cmd/mavenBuild_test.go b/cmd/mavenBuild_test.go index 36647bb3ba..eddd3097b3 100644 --- a/cmd/mavenBuild_test.go +++ b/cmd/mavenBuild_test.go @@ -4,18 +4,14 @@ package cmd import ( - "os" - "path/filepath" "testing" - "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/stretchr/testify/assert" ) var cpe mavenBuildCommonPipelineEnvironment func TestMavenBuild(t *testing.T) { - t.Run("mavenBuild should install the artifact", func(t *testing.T) { mockedUtils := newMavenMockUtils() @@ -123,54 +119,4 @@ func TestMavenBuild(t *testing.T) { assert.Equal(t, mockedUtils.Calls[0].Exec, "mvn") assert.Empty(t, cpe.custom.mavenBuildArtifacts) }) - -} - -func createTempFile(t *testing.T, dir string, filename string, content string) string { - filePath := filepath.Join(dir, filename) - err := os.WriteFile(filePath, []byte(content), 0666) - if err != nil { - t.Fatalf("Failed to create temp file: %s", err) - } - return filePath -} - -func TestGetPurlForThePomAndDeleteIndividualBom(t *testing.T) { - - t.Run("valid BOM file, aggregated BOM", func(t *testing.T) { - tempDir, err := piperutils.Files{}.TempDir("", "test") - if err != nil { - t.Fatalf("Failed to create temp directory: %s", err) - } - - bomContent := ` - - - pkg:maven/com.example/aggregatecomponent@1.0.0 - - - - - - ` - pomFilePath := createTempFile(t, tempDir, "pom.xml", "") - bomDir := filepath.Join(tempDir, "target") - if err := os.MkdirAll(bomDir, 0777); err != nil { - t.Fatalf("Failed to create temp directory: %s", err) - } - bomFilePath := createTempFile(t, bomDir, mvnSimpleBomFilename+".xml", bomContent) - - purl := getPurlForThePom(pomFilePath) - assert.Equal(t, "pkg:maven/com.example/aggregatecomponent@1.0.0", purl) - _, err = os.Stat(bomFilePath) - assert.False(t, os.IsNotExist(err)) // File should not be deleted - }) - - t.Run("BOM file does not exist", func(t *testing.T) { - tempDir := t.TempDir() - pomFilePath := createTempFile(t, tempDir, "pom.xml", "") // Create a temp pom file - - purl := getPurlForThePom(pomFilePath) - assert.Equal(t, "", purl) - }) } diff --git a/cmd/mtaBuild.go b/cmd/mtaBuild.go index 4f1ef4c64c..0806d9ad61 100644 --- a/cmd/mtaBuild.go +++ b/cmd/mtaBuild.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io" "net/http" "os" "path" @@ -14,8 +15,10 @@ import ( "text/template" "time" + "github.com/SAP/jenkins-library/pkg/build" "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/npm" + "github.com/SAP/jenkins-library/pkg/versioning" "github.com/SAP/jenkins-library/pkg/command" piperhttp "github.com/SAP/jenkins-library/pkg/http" @@ -53,7 +56,7 @@ const ( NEO MTABuildTarget = iota // CF ... CF MTABuildTarget = iota - //XSA ... + // XSA ... XSA MTABuildTarget = iota ) @@ -67,7 +70,7 @@ func ValueOfBuildTarget(str string) (MTABuildTarget, error) { case "XSA": return XSA, nil default: - return -1, fmt.Errorf("Unknown Platform: '%s'", str) + return -1, fmt.Errorf("unknown platform: '%s'", str) } } @@ -94,6 +97,9 @@ type mtaBuildUtils interface { SetNpmRegistries(defaultNpmRegistry string) error InstallAllDependencies(defaultNpmRegistry string) error + + Open(name string) (io.ReadWriteCloser, error) + SendRequest(method, url string, body io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error) } type mtaBuildUtilsBundle struct { @@ -131,9 +137,7 @@ func newMtaBuildUtilsBundle() mtaBuildUtils { return &utils } -func mtaBuild(config mtaBuildOptions, - telemetryData *telemetry.CustomData, - commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment) { +func mtaBuild(config mtaBuildOptions, _ *telemetry.CustomData, commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment) { log.Entry().Debugf("Launching mta build") utils := newMtaBuildUtilsBundle() @@ -145,39 +149,31 @@ func mtaBuild(config mtaBuildOptions, } } -func runMtaBuild(config mtaBuildOptions, - commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment, - utils mtaBuildUtils) error { - - var err error - - err = handleSettingsFiles(config, utils) - if err != nil { +func runMtaBuild(config mtaBuildOptions, commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment, utils mtaBuildUtils) error { + if err := handleSettingsFiles(config, utils); err != nil { return err } - err = handleActiveProfileUpdate(config, utils) - if err != nil { + if err := handleActiveProfileUpdate(config, utils); err != nil { return err } - err = utils.SetNpmRegistries(config.DefaultNpmRegistry) + if err := utils.SetNpmRegistries(config.DefaultNpmRegistry); err != nil { + return err + } mtaYamlFile := filepath.Join(getSourcePath(config), "mta.yaml") mtaYamlFileExists, err := utils.FileExists(mtaYamlFile) - if err != nil { return err } if !mtaYamlFileExists { - if err = createMtaYamlFile(mtaYamlFile, config.ApplicationName, utils); err != nil { return err } - } else { - log.Entry().Infof("\"%s\" file found in project sources", mtaYamlFile) + log.Entry().Infof(`"%s" file found in project sources`, mtaYamlFile) } if config.EnableSetTimestamp { @@ -187,20 +183,17 @@ func runMtaBuild(config mtaBuildOptions, } mtarName, isMtarNativelySuffixed, err := getMtarName(config, mtaYamlFile, utils) - if err != nil { return err } - var call []string - platform, err := ValueOfBuildTarget(config.Platform) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err } - call = append(call, "mbt", "build", "--mtar", mtarName, "--platform", platform.String()) + call := []string{"mbt", "build", "--mtar", mtarName, "--platform", platform.String()} if len(config.Extensions) != 0 { call = append(call, fmt.Sprintf("--extensions=%s", config.Extensions)) } @@ -229,7 +222,7 @@ func runMtaBuild(config mtaBuildOptions, utils.AppendEnv([]string{"MAVEN_OPTS=-Dmaven.repo.local=" + absolutePath}) } - log.Entry().Infof("Executing mta build call: \"%s\"", strings.Join(call, " ")) + log.Entry().Infof(`Executing mta build call: "%s"`, strings.Join(call, " ")) if err := utils.RunExecutable(call[0], call[1:]...); err != nil { log.SetErrorCategory(log.ErrorBuild) @@ -261,65 +254,97 @@ func runMtaBuild(config mtaBuildOptions, commonPipelineEnvironment.custom.mtaBuildToolDesc = filepath.ToSlash(mtaYamlFile) if config.InstallArtifacts { - // install maven artifacts in local maven repo because `mbt build` executes `mvn package -B` - err = installMavenArtifacts(utils, config) - if err != nil { + if err = installMavenArtifacts(utils, config); err != nil { return err } - // mta-builder executes 'npm install --production', therefore we need 'npm ci/install' to install the dev-dependencies - err = utils.InstallAllDependencies(config.DefaultNpmRegistry) - if err != nil { + if err = utils.InstallAllDependencies(config.DefaultNpmRegistry); err != nil { return err } } if config.Publish { - log.Entry().Infof("publish detected") - if (len(config.MtaDeploymentRepositoryPassword) > 0) && (len(config.MtaDeploymentRepositoryUser) > 0) && - (len(config.MtaDeploymentRepositoryURL) > 0) { - if (len(config.MtarGroup) > 0) && (len(config.Version) > 0) { - httpClient := &piperhttp.Client{} + if err = handlePublish(config, commonPipelineEnvironment, utils, mtarName, isMtarNativelySuffixed); err != nil { + return err + } + } else { + log.Entry().Infof("no publish detected, skipping upload of mtar artifact") + } - credentialsEncoded := "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", config.MtaDeploymentRepositoryUser, config.MtaDeploymentRepositoryPassword))) - headers := http.Header{} - headers.Add("Authorization", credentialsEncoded) + return nil +} - config.MtarGroup = strings.ReplaceAll(config.MtarGroup, ".", "/") +func handlePublish(config mtaBuildOptions, commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment, utils mtaBuildUtils, mtarName string, isMtarNativelySuffixed bool) error { + log.Entry().Infof("publish detected") - mtarArtifactName := mtarName + if len(config.MtaDeploymentRepositoryPassword) == 0 || + len(config.MtaDeploymentRepositoryUser) == 0 || + len(config.MtaDeploymentRepositoryURL) == 0 { + return errors.New("mtaDeploymentRepositoryUser, mtaDeploymentRepositoryPassword and mtaDeploymentRepositoryURL not found, must be present") + } - // only trim the .mtar suffix from the mtarName - if !isMtarNativelySuffixed { - mtarArtifactName = strings.TrimSuffix(mtarArtifactName, ".mtar") - } + if len(config.MtarGroup) == 0 || len(config.Version) == 0 { + return errors.New("mtarGroup, version not found and must be present") + } - config.MtaDeploymentRepositoryURL += config.MtarGroup + "/" + mtarArtifactName + "/" + config.Version + "/" + fmt.Sprintf("%v-%v.%v", mtarArtifactName, config.Version, "mtar") + credentialsEncoded := "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", config.MtaDeploymentRepositoryUser, config.MtaDeploymentRepositoryPassword))) + headers := http.Header{} + headers.Add("Authorization", credentialsEncoded) - commonPipelineEnvironment.custom.mtarPublishedURL = config.MtaDeploymentRepositoryURL + config.MtarGroup = strings.ReplaceAll(config.MtarGroup, ".", "/") + mtarArtifactName := mtarName + if !isMtarNativelySuffixed { + mtarArtifactName = strings.TrimSuffix(mtarArtifactName, ".mtar") + } - log.Entry().Infof("pushing mtar artifact to repository : %s", config.MtaDeploymentRepositoryURL) + config.MtaDeploymentRepositoryURL += config.MtarGroup + "/" + mtarArtifactName + "/" + config.Version + "/" + fmt.Sprintf("%v-%v.%v", mtarArtifactName, config.Version, "mtar") + commonPipelineEnvironment.custom.mtarPublishedURL = config.MtaDeploymentRepositoryURL - data, err := os.Open(getMtarFilePath(config, mtarName)) - if err != nil { - return errors.Wrap(err, "failed to open mtar archive for upload") - } - _, httpErr := httpClient.SendRequest("PUT", config.MtaDeploymentRepositoryURL, data, headers, nil) + log.Entry().Infof("pushing mtar artifact to repository : %s", config.MtaDeploymentRepositoryURL) - if httpErr != nil { - return errors.Wrap(err, "failed to upload mtar to repository") - } - } else { - return errors.New("mtarGroup, version not found and must be present") + mtarPath := getMtarFilePath(config, mtarName) + data, err := utils.Open(mtarPath) + if err != nil { + return errors.Wrap(err, "failed to open mtar archive for upload") + } + defer data.Close() - } + if _, httpErr := utils.SendRequest("PUT", config.MtaDeploymentRepositoryURL, data, headers, nil); httpErr != nil { + return errors.Wrap(httpErr, "failed to upload mtar to repository") + } - } else { - return errors.New("mtaDeploymentRepositoryUser, mtaDeploymentRepositoryPassword and mtaDeploymentRepositoryURL not found , must be present") + if config.CreateBuildArtifactsMetadata { + if err := buildArtifactsMetadata(config, commonPipelineEnvironment, mtarPath); err != nil { + log.Entry().Warnf("unable to create build artifacts metadata: %v", err) + return nil } - } else { - log.Entry().Infof("no publish detected, skipping upload of mtar artifact") } - return err + + return nil +} + +func buildArtifactsMetadata(config mtaBuildOptions, commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment, mtarPath string) error { + mtarDir := filepath.Dir(mtarPath) + buildArtifacts := build.BuildArtifacts{ + Coordinates: []versioning.Coordinates{ + { + GroupID: config.MtarGroup, + ArtifactID: config.MtarName, + Version: config.Version, + Packaging: "mtar", + BuildPath: getSourcePath(config), + URL: config.MtaDeploymentRepositoryURL, + PURL: piperutils.GetPurl(filepath.Join(mtarDir, "sbom-gen/bom-mta.xml")), + }, + }, + } + + jsonResult, err := json.Marshal(buildArtifacts) + if err != nil { + return fmt.Errorf("failed to marshal build artifacts: %v", err) + } + + commonPipelineEnvironment.custom.mtaBuildArtifacts = string(jsonResult) + return nil } func handleActiveProfileUpdate(config mtaBuildOptions, utils mtaBuildUtils) error { @@ -355,15 +380,12 @@ func addNpmBinToPath(utils mtaBuildUtils) error { } func getMtarName(config mtaBuildOptions, mtaYamlFile string, utils mtaBuildUtils) (string, bool, error) { - mtarName := config.MtarName isMtarNativelySuffixed := false if len(mtarName) == 0 { - - log.Entry().Debugf("mtar name not provided via config. Extracting from file \"%s\"", mtaYamlFile) + log.Entry().Debugf(`mtar name not provided via config. Extracting from file "%s"`, mtaYamlFile) mtaID, err := getMtaID(mtaYamlFile, utils) - if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return "", isMtarNativelySuffixed, err @@ -371,10 +393,10 @@ func getMtarName(config mtaBuildOptions, mtaYamlFile string, utils mtaBuildUtils if len(mtaID) == 0 { log.SetErrorCategory(log.ErrorConfiguration) - return "", isMtarNativelySuffixed, fmt.Errorf("Invalid mtar ID. Was empty") + return "", isMtarNativelySuffixed, fmt.Errorf("invalid mtar ID. Was empty") } - log.Entry().Debugf("mtar name extracted from file \"%s\": \"%s\"", mtaYamlFile, mtaID) + log.Entry().Debugf(`mtar name extracted from file "%s": "%s"`, mtaYamlFile, mtaID) // there can be cases where the mtaId itself has the value com.myComapany.mtar , adding an extra .mtar causes .mtar.mtar if !strings.HasSuffix(mtaID, ".mtar") { @@ -387,11 +409,9 @@ func getMtarName(config mtaBuildOptions, mtaYamlFile string, utils mtaBuildUtils } return mtarName, isMtarNativelySuffixed, nil - } func setTimeStamp(mtaYamlFile string, utils mtaBuildUtils) error { - mtaYaml, err := utils.FileRead(mtaYamlFile) if err != nil { return err @@ -406,9 +426,9 @@ func setTimeStamp(mtaYamlFile string, utils mtaBuildUtils) error { log.SetErrorCategory(log.ErrorConfiguration) return err } - log.Entry().Infof("Timestamp replaced in \"%s\"", mtaYamlFile) + log.Entry().Infof(`Timestamp replaced in "%s"`, mtaYamlFile) } else { - log.Entry().Infof("No timestamp contained in \"%s\". File has not been modified.", mtaYamlFile) + log.Entry().Infof(`No timestamp contained in "%s". File has not been modified.`, mtaYamlFile) } return nil @@ -420,14 +440,16 @@ func getTimestamp() string { } func createMtaYamlFile(mtaYamlFile, applicationName string, utils mtaBuildUtils) error { - - log.Entry().Infof("\"%s\" file not found in project sources", mtaYamlFile) + log.Entry().Infof(`"%s" file not found in project sources`, mtaYamlFile) if len(applicationName) == 0 { return fmt.Errorf("'%[1]s' not found in project sources and 'applicationName' not provided as parameter - cannot generate '%[1]s' file", mtaYamlFile) } packageFileExists, err := utils.FileExists("package.json") + if err != nil { + return err + } if !packageFileExists { return fmt.Errorf("package.json file does not exist") } @@ -437,16 +459,18 @@ func createMtaYamlFile(mtaYamlFile, applicationName string, utils mtaBuildUtils) if err != nil { return err } - json.Unmarshal(pContent, &result) + if err := json.Unmarshal(pContent, &result); err != nil { + return fmt.Errorf("failed to unmarshal package.json: %w", err) + } version, ok := result["version"].(string) if !ok { - return fmt.Errorf("Version not found in \"package.json\" (or wrong type)") + return fmt.Errorf(`version not found in "package.json" (or wrong type)`) } name, ok := result["name"].(string) if !ok { - return fmt.Errorf("Name not found in \"package.json\" (or wrong type)") + return fmt.Errorf(`name not found in "package.json" (or wrong type)`) } mtaConfig, err := generateMta(name, applicationName, version) @@ -457,7 +481,7 @@ func createMtaYamlFile(mtaYamlFile, applicationName string, utils mtaBuildUtils) if err := utils.FileWrite(mtaYamlFile, []byte(mtaConfig), 0644); err != nil { return fmt.Errorf("failed to write %v: %w", mtaYamlFile, err) } - log.Entry().Infof("\"%s\" created.", mtaYamlFile) + log.Entry().Infof(`"%s" created.`, mtaYamlFile) return nil } @@ -467,15 +491,14 @@ func handleSettingsFiles(config mtaBuildOptions, utils mtaBuildUtils) error { } func generateMta(id, applicationName, version string) (string, error) { - if len(id) == 0 { - return "", fmt.Errorf("Generating mta file: ID not provided") + return "", fmt.Errorf("generating mta file: ID not provided") } if len(applicationName) == 0 { - return "", fmt.Errorf("Generating mta file: ApplicationName not provided") + return "", fmt.Errorf("generating mta file: ApplicationName not provided") } if len(version) == 0 { - return "", fmt.Errorf("Generating mta file: Version not provided") + return "", fmt.Errorf("generating mta file: Version not provided") } tmpl, e := template.New("mta.yaml").Parse(templateMtaYml) @@ -499,7 +522,6 @@ func generateMta(id, applicationName, version string) (string, error) { } func getMtaID(mtaYamlFile string, utils mtaBuildUtils) (string, error) { - var result map[string]interface{} p, err := utils.FileRead(mtaYamlFile) if err != nil { @@ -512,7 +534,7 @@ func getMtaID(mtaYamlFile string, utils mtaBuildUtils) (string, error) { id, ok := result["ID"].(string) if !ok || len(id) == 0 { - return "", fmt.Errorf("Id not found in mta yaml file (or wrong type)") + return "", fmt.Errorf("id not found in mta yaml file (or wrong type)") } return id, nil diff --git a/cmd/mtaBuild_generated.go b/cmd/mtaBuild_generated.go index 5a15ee3b8e..3c03768c6a 100644 --- a/cmd/mtaBuild_generated.go +++ b/cmd/mtaBuild_generated.go @@ -45,6 +45,7 @@ type mtaBuildOptions struct { BuildSettingsInfo string `json:"buildSettingsInfo,omitempty"` CreateBOM bool `json:"createBOM,omitempty"` EnableSetTimestamp bool `json:"enableSetTimestamp,omitempty"` + CreateBuildArtifactsMetadata bool `json:"createBuildArtifactsMetadata,omitempty"` } type mtaBuildCommonPipelineEnvironment struct { @@ -53,6 +54,7 @@ type mtaBuildCommonPipelineEnvironment struct { mtaBuildToolDesc string mtarPublishedURL string buildSettingsInfo string + mtaBuildArtifacts string } } @@ -66,6 +68,7 @@ func (p *mtaBuildCommonPipelineEnvironment) persist(path, resourceName string) { {category: "custom", name: "mtaBuildToolDesc", value: p.custom.mtaBuildToolDesc}, {category: "custom", name: "mtarPublishedUrl", value: p.custom.mtarPublishedURL}, {category: "custom", name: "buildSettingsInfo", value: p.custom.buildSettingsInfo}, + {category: "custom", name: "mtaBuildArtifacts", value: p.custom.mtaBuildArtifacts}, } errCount := 0 @@ -266,6 +269,7 @@ func addMtaBuildFlags(cmd *cobra.Command, stepConfig *mtaBuildOptions) { cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "build settings info is typically filled by the step automatically to create information about the build settings that were used during the mta build . This information is typically used for compliance related processes.") cmd.Flags().BoolVar(&stepConfig.CreateBOM, "createBOM", false, "Creates the bill of materials (BOM) using CycloneDX plugin.") cmd.Flags().BoolVar(&stepConfig.EnableSetTimestamp, "enableSetTimestamp", true, "Enables setting the timestamp in the `mta.yaml` when it contains `${timestamp}`. Disable this when you want the MTA Deploy Service to do this instead.") + cmd.Flags().BoolVar(&stepConfig.CreateBuildArtifactsMetadata, "createBuildArtifactsMetadata", false, "metadata about the artifacts that are build and published, this metadata is generally used by steps downstream in the pipeline") } @@ -529,6 +533,15 @@ func mtaBuildMetadata() config.StepData { Aliases: []config.Alias{}, Default: true, }, + { + Name: "createBuildArtifactsMetadata", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: false, + }, }, }, Containers: []config.Container{ @@ -544,6 +557,7 @@ func mtaBuildMetadata() config.StepData { {"name": "custom/mtaBuildToolDesc"}, {"name": "custom/mtarPublishedUrl"}, {"name": "custom/buildSettingsInfo"}, + {"name": "custom/mtaBuildArtifacts"}, }, }, { diff --git a/cmd/mtaBuild_test.go b/cmd/mtaBuild_test.go index 3f127c4944..273ff3af91 100644 --- a/cmd/mtaBuild_test.go +++ b/cmd/mtaBuild_test.go @@ -1,12 +1,15 @@ package cmd import ( + "bytes" "errors" + "io" "net/http" "os" "path/filepath" "testing" + "github.com/SAP/jenkins-library/pkg/config" "github.com/SAP/jenkins-library/pkg/mock" "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" @@ -18,6 +21,8 @@ type mtaBuildTestUtilsBundle struct { projectSettingsFile string globalSettingsFile string registryUsedInSetNpmRegistries string + openReturns string + sendRequestReturns func() (*http.Response, error) } func (m *mtaBuildTestUtilsBundle) SetNpmRegistries(defaultNpmRegistry string) error { @@ -26,7 +31,7 @@ func (m *mtaBuildTestUtilsBundle) SetNpmRegistries(defaultNpmRegistry string) er } func (m *mtaBuildTestUtilsBundle) InstallAllDependencies(defaultNpmRegistry string) error { - return errors.New("Test should not install dependencies.") //TODO implement test + return errors.New("Test should not install dependencies.") // TODO implement test } func (m *mtaBuildTestUtilsBundle) DownloadAndCopySettingsFiles(globalSettingsFile string, projectSettingsFile string) error { @@ -39,6 +44,36 @@ func (m *mtaBuildTestUtilsBundle) DownloadFile(url, filename string, header http return errors.New("Test should not download files.") } +func (m *mtaBuildTestUtilsBundle) Open(name string) (io.ReadWriteCloser, error) { + if m.openReturns != "" { + return NewMockReadCloser(m.openReturns), nil + } + return nil, errors.New("Test should not open files.") +} + +// MockReadCloser is a struct that implements io.ReadCloser +type MockReadWriteCloser struct { + io.Reader + io.Writer +} + +// Close is a no-op method to satisfy the io.Closer interface +func (m *MockReadWriteCloser) Close() error { + return nil +} + +// NewMockReadCloser returns a new MockReadCloser with the given data +func NewMockReadCloser(data string) io.ReadWriteCloser { + return &MockReadWriteCloser{Reader: bytes.NewBufferString(data)} +} + +func (m *mtaBuildTestUtilsBundle) SendRequest(method, url string, body io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error) { + if m.sendRequestReturns != nil { + return m.sendRequestReturns() + } + return nil, errors.New("Test should not send requests.") +} + func newMtaBuildTestUtilsBundle() *mtaBuildTestUtilsBundle { utilsBundle := mtaBuildTestUtilsBundle{ ExecMockRunner: &mock.ExecMockRunner{}, @@ -48,11 +83,11 @@ func newMtaBuildTestUtilsBundle() *mtaBuildTestUtilsBundle { } func TestMtaBuild(t *testing.T) { - cpe := mtaBuildCommonPipelineEnvironment{} - + SetConfigOptions(ConfigCommandOptions{ + OpenFile: config.OpenPiperFile, + }) t.Run("Application name not set", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() options := mtaBuildOptions{} @@ -60,15 +95,20 @@ func TestMtaBuild(t *testing.T) { assert.NotNil(t, err) assert.Equal(t, "'mta.yaml' not found in project sources and 'applicationName' not provided as parameter - cannot generate 'mta.yaml' file", err.Error()) - }) t.Run("Provide default npm registry", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", DefaultNpmRegistry: "https://example.org/npm", MtarName: "myName", Source: "./", Target: "./"} + options := mtaBuildOptions{ + ApplicationName: "myApp", + Platform: "CF", + DefaultNpmRegistry: "https://example.org/npm", + MtarName: "myName", + Source: "./", + Target: "./", + } - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -78,7 +118,6 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Package json does not exist", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() options := mtaBuildOptions{ApplicationName: "myApp"} @@ -88,16 +127,21 @@ func TestMtaBuild(t *testing.T) { assert.NotNil(t, err) assert.Equal(t, "package.json file does not exist", err.Error()) - }) t.Run("Write yaml file", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", MtarName: "myName", Source: "./", Target: "./", EnableSetTimestamp: true} + options := mtaBuildOptions{ + ApplicationName: "myApp", + Platform: "CF", + MtarName: "myName", + Source: "./", + Target: "./", + EnableSetTimestamp: true, + } - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -125,16 +169,14 @@ func TestMtaBuild(t *testing.T) { assert.Equal(t, "myApp", result.Modules[0].Name) assert.Regexp(t, "^1\\.2\\.3-[\\d]{14}$", result.Modules[0].Parameters["version"]) assert.Equal(t, "myApp", result.Modules[0].Parameters["name"]) - }) t.Run("Dont write mta yaml file when already present no timestamp placeholder", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() options := mtaBuildOptions{ApplicationName: "myApp"} - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) utilsMock.AddFile("mta.yaml", []byte("already there")) _ = runMtaBuild(options, &cpe, utilsMock) @@ -143,12 +185,14 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Write mta yaml file when already present with timestamp placeholder", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - options := mtaBuildOptions{ApplicationName: "myApp", EnableSetTimestamp: true} + options := mtaBuildOptions{ + ApplicationName: "myApp", + EnableSetTimestamp: true, + } - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) utilsMock.AddFile("mta.yaml", []byte("already there with-${timestamp}")) _ = runMtaBuild(options, &cpe, utilsMock) @@ -157,14 +201,13 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Mta build mbt toolset", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() cpe.mtarFilePath = "" options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", MtarName: "myName.mtar", Source: "./", Target: "./"} - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -179,14 +222,19 @@ func TestMtaBuild(t *testing.T) { t.Run("Source and target related tests", func(t *testing.T) { t.Run("Mta build mbt toolset with custom source and target paths", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() cpe.mtarFilePath = "" - options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", MtarName: "myName.mtar", Source: "mySourcePath/", Target: "myTargetPath/"} + options := mtaBuildOptions{ + ApplicationName: "myApp", + Platform: "CF", + MtarName: "myName.mtar", + Source: "mySourcePath/", + Target: "myTargetPath/", + } - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -194,9 +242,11 @@ func TestMtaBuild(t *testing.T) { if assert.Len(t, utilsMock.Calls, 1) { assert.Equal(t, "mbt", utilsMock.Calls[0].Exec) - assert.Equal(t, []string{"build", "--mtar", "myName.mtar", "--platform", "CF", + assert.Equal(t, []string{ + "build", "--mtar", "myName.mtar", "--platform", "CF", "--source", filepath.FromSlash("mySourcePath/"), - "--target", filepath.Join(_ignoreError(os.Getwd()), filepath.FromSlash("mySourcePath/myTargetPath/"))}, + "--target", filepath.Join(_ignoreError(os.Getwd()), filepath.FromSlash("mySourcePath/myTargetPath/")), + }, utilsMock.Calls[0].Params) } assert.Equal(t, "mySourcePath/myTargetPath/myName.mtar", cpe.mtarFilePath) @@ -206,14 +256,20 @@ func TestMtaBuild(t *testing.T) { t.Run("M2Path related tests", func(t *testing.T) { t.Run("Mta build mbt toolset with m2Path", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() utilsMock.CurrentDir = "root_folder/workspace" cpe.mtarFilePath = "" - options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", MtarName: "myName.mtar", Source: "./", Target: "./", M2Path: ".pipeline/local_repo"} + options := mtaBuildOptions{ + ApplicationName: "myApp", + Platform: "CF", + MtarName: "myName.mtar", + Source: "./", + Target: "./", + M2Path: ".pipeline/local_repo", + } - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -223,13 +279,18 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Settings file releatd tests", func(t *testing.T) { - t.Run("Copy global settings file", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) - - options := mtaBuildOptions{ApplicationName: "myApp", GlobalSettingsFile: "/opt/maven/settings.xml", Platform: "CF", MtarName: "myName", Source: "./", Target: "./"} + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "myName", + Source: "./", + Target: "./", + } err := runMtaBuild(options, &cpe, utilsMock) @@ -240,9 +301,8 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Copy project settings file", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) options := mtaBuildOptions{ApplicationName: "myApp", ProjectSettingsFile: "/my/project/settings.xml", Platform: "CF", MtarName: "myName", Source: "./", Target: "./"} @@ -256,36 +316,102 @@ func TestMtaBuild(t *testing.T) { }) t.Run("publish related tests", func(t *testing.T) { - t.Run("error when no repository url", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) - - options := mtaBuildOptions{ApplicationName: "myApp", GlobalSettingsFile: "/opt/maven/settings.xml", Platform: "CF", MtarName: "myName", Source: "./", Target: "./", Publish: true} + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "myName", + Source: "./", + Target: "./", + Publish: true, + } err := runMtaBuild(options, &cpe, utilsMock) - assert.Equal(t, "mtaDeploymentRepositoryUser, mtaDeploymentRepositoryPassword and mtaDeploymentRepositoryURL not found , must be present", err.Error()) + assert.Equal(t, "mtaDeploymentRepositoryUser, mtaDeploymentRepositoryPassword and mtaDeploymentRepositoryURL not found, must be present", err.Error()) }) t.Run("error when no mtar group", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) - - options := mtaBuildOptions{ApplicationName: "myApp", GlobalSettingsFile: "/opt/maven/settings.xml", Platform: "CF", MtarName: "myName", Source: "./", Target: "./", Publish: true, - MtaDeploymentRepositoryURL: "dummy", MtaDeploymentRepositoryPassword: "dummy", MtaDeploymentRepositoryUser: "dummy"} + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "myName", + Source: "./", + Target: "./", + Publish: true, + MtaDeploymentRepositoryURL: "dummy", + MtaDeploymentRepositoryPassword: "dummy", + MtaDeploymentRepositoryUser: "dummy", + } err := runMtaBuild(options, &cpe, utilsMock) assert.Equal(t, "mtarGroup, version not found and must be present", err.Error()) }) + + t.Run("successful publish", func(t *testing.T) { + utilsMock := newMtaBuildTestUtilsBundle() + utilsMock.sendRequestReturns = func() (*http.Response, error) { + return &http.Response{StatusCode: 200}, nil + } + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + utilsMock.openReturns = `{"version":"1.2.3"}` + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "test", + Source: "./", + Target: "./", + Publish: true, + MtaDeploymentRepositoryURL: "dummy", + MtaDeploymentRepositoryPassword: "dummy", + MtaDeploymentRepositoryUser: "dummy", + MtarGroup: "dummy", + Version: "dummy", + } + err := runMtaBuild(options, &cpe, utilsMock) + assert.Nil(t, err) + }) + + t.Run("succesful build artifact", func(t *testing.T) { + utilsMock := newMtaBuildTestUtilsBundle() + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + utilsMock.openReturns = `{"version":"1.2.3"}` + utilsMock.sendRequestReturns = func() (*http.Response, error) { + return &http.Response{StatusCode: 200}, nil + } + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "test", + Source: "./", + Target: "./", + Publish: true, + MtaDeploymentRepositoryURL: "dummy", + MtaDeploymentRepositoryPassword: "dummy", + MtaDeploymentRepositoryUser: "dummy", + MtarGroup: "dummy", + Version: "dummy", + CreateBuildArtifactsMetadata: true, + } + err := runMtaBuild(options, &cpe, utilsMock) + assert.Nil(t, err) + assert.Equal(t, cpe.custom.mtaBuildArtifacts, `{"Coordinates":[{"groupId":"dummy","artifactId":"test","version":"dummy","packaging":"mtar","buildPath":"./","url":"dummydummy/test/dummy/test-dummy.mtar","purl":""}]}`) + }) }) } func TestMtaBuildSourceDir(t *testing.T) { - cpe := mtaBuildCommonPipelineEnvironment{} t.Run("getSourcePath", func(t *testing.T) { t.Parallel() @@ -338,7 +464,7 @@ func TestMtaBuildSourceDir(t *testing.T) { t.Run("create mta.yaml from config.source", func(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) _ = runMtaBuild(mtaBuildOptions{ApplicationName: "myApp", Source: "create"}, &cpe, utilsMock) @@ -359,35 +485,33 @@ func TestMtaBuildSourceDir(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", DefaultNpmRegistry: "https://example.org/npm", MtarName: "myName", Source: "./", Target: "./", CreateBOM: true} - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) assert.Nil(t, err) assert.Contains(t, utilsMock.Calls[0].Params, "--sbom-file-path") - }) } func TestMtaBuildMtar(t *testing.T) { - t.Run("getMtarName", func(t *testing.T) { t.Parallel() t.Run("mtar name from yaml", func(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"nameFromMtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "nameFromMtar"`)) assert.Equal(t, filepath.FromSlash("nameFromMtar.mtar"), _ignoreErrorForGetMtarName(getMtarName(mtaBuildOptions{MtarName: ""}, "mta.yaml", utilsMock))) }) t.Run("mtar name from yaml with suffixed value", func(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"nameFromMtar.mtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "nameFromMtar.mtar"`)) assert.Equal(t, filepath.FromSlash("nameFromMtar.mtar"), _ignoreErrorForGetMtarName(getMtarName(mtaBuildOptions{MtarName: ""}, "mta.yaml", utilsMock))) }) t.Run("mtar name from config", func(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"nameFromMtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "nameFromMtar"`)) assert.Equal(t, filepath.FromSlash("nameFromConfig.mtar"), _ignoreErrorForGetMtarName(getMtarName(mtaBuildOptions{MtarName: "nameFromConfig.mtar"}, "mta.yaml", utilsMock))) }) @@ -412,7 +536,6 @@ func TestMtaBuildMtar(t *testing.T) { assert.Equal(t, filepath.FromSlash("source/target/mta.mtar"), getMtarFilePath(mtaBuildOptions{Source: "source", Target: "target"}, "mta.mtar")) }) }) - } func _ignoreError(s string, e error) string { diff --git a/pkg/npm/publish.go b/pkg/npm/publish.go index 3466e89aea..a5626281b3 100644 --- a/pkg/npm/publish.go +++ b/pkg/npm/publish.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperutils" CredentialUtils "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/versioning" ) @@ -217,7 +218,7 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p coordinate.BuildPath = filepath.Dir(packageJSON) coordinate.URL = registry coordinate.Packaging = "tgz" - coordinate.PURL = getPurl(packageJSON) + coordinate.PURL = piperutils.GetPurl(filepath.Join(filepath.Dir(packageJSON), npmBomFilename)) *buildCoordinates = append(*buildCoordinates, coordinate) } @@ -226,21 +227,6 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p return nil } -func getPurl(packageJSON string) string { - expectedBomFilePath := filepath.Join(filepath.Dir(packageJSON) + "/" + npmBomFilename) - exists, _ := CredentialUtils.FileExists(expectedBomFilePath) - if !exists { - log.Entry().Debugf("bom file doesn't exist and hence no pURL info: %v", expectedBomFilePath) - return "" - } - bom, err := CredentialUtils.GetBom(expectedBomFilePath) - if err != nil { - log.Entry().Warnf("unable to get bom metdata : %v", err) - return "" - } - return bom.Metadata.Component.Purl -} - func (exec *Execute) readPackageScope(packageJSON string) (string, error) { b, err := exec.Utils.FileRead(packageJSON) if err != nil { diff --git a/pkg/npm/publish_test.go b/pkg/npm/publish_test.go index 4c7b24f95d..bdbdf8b655 100644 --- a/pkg/npm/publish_test.go +++ b/pkg/npm/publish_test.go @@ -4,15 +4,15 @@ package npm import ( - "github.com/SAP/jenkins-library/pkg/mock" "io" "path/filepath" "testing" + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/versioning" "github.com/stretchr/testify/assert" - "os" ) type npmMockUtilsBundleRelativeGlob struct { @@ -531,7 +531,7 @@ func TestNpmPublish(t *testing.T) { // This stub simulates the behavior of npm pack and puts a tgz into the requested utils.execRunner.Stub = func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error { - //tgzTargetPath := filepath.Dir(test.packageDescriptors[0]) + // tgzTargetPath := filepath.Dir(test.packageDescriptors[0]) utils.AddFile(filepath.Join(".", "package.tgz"), []byte("this is a tgz file")) return nil } @@ -574,46 +574,3 @@ func TestNpmPublish(t *testing.T) { }) } } - -func createTempFile(t *testing.T, dir string, filename string, content string) string { - filePath := filepath.Join(dir, filename) - err := os.WriteFile(filePath, []byte(content), 0666) - if err != nil { - t.Fatalf("Failed to create temp file: %s", err) - } - return filePath -} - -func TestGetPurl(t *testing.T) { - t.Run("valid BOM file", func(t *testing.T) { - tempDir, err := piperutils.Files{}.TempDir("", "test") - if err != nil { - t.Fatalf("Failed to create temp directory: %s", err) - } - - bomContent := ` - - - pkg:npm/com.example/mycomponent@1.0.0 - - - - - - ` - packageJsonFilePath := createTempFile(t, tempDir, "package.json", "") - bomFilePath := createTempFile(t, tempDir, npmBomFilename, bomContent) - defer os.Remove(bomFilePath) - - purl := getPurl(packageJsonFilePath) - assert.Equal(t, "pkg:npm/com.example/mycomponent@1.0.0", purl) - }) - - t.Run("BOM file does not exist", func(t *testing.T) { - tempDir := t.TempDir() - packageJsonFilePath := createTempFile(t, tempDir, "pom.xml", "") // Create a temp pom file - - purl := getPurl(packageJsonFilePath) - assert.Equal(t, "", purl) - }) -} diff --git a/pkg/piperutils/cyclonedxBom.go b/pkg/piperutils/cyclonedxBom.go index fddebae8a6..51c8685235 100644 --- a/pkg/piperutils/cyclonedxBom.go +++ b/pkg/piperutils/cyclonedxBom.go @@ -2,9 +2,10 @@ package piperutils import ( "encoding/xml" - "github.com/SAP/jenkins-library/pkg/log" "io" "os" + + "github.com/SAP/jenkins-library/pkg/log" ) // To serialize the cyclonedx BOM file @@ -46,3 +47,12 @@ func GetBom(absoluteBomPath string) (Bom, error) { } return bom, nil } + +func GetPurl(bomFilePath string) string { + bom, err := GetBom(bomFilePath) + if err != nil { + log.Entry().Warnf("unable to get bom metadata: %v", err) + return "" + } + return bom.Metadata.Component.Purl +} diff --git a/pkg/piperutils/cyclonedxbom_test.go b/pkg/piperutils/cyclonedxbom_test.go index d9d25e6256..d4f6824293 100644 --- a/pkg/piperutils/cyclonedxbom_test.go +++ b/pkg/piperutils/cyclonedxbom_test.go @@ -18,6 +18,18 @@ func createTempFile(t *testing.T, content string) (string, func()) { } } +const validBom = ` + + + pkg:maven/com.example/mycomponent@1.0.0 + + + + + + + ` + func TestGetBom(t *testing.T) { tests := []struct { name string @@ -27,18 +39,8 @@ func TestGetBom(t *testing.T) { expectedError string }{ { - name: "valid file", - xmlContent: ` - - - pkg:maven/com.example/mycomponent@1.0.0 - - - - - - - `, + name: "valid file", + xmlContent: validBom, expectedBom: Bom{ Metadata: Metadata{ Component: BomComponent{ @@ -73,12 +75,8 @@ func TestGetBom(t *testing.T) { var fileName string var cleanup func() if tt.xmlContent != "" { - var err error fileName, cleanup = createTempFile(t, tt.xmlContent) defer cleanup() - if err != nil { - t.Fatalf("Failed to create temp file: %s", err) - } } else { // Use a non-existent file path fileName = "nonexistent.xml" @@ -102,6 +100,57 @@ func TestGetBom(t *testing.T) { } } +func TestGetPurl(t *testing.T) { + tests := []struct { + name string + filePath string + bomFilename string + xmlContent string + expectedPurl string + expectError bool + expectedError string + }{ + { + name: "valid BOM file", + xmlContent: validBom, + expectedPurl: "pkg:maven/com.example/mycomponent@1.0.0", + }, + { + name: "BOM file not found", + xmlContent: "", + expectedPurl: "", + expectError: true, + expectedError: "no such file or directory", + }, + { + name: "invalid BOM file", + xmlContent: "invalid xml", + expectedPurl: "", + expectError: true, + expectedError: "XML syntax error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var filePath string + var cleanup func() + if tt.xmlContent != "" { + filePath, cleanup = createTempFile(t, tt.xmlContent) + defer cleanup() + } else { + // Use a non-existent file path + filePath = "nonexistent.xml" + } + + purl := GetPurl(filePath) + if purl != tt.expectedPurl { + t.Errorf("Expected PURL: %v, got: %v", tt.expectedPurl, purl) + } + }) + } +} + func bomEquals(a, b Bom) bool { // compare a and b manually since reflect.DeepEqual can be problematic with slices and nil values return a.Metadata.Component.Purl == b.Metadata.Component.Purl && diff --git a/resources/metadata/mtaBuild.yaml b/resources/metadata/mtaBuild.yaml index acbe5c06a7..67d8fcdb3b 100644 --- a/resources/metadata/mtaBuild.yaml +++ b/resources/metadata/mtaBuild.yaml @@ -244,6 +244,14 @@ spec: - STAGES - PARAMETERS default: true + - name: createBuildArtifactsMetadata + type: bool + default: false + description: metadata about the artifacts that are build and published, this metadata is generally used by steps downstream in the pipeline + scope: + - STEPS + - STAGES + - PARAMETERS outputs: resources: - name: commonPipelineEnvironment @@ -253,6 +261,7 @@ spec: - name: custom/mtaBuildToolDesc - name: custom/mtarPublishedUrl - name: custom/buildSettingsInfo + - name: custom/mtaBuildArtifacts - name: reports type: reports params: From d4e298464e978a5505e3673ef5db08b6dec94b03 Mon Sep 17 00:00:00 2001 From: Holger Partsch Date: Mon, 4 Nov 2024 14:05:47 +0100 Subject: [PATCH 19/28] Final round of adding quoting to prevent command injection (#5167) * refactor: use import alias * fix: add quoting to further shell steps --- vars/npmExecute.groovy | 4 +++- vars/piperExecuteBin.groovy | 5 +++-- vars/sonarExecuteScan.groovy | 8 +++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/vars/npmExecute.groovy b/vars/npmExecute.groovy index 75eab2b6d0..f2dd21f3bc 100644 --- a/vars/npmExecute.groovy +++ b/vars/npmExecute.groovy @@ -1,4 +1,6 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q + import com.sap.piper.GenerateDocumentation import com.sap.piper.ConfigurationHelper import com.sap.piper.Utils @@ -65,7 +67,7 @@ void call(Map parameters = [:], body = null) { npm --version """ if (configuration.defaultNpmRegistry) { - sh "npm config set registry ${configuration.defaultNpmRegistry}" + sh "npm config set registry ${q(configuration.defaultNpmRegistry)}" } if (configuration.npmCommand) { sh "npm ${configuration.npmCommand}" diff --git a/vars/piperExecuteBin.groovy b/vars/piperExecuteBin.groovy index 87eec381a5..311242c81a 100644 --- a/vars/piperExecuteBin.groovy +++ b/vars/piperExecuteBin.groovy @@ -9,6 +9,7 @@ import com.sap.piper.analytics.InfluxData import groovy.transform.Field import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q @Field String STEP_NAME = getClass().getName() @@ -132,7 +133,7 @@ static String getCustomDefaultConfigs() { // resources by setupCommonPipelineEnvironment.groovy into .pipeline/. List customDefaults = DefaultValueCache.getInstance().getCustomDefaults() for (int i = 0; i < customDefaults.size(); i++) { - customDefaults[i] = BashUtils.quoteAndEscape(".pipeline/${customDefaults[i]}") + customDefaults[i] = q(".pipeline/${customDefaults[i]}") } return customDefaults.join(',') } @@ -151,7 +152,7 @@ static String getCustomConfigArg(def script) { if (script?.commonPipelineEnvironment?.configurationFile && script.commonPipelineEnvironment.configurationFile != '.pipeline/config.yml' && script.commonPipelineEnvironment.configurationFile != '.pipeline/config.yaml') { - return " --customConfig ${BashUtils.quoteAndEscape(script.commonPipelineEnvironment.configurationFile)}" + return " --customConfig ${q(script.commonPipelineEnvironment.configurationFile)}" } return '' } diff --git a/vars/sonarExecuteScan.groovy b/vars/sonarExecuteScan.groovy index 94d4c4fff6..dcb59e5cb8 100644 --- a/vars/sonarExecuteScan.groovy +++ b/vars/sonarExecuteScan.groovy @@ -1,7 +1,9 @@ +import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q + import com.sap.piper.JenkinsUtils import com.sap.piper.Utils import com.sap.piper.analytics.InfluxData -import static com.sap.piper.Prerequisites.checkScript import groovy.transform.Field import java.nio.charset.StandardCharsets @@ -40,7 +42,7 @@ void call(Map parameters = [:]) { // & `legacyPRHandling` & `inferBranchName` // writePipelineEnv needs to be called here as owner and repository may come from the pipeline environment writePipelineEnv(script: script, piperGoPath: piperGoPath) - Map stepConfig = readJSON(text: sh(returnStdout: true, script: "${piperGoPath} getConfig --stepMetadata '.pipeline/tmp/${METADATA_FILE}'${customDefaultConfig}${customConfigArg}")) + Map stepConfig = readJSON(text: sh(returnStdout: true, script: "${piperGoPath} getConfig --stepMetadata ${q('.pipeline/tmp/' + METADATA_FILE)}${customDefaultConfig}${customConfigArg}")) echo "Step Config: ${stepConfig}" List environment = [] @@ -135,7 +137,7 @@ private void loadCertificates(Map config) { def filename = new File(url).getName() filename = URLDecoder.decode(filename, StandardCharsets.UTF_8.name()) sh "wget ${wgetOptions.join(' ')} ${url}" - sh "keytool ${keytoolOptions.join(' ')} -alias '${filename}' -file '${certificateFolder}${filename}'" + sh "keytool ${keytoolOptions.join(' ')} -alias ${q(filename)} -file '${certificateFolder}${filename}'" } } } From 364238f1541540bb2e135b56a87ad9962acd649c Mon Sep 17 00:00:00 2001 From: Manjunath Date: Mon, 4 Nov 2024 14:20:37 +0100 Subject: [PATCH 20/28] handle error while fetching working directory (#5168) --- cmd/abapAddonAssemblyKitCheckCVs_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitCheckPV_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitCheck_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitCreateTargetVector_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitPublishTargetVector_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitRegisterPackages_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitReleasePackages_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitReserveNextPackages_generated.go | 7 +++++-- cmd/abapEnvironmentAssembleConfirm_generated.go | 7 +++++-- cmd/abapEnvironmentAssemblePackages_generated.go | 7 +++++-- cmd/abapEnvironmentBuild_generated.go | 7 +++++-- cmd/abapEnvironmentCheckoutBranch_generated.go | 7 +++++-- cmd/abapEnvironmentCloneGitRepo_generated.go | 7 +++++-- cmd/abapEnvironmentCreateSystem_generated.go | 7 +++++-- cmd/abapEnvironmentCreateTag_generated.go | 7 +++++-- cmd/abapEnvironmentPullGitRepo_generated.go | 7 +++++-- cmd/abapEnvironmentPushATCSystemConfig_generated.go | 7 +++++-- cmd/abapEnvironmentRunATCCheck_generated.go | 7 +++++-- cmd/abapEnvironmentRunAUnitTest_generated.go | 7 +++++-- cmd/abapLandscapePortalUpdateAddOnProduct_generated.go | 7 +++++-- cmd/ansSendEvent_generated.go | 7 +++++-- cmd/apiKeyValueMapDownload_generated.go | 7 +++++-- cmd/apiKeyValueMapUpload_generated.go | 7 +++++-- cmd/apiProviderDownload_generated.go | 7 +++++-- cmd/apiProviderList_generated.go | 7 +++++-- cmd/apiProviderUpload_generated.go | 7 +++++-- cmd/apiProxyDownload_generated.go | 7 +++++-- cmd/apiProxyList_generated.go | 7 +++++-- cmd/apiProxyUpload_generated.go | 7 +++++-- cmd/artifactPrepareVersion_generated.go | 7 +++++-- cmd/ascAppUpload_generated.go | 7 +++++-- cmd/awsS3Upload_generated.go | 7 +++++-- cmd/azureBlobUpload_generated.go | 7 +++++-- cmd/batsExecuteTests_generated.go | 7 +++++-- cmd/checkmarxExecuteScan_generated.go | 7 +++++-- cmd/checkmarxOneExecuteScan_generated.go | 7 +++++-- cmd/cloudFoundryCreateServiceKey_generated.go | 7 +++++-- cmd/cloudFoundryCreateService_generated.go | 7 +++++-- cmd/cloudFoundryCreateSpace_generated.go | 7 +++++-- cmd/cloudFoundryDeleteService_generated.go | 7 +++++-- cmd/cloudFoundryDeleteSpace_generated.go | 7 +++++-- cmd/cloudFoundryDeploy_generated.go | 7 +++++-- cmd/cnbBuild_generated.go | 7 +++++-- cmd/codeqlExecuteScan_generated.go | 7 +++++-- cmd/containerExecuteStructureTests_generated.go | 7 +++++-- cmd/containerSaveImage_generated.go | 7 +++++-- cmd/contrastExecuteScan_generated.go | 7 +++++-- cmd/credentialdiggerScan_generated.go | 7 +++++-- cmd/detectExecuteScan_generated.go | 7 +++++-- cmd/fortifyExecuteScan_generated.go | 7 +++++-- cmd/gaugeExecuteTests_generated.go | 7 +++++-- cmd/gcpPublishEvent_generated.go | 7 +++++-- cmd/gctsCloneRepository_generated.go | 7 +++++-- cmd/gctsCreateRepository_generated.go | 7 +++++-- cmd/gctsDeploy_generated.go | 7 +++++-- cmd/gctsExecuteABAPQualityChecks_generated.go | 7 +++++-- cmd/gctsExecuteABAPUnitTests_generated.go | 7 +++++-- cmd/gctsRollback_generated.go | 7 +++++-- cmd/githubCheckBranchProtection_generated.go | 7 +++++-- cmd/githubCommentIssue_generated.go | 7 +++++-- cmd/githubCreateIssue_generated.go | 7 +++++-- cmd/githubCreatePullRequest_generated.go | 7 +++++-- cmd/githubPublishRelease_generated.go | 7 +++++-- cmd/githubSetCommitStatus_generated.go | 7 +++++-- cmd/gitopsUpdateDeployment_generated.go | 7 +++++-- cmd/golangBuild_generated.go | 7 +++++-- cmd/gradleExecuteBuild_generated.go | 7 +++++-- cmd/hadolintExecute_generated.go | 7 +++++-- cmd/helmExecute_generated.go | 7 +++++-- cmd/imagePushToRegistry_generated.go | 7 +++++-- cmd/influxWriteData_generated.go | 7 +++++-- cmd/integrationArtifactDeploy_generated.go | 7 +++++-- cmd/integrationArtifactDownload_generated.go | 7 +++++-- cmd/integrationArtifactGetMplStatus_generated.go | 7 +++++-- cmd/integrationArtifactGetServiceEndpoint_generated.go | 7 +++++-- cmd/integrationArtifactResource_generated.go | 7 +++++-- cmd/integrationArtifactTransport_generated.go | 7 +++++-- cmd/integrationArtifactTriggerIntegrationTest_generated.go | 7 +++++-- cmd/integrationArtifactUnDeploy_generated.go | 7 +++++-- cmd/integrationArtifactUpdateConfiguration_generated.go | 7 +++++-- cmd/integrationArtifactUpload_generated.go | 7 +++++-- cmd/isChangeInDevelopment_generated.go | 7 +++++-- cmd/jsonApplyPatch_generated.go | 7 +++++-- cmd/kanikoExecute_generated.go | 7 +++++-- cmd/karmaExecuteTests_generated.go | 7 +++++-- cmd/kubernetesDeploy_generated.go | 7 +++++-- cmd/malwareExecuteScan_generated.go | 7 +++++-- cmd/mavenBuild_generated.go | 7 +++++-- cmd/mavenExecuteIntegration_generated.go | 7 +++++-- cmd/mavenExecuteStaticCodeChecks_generated.go | 7 +++++-- cmd/mavenExecute_generated.go | 7 +++++-- cmd/mtaBuild_generated.go | 7 +++++-- cmd/newmanExecute_generated.go | 7 +++++-- cmd/nexusUpload_generated.go | 7 +++++-- cmd/npmExecuteLint_generated.go | 7 +++++-- cmd/npmExecuteScripts_generated.go | 7 +++++-- cmd/pipelineCreateScanSummary_generated.go | 7 +++++-- cmd/protecodeExecuteScan_generated.go | 7 +++++-- cmd/pythonBuild_generated.go | 7 +++++-- cmd/shellExecute_generated.go | 7 +++++-- cmd/sonarExecuteScan_generated.go | 7 +++++-- cmd/terraformExecute_generated.go | 7 +++++-- cmd/tmsExport_generated.go | 7 +++++-- cmd/tmsUpload_generated.go | 7 +++++-- cmd/transportRequestDocIDFromGit_generated.go | 7 +++++-- cmd/transportRequestReqIDFromGit_generated.go | 7 +++++-- cmd/transportRequestUploadCTS_generated.go | 7 +++++-- cmd/transportRequestUploadRFC_generated.go | 7 +++++-- cmd/transportRequestUploadSOLMAN_generated.go | 7 +++++-- cmd/uiVeri5ExecuteTests_generated.go | 7 +++++-- cmd/vaultRotateSecretId_generated.go | 7 +++++-- cmd/whitesourceExecuteScan_generated.go | 7 +++++-- cmd/xsDeploy_generated.go | 7 +++++-- pkg/generator/helper/helper.go | 7 +++++-- .../helper/testdata/TestProcessMetaFiles/README.md | 1 + .../TestProcessMetaFiles/custom_step_code_generated.golden | 7 +++++-- .../TestProcessMetaFiles/step_code_generated.golden | 7 +++++-- 117 files changed, 581 insertions(+), 232 deletions(-) create mode 100644 pkg/generator/helper/testdata/TestProcessMetaFiles/README.md diff --git a/cmd/abapAddonAssemblyKitCheckCVs_generated.go b/cmd/abapAddonAssemblyKitCheckCVs_generated.go index 651d25e833..9f1af748f1 100644 --- a/cmd/abapAddonAssemblyKitCheckCVs_generated.go +++ b/cmd/abapAddonAssemblyKitCheckCVs_generated.go @@ -84,11 +84,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitCheckPV_generated.go b/cmd/abapAddonAssemblyKitCheckPV_generated.go index c3d4ac2206..68cbc03785 100644 --- a/cmd/abapAddonAssemblyKitCheckPV_generated.go +++ b/cmd/abapAddonAssemblyKitCheckPV_generated.go @@ -84,11 +84,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitCheck_generated.go b/cmd/abapAddonAssemblyKitCheck_generated.go index 0dec1e978e..6bf2fc877c 100644 --- a/cmd/abapAddonAssemblyKitCheck_generated.go +++ b/cmd/abapAddonAssemblyKitCheck_generated.go @@ -86,11 +86,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go b/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go index 2d237e7764..92dfb268f2 100644 --- a/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go +++ b/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go @@ -85,11 +85,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go b/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go index 87de4808af..3b99470eb7 100644 --- a/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go +++ b/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go @@ -56,11 +56,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitRegisterPackages_generated.go b/cmd/abapAddonAssemblyKitRegisterPackages_generated.go index 3d2eeffa73..71cdfd42bb 100644 --- a/cmd/abapAddonAssemblyKitRegisterPackages_generated.go +++ b/cmd/abapAddonAssemblyKitRegisterPackages_generated.go @@ -86,11 +86,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitReleasePackages_generated.go b/cmd/abapAddonAssemblyKitReleasePackages_generated.go index ca9f78c1fc..2a363fc4e9 100644 --- a/cmd/abapAddonAssemblyKitReleasePackages_generated.go +++ b/cmd/abapAddonAssemblyKitReleasePackages_generated.go @@ -86,11 +86,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go b/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go index 58f6e627b4..7d48f492b1 100644 --- a/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go +++ b/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go @@ -90,11 +90,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentAssembleConfirm_generated.go b/cmd/abapEnvironmentAssembleConfirm_generated.go index 9b7bfafdc2..b7f6880a97 100644 --- a/cmd/abapEnvironmentAssembleConfirm_generated.go +++ b/cmd/abapEnvironmentAssembleConfirm_generated.go @@ -83,11 +83,14 @@ func AbapEnvironmentAssembleConfirmCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentAssemblePackages_generated.go b/cmd/abapEnvironmentAssemblePackages_generated.go index e9ac906aaf..adfb53fea4 100644 --- a/cmd/abapEnvironmentAssemblePackages_generated.go +++ b/cmd/abapEnvironmentAssemblePackages_generated.go @@ -85,11 +85,14 @@ Platform ABAP Environment system and saves the corresponding [SAR archive](https GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentBuild_generated.go b/cmd/abapEnvironmentBuild_generated.go index c87a380f8d..85a20bc2b5 100644 --- a/cmd/abapEnvironmentBuild_generated.go +++ b/cmd/abapEnvironmentBuild_generated.go @@ -98,11 +98,14 @@ func AbapEnvironmentBuildCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentCheckoutBranch_generated.go b/cmd/abapEnvironmentCheckoutBranch_generated.go index 06fc95284f..342b11225e 100644 --- a/cmd/abapEnvironmentCheckoutBranch_generated.go +++ b/cmd/abapEnvironmentCheckoutBranch_generated.go @@ -59,11 +59,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentCloneGitRepo_generated.go b/cmd/abapEnvironmentCloneGitRepo_generated.go index c8759f5b83..e1da421535 100644 --- a/cmd/abapEnvironmentCloneGitRepo_generated.go +++ b/cmd/abapEnvironmentCloneGitRepo_generated.go @@ -62,11 +62,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentCreateSystem_generated.go b/cmd/abapEnvironmentCreateSystem_generated.go index 9928abc03c..7cf1d951f8 100644 --- a/cmd/abapEnvironmentCreateSystem_generated.go +++ b/cmd/abapEnvironmentCreateSystem_generated.go @@ -58,11 +58,14 @@ func AbapEnvironmentCreateSystemCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentCreateTag_generated.go b/cmd/abapEnvironmentCreateTag_generated.go index 22e9260669..2d0a2daa30 100644 --- a/cmd/abapEnvironmentCreateTag_generated.go +++ b/cmd/abapEnvironmentCreateTag_generated.go @@ -62,11 +62,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentPullGitRepo_generated.go b/cmd/abapEnvironmentPullGitRepo_generated.go index 05cc2a0c21..d763185c8d 100644 --- a/cmd/abapEnvironmentPullGitRepo_generated.go +++ b/cmd/abapEnvironmentPullGitRepo_generated.go @@ -61,11 +61,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentPushATCSystemConfig_generated.go b/cmd/abapEnvironmentPushATCSystemConfig_generated.go index c33c2d9eb9..ad1834feda 100644 --- a/cmd/abapEnvironmentPushATCSystemConfig_generated.go +++ b/cmd/abapEnvironmentPushATCSystemConfig_generated.go @@ -56,11 +56,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentRunATCCheck_generated.go b/cmd/abapEnvironmentRunATCCheck_generated.go index 04a52ad14d..43983a3115 100644 --- a/cmd/abapEnvironmentRunATCCheck_generated.go +++ b/cmd/abapEnvironmentRunATCCheck_generated.go @@ -61,11 +61,14 @@ Regardless of the option you chose, please make sure to provide the configuratio GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentRunAUnitTest_generated.go b/cmd/abapEnvironmentRunAUnitTest_generated.go index a3e865cd2d..19e35e7f5d 100644 --- a/cmd/abapEnvironmentRunAUnitTest_generated.go +++ b/cmd/abapEnvironmentRunAUnitTest_generated.go @@ -60,11 +60,14 @@ Regardless of the option you chose, please make sure to provide the object set c GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go b/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go index a44196072b..6be7825b22 100644 --- a/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go +++ b/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go @@ -44,11 +44,14 @@ func AbapLandscapePortalUpdateAddOnProductCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/ansSendEvent_generated.go b/cmd/ansSendEvent_generated.go index 9993c7745a..3f4e002034 100644 --- a/cmd/ansSendEvent_generated.go +++ b/cmd/ansSendEvent_generated.go @@ -53,11 +53,14 @@ func AnsSendEventCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiKeyValueMapDownload_generated.go b/cmd/apiKeyValueMapDownload_generated.go index 2ced0e9124..d69d948a8e 100644 --- a/cmd/apiKeyValueMapDownload_generated.go +++ b/cmd/apiKeyValueMapDownload_generated.go @@ -45,11 +45,14 @@ Learn more about the SAP API Management API for downloading an Key Value Map art GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiKeyValueMapUpload_generated.go b/cmd/apiKeyValueMapUpload_generated.go index c62d559b3f..75e0456bf8 100644 --- a/cmd/apiKeyValueMapUpload_generated.go +++ b/cmd/apiKeyValueMapUpload_generated.go @@ -46,11 +46,14 @@ Learn more about the SAP API Management API for creating an API key value map ar GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProviderDownload_generated.go b/cmd/apiProviderDownload_generated.go index def4eb576c..b972acde58 100644 --- a/cmd/apiProviderDownload_generated.go +++ b/cmd/apiProviderDownload_generated.go @@ -44,11 +44,14 @@ func ApiProviderDownloadCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProviderList_generated.go b/cmd/apiProviderList_generated.go index 76ce80c0f1..cb9ee968b2 100644 --- a/cmd/apiProviderList_generated.go +++ b/cmd/apiProviderList_generated.go @@ -81,11 +81,14 @@ func ApiProviderListCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProviderUpload_generated.go b/cmd/apiProviderUpload_generated.go index 1b1728099f..daf844d013 100644 --- a/cmd/apiProviderUpload_generated.go +++ b/cmd/apiProviderUpload_generated.go @@ -44,11 +44,14 @@ Learn more about API Management api for creating an API provider artifact [here] GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProxyDownload_generated.go b/cmd/apiProxyDownload_generated.go index 7f0238202b..0cae8c928b 100644 --- a/cmd/apiProxyDownload_generated.go +++ b/cmd/apiProxyDownload_generated.go @@ -44,11 +44,14 @@ func ApiProxyDownloadCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProxyList_generated.go b/cmd/apiProxyList_generated.go index 53f8c86d7b..2f4a2e2e96 100644 --- a/cmd/apiProxyList_generated.go +++ b/cmd/apiProxyList_generated.go @@ -81,11 +81,14 @@ func ApiProxyListCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProxyUpload_generated.go b/cmd/apiProxyUpload_generated.go index 51bf6c42a1..f8036b1407 100644 --- a/cmd/apiProxyUpload_generated.go +++ b/cmd/apiProxyUpload_generated.go @@ -44,11 +44,14 @@ Learn more about the SAP API Management API for uploading an api proxy artifact GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/artifactPrepareVersion_generated.go b/cmd/artifactPrepareVersion_generated.go index e39872d89b..24e29a34d1 100644 --- a/cmd/artifactPrepareVersion_generated.go +++ b/cmd/artifactPrepareVersion_generated.go @@ -172,11 +172,14 @@ Define ` + "`" + `buildTool: custom` + "`" + `, ` + "`" + `filePath: --install --force --namespace Date: Wed, 6 Nov 2024 12:51:14 +0200 Subject: [PATCH 21/28] docs: update pr template to include inner source update reminder (#5169) --- .github/PULL_REQUEST_TEMPLATE.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 74a197d2ed..2e60b49eb0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,7 @@ -# Changes +# Description + +# Checklist - [ ] Tests - [ ] Documentation +- [ ] Inner source library needs updating From 9626bfca0d830285917a218b837e14586039d5c4 Mon Sep 17 00:00:00 2001 From: Adrien <99400874+hubadr@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:14:04 +0100 Subject: [PATCH 22/28] checkmarxOneExecuteScan - Fix report generation in CxOne 3.20 (#5170) * Initial in progress * compiling but not yet functional * Missed file * updated checkmarxone step * Working up to fetching a project then breaks * Missed file * Breaks when retrieving projects+proxy set * Create project & run scan working, now polling * Fixed polling * added back the zipfile remove command * Fixed polling again * Generates and downloads PDF report * Updated and working, prep for refactor * Added compliance steps * Cleanup, reporting, added groovy connector * fixed groovy file * checkmarxone to checkmarxOne * checkmarxone to checkmarxOne * split credentials (id+secret, apikey), renamed pullrequestname to branch, groovy fix * Fixed filenames & yaml * missed the metadata_generated.go * added json to sarif conversion * fix:type in new checkmarxone package * fix:type in new checkmarxone package * removed test logs, added temp error log for creds * extra debugging to fix crash * improved auth logging, fixed query parse issue * fixed bug with group fetch when using oauth user * CWE can be -1 if not defined, can't be uint * Query also had CweID * Disabled predicates-fetch in sarif generation * Removing leftover info log message * Better error handling * fixed default preset configuration * removing .bat files - sorry * Cleanup per initial review * refactoring per Gist, fixed project find, add apps * small fix - sorry for commit noise while testing * Fixing issues with incremental scans. * removing maxretries * Updated per PR feedback, further changes todo toda * JSON Report changes and reporting cleanup * removing .bat (again?) * adding docs, groovy unit test, linter fixes * Started adding tests maybe 15% covered * fix(checkmarxOne): test cases for pkg and reporting * fix(checkmarxOne):fix formatting * feat(checkmarxone): update interface with missing method * feat(checkmarxone):change runStep signature to be able to inject dependency * feat(checkmarxone): add tests for step (wip) * Adding a bit more coverage * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix integration test PR * adding scan-summary bug workaround, reportgen fail * enforceThresholds fix when no results passed in * fixed gap when preset empty in yaml & project conf * fixed another gap in preset selection * fix 0-result panic * fail when no preset is set anywhere * removed comment * initial project-under-app support * fixing sarif reportgen * some cleanup of error messages * post-merge test fixes * revert previous upstream merge * adding "incremental" to "full" triggers * wrong boolean * project-in-application api change prep * Fixing SARIF report without preset access * fix sarif deeplink * removing comments * fix(cxone):formatting * fix(cxone):formatting * small sarif fixes * fixed merge * attempt at pulling git source repo branch * fix(cxone):new endpoint for project creation * fix(cxOne): taxa is an array * fix(cxOne): get Git branch from commonPipelineEnvironment * fix(cxOne): add params to tag a scan and a project * fix(cxOne): unit test - update project * fix(cxOne): unit test - update project tags * fix(cxOne): improve logs * fix(cxOne): improve logs * adding RequestNewPDFReport function using v2 api * added version check * fix(cxone): JSON report using v2 API * update to set reportType in v2 reportgen --------- Co-authored-by: michael kubiaczyk Co-authored-by: thtri Co-authored-by: Thanh-Hai Trinh Co-authored-by: michaelkubiaczyk <48311127+michaelkubiaczyk@users.noreply.github.com> Co-authored-by: sumeet patil --- pkg/checkmarxone/checkmarxone.go | 95 ++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 4 deletions(-) diff --git a/pkg/checkmarxone/checkmarxone.go b/pkg/checkmarxone/checkmarxone.go index f28a87a3cc..705aea76da 100644 --- a/pkg/checkmarxone/checkmarxone.go +++ b/pkg/checkmarxone/checkmarxone.go @@ -8,12 +8,10 @@ import ( "net/http" "net/url" "os" - - //"strconv" + "strconv" "strings" "time" - //"encoding/xml" piperHttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" @@ -1314,6 +1312,19 @@ func (sys *SystemInstance) GetResultsPredicates(SimilarityID int64, ProjectID st // RequestNewReport triggers the generation of a report for a specific scan addressed by scanID func (sys *SystemInstance) RequestNewReport(scanID, projectID, branch, reportType string) (string, error) { + if strings.EqualFold("pdf", reportType) || strings.EqualFold("json", reportType) { + version, err := sys.GetVersion() + if err == nil { + if version.CheckCxOne("3.20.0") >= 0 && version.CheckCxOne("3.21.0") == -1 { + sys.logger.Debugf("Current version is %v - between 3.20.0 and 3.21.0 - using v2 %v report", reportType, version.CxOne) + return sys.RequestNewReportV2(scanID, reportType) + } + sys.logger.Debugf("Current version is %v - using v1 %v report", reportType, version.CxOne) + } else { + sys.logger.Errorf("Failed to get the CxOne version during report-gen request, will use v1 %v report. Error: %s", reportType, err) + } + } + jsonData := map[string]interface{}{ "fileFormat": reportType, "reportType": "ui", @@ -1352,13 +1363,52 @@ func (sys *SystemInstance) RequestNewReport(scanID, projectID, branch, reportTyp return reportResponse.ReportId, err } +// Use the new V2 Report API to generate a PDF report +func (sys *SystemInstance) RequestNewReportV2(scanID, reportType string) (string, error) { + jsonData := map[string]interface{}{ + "reportName": "improved-scan-report", + "entities": []map[string]interface{}{ + { + "entity": "scan", + "ids": []string{scanID}, + "tags": []string{}, + }, + }, + "filters": map[string][]string{ + "scanners": {"sast"}, + }, + "reportType": "ui", + "fileFormat": reportType, + } + + jsonValue, _ := json.Marshal(jsonData) + + header := http.Header{} + header.Set("cxOrigin", cxOrigin) + header.Set("Content-Type", "application/json") + data, err := sendRequest(sys, http.MethodPost, "/reports/v2", bytes.NewBuffer(jsonValue), header, []int{}) + if err != nil { + return "", errors.Wrapf(err, "Failed to trigger report generation for scan %v", scanID) + } else { + sys.logger.Infof("Generating report %v", string(data)) + } + + var reportResponse struct { + ReportId string + } + err = json.Unmarshal(data, &reportResponse) + + return reportResponse.ReportId, err + +} + // GetReportStatus returns the status of the report generation process func (sys *SystemInstance) GetReportStatus(reportID string) (ReportStatus, error) { var response ReportStatus header := http.Header{} header.Set("Accept", "application/json") - data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/reports/%v", reportID), nil, header, []int{}) + data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/reports/%v?returnUrl=true", reportID), nil, header, []int{}) if err != nil { sys.logger.Errorf("Failed to fetch report status for reportID %v: %s", reportID, err) return response, errors.Wrapf(err, "failed to fetch report status for reportID %v", reportID) @@ -1412,3 +1462,40 @@ func (sys *SystemInstance) GetVersion() (VersionInfo, error) { err = json.Unmarshal(data, &version) return version, err } + +func (v VersionInfo) CheckCxOne(version string) int { + check := versionStringToInts(version) + cx1 := versionStringToInts(v.CxOne) + + if check[0] < cx1[0] { + return 1 + } else if check[0] > cx1[0] { + return -1 + } else { + if check[1] < cx1[1] { + return 1 + } else if check[1] > cx1[1] { + return -1 + } else { + if check[2] < cx1[2] { + return 1 + } else if check[2] > cx1[2] { + return -1 + } else { + return 0 + } + } + } +} + +func versionStringToInts(version string) []int64 { + if version == "" { + return []int64{0, 0, 0} + } + str := strings.Split(version, ".") + ints := make([]int64, len(str)) + for id, val := range str { + ints[id], _ = strconv.ParseInt(val, 10, 64) + } + return ints +} From 1edf8299f3f3457536c38c3073b6e510c96183c7 Mon Sep 17 00:00:00 2001 From: christian <153755613+skateball@users.noreply.github.com> Date: Fri, 8 Nov 2024 09:40:51 +0100 Subject: [PATCH 23/28] Update URL to current SapMachine (#5126) * Update URL to recent SapMachine 11.0.2 in from Jan2019 := 5 years old use the recent version => 11.0.24 * 11.0.24 to 25 meanwhile 11.0.25 is the latest version * update from 11.0.25 to stable 21 link now to use https://sap.github.io/SapMachine/latest/21/linux-x64/jre/ * applied generated go code * Update whitesourceExecuteScan.yaml * from 21-latest to 17.0.13 to speed this up * revert description * generate for new url --------- Co-authored-by: Harald Aamot Co-authored-by: Oliver Feldmann --- cmd/whitesourceExecuteScan_generated.go | 4 ++-- resources/metadata/whitesourceExecuteScan.yaml | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/whitesourceExecuteScan_generated.go b/cmd/whitesourceExecuteScan_generated.go index 19a1f2021a..c41928a315 100644 --- a/cmd/whitesourceExecuteScan_generated.go +++ b/cmd/whitesourceExecuteScan_generated.go @@ -369,7 +369,7 @@ func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceE cmd.Flags().BoolVar(&stepConfig.FailOnSevereVulnerabilities, "failOnSevereVulnerabilities", true, "Whether to fail the step on severe vulnerabilties or not") cmd.Flags().StringSliceVar(&stepConfig.Includes, "includes", []string{}, "List of file path patterns to include in the scan.") cmd.Flags().StringVar(&stepConfig.InstallCommand, "installCommand", os.Getenv("PIPER_installCommand"), "Install command that can be used to populate the default docker image for some scenarios.") - cmd.Flags().StringVar(&stepConfig.JreDownloadURL, "jreDownloadUrl", `https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz`, "URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent.") + cmd.Flags().StringVar(&stepConfig.JreDownloadURL, "jreDownloadUrl", `https://github.com/SAP/SapMachine/releases/download/sapmachine-17.0.13/sapmachine-jre-17.0.13_linux-x64_bin.tar.gz`, "URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent.") cmd.Flags().BoolVar(&stepConfig.LicensingVulnerabilities, "licensingVulnerabilities", true, "[NOT IMPLEMENTED] Whether license compliance is considered and reported as part of the assessment.") cmd.Flags().StringVar(&stepConfig.OrgToken, "orgToken", os.Getenv("PIPER_orgToken"), "WhiteSource token identifying your organization.") cmd.Flags().StringVar(&stepConfig.ProductName, "productName", os.Getenv("PIPER_productName"), "Name of the WhiteSource product used for results aggregation. This parameter is mandatory if the parameter `createProductFromPipeline` is set to `true` and the WhiteSource product does not yet exist. It is also mandatory if the parameter `productToken` is not provided.") @@ -692,7 +692,7 @@ func whitesourceExecuteScanMetadata() config.StepData { Type: "string", Mandatory: false, Aliases: []config.Alias{{Name: "whitesource/jreDownloadUrl", Deprecated: true}}, - Default: `https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz`, + Default: `https://github.com/SAP/SapMachine/releases/download/sapmachine-17.0.13/sapmachine-jre-17.0.13_linux-x64_bin.tar.gz`, }, { Name: "licensingVulnerabilities", diff --git a/resources/metadata/whitesourceExecuteScan.yaml b/resources/metadata/whitesourceExecuteScan.yaml index 9b030ac5b4..85298d9c76 100644 --- a/resources/metadata/whitesourceExecuteScan.yaml +++ b/resources/metadata/whitesourceExecuteScan.yaml @@ -257,14 +257,13 @@ spec: deprecated: true type: string description: - "URL used for downloading the Java Runtime Environment (JRE) required to run the - WhiteSource Unified Agent." + "URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent." scope: - GENERAL - PARAMETERS - STAGES - STEPS - default: "https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz" + default: "https://github.com/SAP/SapMachine/releases/download/sapmachine-17.0.13/sapmachine-jre-17.0.13_linux-x64_bin.tar.gz" - name: licensingVulnerabilities type: bool description: "[NOT IMPLEMENTED] Whether license compliance is considered and reported as part of the assessment." From 758d10b06e3dee6673642d89e66ff77b00a76dd9 Mon Sep 17 00:00:00 2001 From: maxcask Date: Tue, 12 Nov 2024 15:33:38 +0400 Subject: [PATCH 24/28] handle new naming strategy for Central Build stage name (#5171) * config helper rename * rename in name provider --------- Co-authored-by: maxcask --- src/com/sap/piper/ConfigurationHelper.groovy | 3 +++ src/com/sap/piper/StageNameProvider.groovy | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index 2eeaaf0e6f..b6b1558ddf 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -10,6 +10,9 @@ class ConfigurationHelper implements Serializable { } ConfigurationHelper loadStepDefaults(Map compatibleParameters = [:], String stageName = step.env.STAGE_NAME) { + if (stageName == 'Central Build'){ + stageName = 'Build' + } DefaultValueCache.prepare(step) this.config = ConfigurationLoader.defaultGeneralConfiguration() mixin(ConfigurationLoader.defaultGeneralConfiguration(), null, compatibleParameters) diff --git a/src/com/sap/piper/StageNameProvider.groovy b/src/com/sap/piper/StageNameProvider.groovy index 675387ca4c..cd2fa71f06 100644 --- a/src/com/sap/piper/StageNameProvider.groovy +++ b/src/com/sap/piper/StageNameProvider.groovy @@ -3,13 +3,18 @@ package com.sap.piper @Singleton class StageNameProvider implements Serializable { static final long serialVersionUID = 1L + static final String CENTRAL_BUILD = "Central Build"; + static final String BUILD = "Build"; /** Stores a feature toggle for defaulting to technical names in stages */ boolean useTechnicalStageNames = false String getStageName(Script script, Map parameters, Script step) { + String stageName = null if (parameters.stageName in CharSequence) { - return parameters.stageName + stageName = parameters.stageName + stageName = replaceCentralBuild(stageName); + return stageName } if (this.useTechnicalStageNames) { String technicalStageName = getTechnicalStageName(step) @@ -17,7 +22,15 @@ class StageNameProvider implements Serializable { return technicalStageName } } - return script.env.STAGE_NAME + if (stageName == null) { + stageName = script.env.STAGE_NAME + stageName = replaceCentralBuild(stageName); + } + return stageName + } + + private String replaceCentralBuild(String stageName) { + return CENTRAL_BUILD.equals(stageName) ? BUILD : stageName; } static String getTechnicalStageName(Script step) { From f54dbfd433a68a9b3f0f31b06f6ce8e2b5ab0c42 Mon Sep 17 00:00:00 2001 From: Ralf Pannemans Date: Mon, 18 Nov 2024 09:51:35 +0100 Subject: [PATCH 25/28] Remove commit message from project-metadata.toml (#5176) --- pkg/cnbutils/project/metadata/metadata.go | 3 +-- pkg/cnbutils/project/metadata/metadata_test.go | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/cnbutils/project/metadata/metadata.go b/pkg/cnbutils/project/metadata/metadata.go index ec6483d3d2..556e89b8ec 100644 --- a/pkg/cnbutils/project/metadata/metadata.go +++ b/pkg/cnbutils/project/metadata/metadata.go @@ -36,8 +36,7 @@ func extractMetadataFromCPE(piperEnvRoot string, utils cnbutils.BuildUtils) file Source: &files.ProjectSource{ Type: "git", Version: map[string]interface{}{ - "commit": piperenv.GetResourceParameter(cpePath, "git", "headCommitId"), - "describe": piperenv.GetResourceParameter(cpePath, "git", "commitMessage"), + "commit": piperenv.GetResourceParameter(cpePath, "git", "headCommitId"), }, Metadata: map[string]interface{}{ "refs": []string{ diff --git a/pkg/cnbutils/project/metadata/metadata_test.go b/pkg/cnbutils/project/metadata/metadata_test.go index cf36c9eaaa..95a693098e 100644 --- a/pkg/cnbutils/project/metadata/metadata_test.go +++ b/pkg/cnbutils/project/metadata/metadata_test.go @@ -19,7 +19,6 @@ func TestWriteProjectMetadata(t *testing.T) { type = "git" [source.version] commit = "012548" - describe = "test-commit" [source.metadata] refs = ["main"] ` @@ -31,9 +30,8 @@ func TestWriteProjectMetadata(t *testing.T) { fileutils := piperutils.Files{} cpeFiles := map[string]string{ - "headCommitId": "012548", - "commitMessage": "test-commit", - "branch": "main", + "headCommitId": "012548", + "branch": "main", } dir := t.TempDir() From 10f535c8fa2643981cd8672d1dafb87d88b1cba4 Mon Sep 17 00:00:00 2001 From: maxcask Date: Tue, 19 Nov 2024 11:58:55 +0400 Subject: [PATCH 26/28] fix handle new naming strategy for Central Build stage name if extension is used (#5178) * handle extension filename --------- Co-authored-by: maxcask Co-authored-by: Googlom --- vars/piperStageWrapper.groovy | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/vars/piperStageWrapper.groovy b/vars/piperStageWrapper.groovy index 1d6f1a3195..142a89617a 100644 --- a/vars/piperStageWrapper.groovy +++ b/vars/piperStageWrapper.groovy @@ -92,6 +92,7 @@ private void stageLocking(Map config, Closure body) { private void executeStage(script, originalStage, stageName, config, utils, telemetryDisabled = false) { boolean projectExtensions boolean globalExtensions + def startTime = System.currentTimeMillis() try { @@ -103,6 +104,18 @@ private void executeStage(script, originalStage, stageName, config, utils, telem */ def projectInterceptorFile = "${config.projectExtensionsDirectory}${stageName}.groovy" def globalInterceptorFile = "${config.globalExtensionsDirectory}${stageName}.groovy" + /* due to renaming stage 'Central Build' to 'Build' need to define extension file name 'Central Build.groovy' + as stageName used to generate it, once all the users will 'Build' as a stageName + and extension filename, below renaming snippet should be removed + */ + if (stageName == 'Build'){ + if (!fileExists(projectInterceptorFile) || !fileExists(globalInterceptorFile)){ + def centralBuildExtensionFileName = "Central Build.groovy" + projectInterceptorFile = "${config.projectExtensionsDirectory}${centralBuildExtensionFileName}" + globalInterceptorFile = "${config.globalExtensionsDirectory}${centralBuildExtensionFileName}" + } + } + projectExtensions = fileExists(projectInterceptorFile) globalExtensions = fileExists(globalInterceptorFile) // Pre-defining the real originalStage in body variable, might be overwritten later if extensions exist From f044ef106dd453b121c4fe8e4f157c1223022bbc Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:30:03 +0500 Subject: [PATCH 27/28] fix(curl): Remove --insecure flag from curl when downlaoding Piper binary (#5179) * remove --insecure flag from binary download * remove --insecure flag from binary download * adjust unit tests --- src/com/sap/piper/PiperGoUtils.groovy | 2 +- .../com/sap/piper/PiperGoUtilsTest.groovy | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/com/sap/piper/PiperGoUtils.groovy b/src/com/sap/piper/PiperGoUtils.groovy index 1206647d0d..3c2f63036e 100644 --- a/src/com/sap/piper/PiperGoUtils.groovy +++ b/src/com/sap/piper/PiperGoUtils.groovy @@ -83,7 +83,7 @@ class PiperGoUtils implements Serializable { private boolean downloadGoBinary(url) { try { - def httpStatus = steps.sh(returnStdout: true, script: "curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out '%{http_code}' --output ${piperExecutable} '${url}'") + def httpStatus = steps.sh(returnStdout: true, script: "curl --silent --retry 5 --retry-max-time 240 --location --write-out '%{http_code}' --output ${piperExecutable} '${url}'") if (httpStatus == '200') { steps.sh(script: "chmod +x ${piperExecutable}") diff --git a/test/groovy/com/sap/piper/PiperGoUtilsTest.groovy b/test/groovy/com/sap/piper/PiperGoUtilsTest.groovy index 94d49b55ed..42e63a74a3 100644 --- a/test/groovy/com/sap/piper/PiperGoUtilsTest.groovy +++ b/test/groovy/com/sap/piper/PiperGoUtilsTest.groovy @@ -63,11 +63,11 @@ class PiperGoUtilsTest extends BasePiperTest { }) shellCallRule.setReturnValue('[ -x ./piper ]', 1) - shellCallRule.setReturnValue('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'', '200') + shellCallRule.setReturnValue('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'', '200') piperGoUtils.unstashPiperBin() assertThat(shellCallRule.shell.size(), is(4)) - assertThat(shellCallRule.shell[1].toString(), is('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'')) + assertThat(shellCallRule.shell[1].toString(), is('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'')) assertThat(shellCallRule.shell[2].toString(), is('chmod +x piper')) assertThat(shellCallRule.shell[3].toString(), is('./piper version')) } @@ -84,11 +84,11 @@ class PiperGoUtilsTest extends BasePiperTest { }) shellCallRule.setReturnValue('[ -x ./piper ]', 1) - shellCallRule.setReturnValue('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/testTag/piper\'', '200') + shellCallRule.setReturnValue('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/testTag/piper\'', '200') piperGoUtils.unstashPiperBin() assertThat(shellCallRule.shell.size(), is(4)) - assertThat(shellCallRule.shell[1].toString(), is('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/testTag/piper\'')) + assertThat(shellCallRule.shell[1].toString(), is('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/testTag/piper\'')) assertThat(shellCallRule.shell[2].toString(), is('chmod +x piper')) assertThat(shellCallRule.shell[3].toString(), is('./piper version')) } @@ -101,8 +101,8 @@ class PiperGoUtilsTest extends BasePiperTest { shellCallRule.setReturnValue('[ -x ./piper ]', 1) shellCallRule.setReturnValue('./piper version', "1.2.3") - shellCallRule.setReturnValue('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/notAvailable/piper\'', '404') - shellCallRule.setReturnValue('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'', '200') + shellCallRule.setReturnValue('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/notAvailable/piper\'', '404') + shellCallRule.setReturnValue('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'', '200') // this mocks utils.unstash - mimic stash not existing helper.registerAllowedMethod("unstash", [String.class], { stashFileName -> @@ -112,8 +112,8 @@ class PiperGoUtilsTest extends BasePiperTest { piperGoUtils.unstashPiperBin() assertThat(shellCallRule.shell.size(), is(5)) assertThat(shellCallRule.shell[0].toString(), is('[ -x ./piper ]')) - assertThat(shellCallRule.shell[1].toString(), is('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/notAvailable/piper\'')) - assertThat(shellCallRule.shell[2].toString(), is('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'')) + assertThat(shellCallRule.shell[1].toString(), is('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/notAvailable/piper\'')) + assertThat(shellCallRule.shell[2].toString(), is('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'')) assertThat(shellCallRule.shell[3].toString(), is('chmod +x piper')) assertThat(shellCallRule.shell[4].toString(), is ('./piper version')) } @@ -124,8 +124,8 @@ class PiperGoUtilsTest extends BasePiperTest { piperGoUtils.metaClass.getLibrariesInfo = {-> return [[name: 'piper-lib-os', version: 'notAvailable']]} shellCallRule.setReturnValue('[ -x ./piper ]', 1) - shellCallRule.setReturnValue('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/notAvailable/piper\'', '404') - shellCallRule.setReturnValue('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'', '500') + shellCallRule.setReturnValue('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/notAvailable/piper\'', '404') + shellCallRule.setReturnValue('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'', '500') helper.registerAllowedMethod("unstash", [String.class], { stashFileName -> return [] @@ -141,8 +141,8 @@ class PiperGoUtilsTest extends BasePiperTest { piperGoUtils.metaClass.getLibrariesInfo = {-> return [[name: 'piper-lib-os', version: 'notAvailable']]} shellCallRule.setReturnValue('[ -x ./piper ]', 1) - shellCallRule.setReturnValue('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/notAvailable/piper\'', '404') - shellCallRule.setReturnValue('curl --insecure --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'', '500') + shellCallRule.setReturnValue('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/download/notAvailable/piper\'', '404') + shellCallRule.setReturnValue('curl --silent --retry 5 --retry-max-time 240 --location --write-out \'%{http_code}\' --output piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper\'', '500') helper.registerAllowedMethod("unstash", [String.class], { stashFileName -> return [] From cd548193a0f604eb568d1bdf0a9d3a5488b2e6d0 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Fri, 22 Nov 2024 14:04:12 +0100 Subject: [PATCH 28/28] Renamed maven simple bom to skip uploading to cumulus (#5180) --- cmd/mavenBuild.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index 5f8a4b0a96..aa653ee98e 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -23,7 +23,7 @@ import ( const ( mvnBomFilename = "bom-maven" - mvnSimpleBomFilename = "bom-simple" + mvnSimpleBomFilename = "simple-bom-maven" ) func mavenBuild(config mavenBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) {