From 2cc83d2ee90c5abe9abc573ca4b3f906757eb1d6 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Sep 2024 15:58:48 -0400 Subject: [PATCH 01/16] begin passing through additional CAs --- cmd/kots/cli/install.go | 12 ++++++++++++ pkg/kotsadm/types/deployoptions.go | 1 + 2 files changed, 13 insertions(+) diff --git a/cmd/kots/cli/install.go b/cmd/kots/cli/install.go index d4f87d5ffa..6a56f31815 100644 --- a/cmd/kots/cli/install.go +++ b/cmd/kots/cli/install.go @@ -277,6 +277,16 @@ func InstallCmd() *cobra.Command { additionalAnnotations[parts[0]] = parts[1] } + caFiles := []string{} + for _, caFile := range v.GetStringSlice("private-ca") { + // read CA file contents + caData, err := os.ReadFile(caFile) + if err != nil { + return fmt.Errorf("failed to read CA file %s: %w", caFile, err) + } + caFiles = append(caFiles, string(caData)) + } + deployOptions := kotsadmtypes.DeployOptions{ Namespace: namespace, Context: v.GetString("context"), @@ -308,6 +318,7 @@ func InstallCmd() *cobra.Command { RequestedChannelSlug: preferredChannelSlug, AdditionalLabels: additionalLabels, AdditionalAnnotations: additionalAnnotations, + AdditionalTrustedCAs: caFiles, RegistryConfig: *registryConfig, @@ -551,6 +562,7 @@ func InstallCmd() *cobra.Command { cmd.Flags().Bool("exclude-admin-console", false, "set to true to exclude the admin console and only install the application") cmd.Flags().StringArray("additional-annotations", []string{}, "additional annotations to add to kotsadm pods") cmd.Flags().StringArray("additional-labels", []string{}, "additional labels to add to kotsadm pods") + cmd.Flags().StringArray("private-ca", []string{}, "Path to a trusted private CA certificate file") registryFlags(cmd.Flags()) diff --git a/pkg/kotsadm/types/deployoptions.go b/pkg/kotsadm/types/deployoptions.go index 35a061d600..89907170ce 100644 --- a/pkg/kotsadm/types/deployoptions.go +++ b/pkg/kotsadm/types/deployoptions.go @@ -59,6 +59,7 @@ type DeployOptions struct { RequestedChannelSlug string AdditionalAnnotations map[string]string AdditionalLabels map[string]string + AdditionalTrustedCAs []string IdentityConfig kotsv1beta1.IdentityConfig IngressConfig kotsv1beta1.IngressConfig From ef6c2e9121b4f9429b36523c6a5fe7350de1b10d Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 12 Sep 2024 13:50:58 -0400 Subject: [PATCH 02/16] refer to existing configmaps --- cmd/kots/cli/install.go | 16 ++++------------ pkg/kotsadm/types/deployoptions.go | 3 ++- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cmd/kots/cli/install.go b/cmd/kots/cli/install.go index 6a56f31815..bd9a246e7c 100644 --- a/cmd/kots/cli/install.go +++ b/cmd/kots/cli/install.go @@ -277,16 +277,6 @@ func InstallCmd() *cobra.Command { additionalAnnotations[parts[0]] = parts[1] } - caFiles := []string{} - for _, caFile := range v.GetStringSlice("private-ca") { - // read CA file contents - caData, err := os.ReadFile(caFile) - if err != nil { - return fmt.Errorf("failed to read CA file %s: %w", caFile, err) - } - caFiles = append(caFiles, string(caData)) - } - deployOptions := kotsadmtypes.DeployOptions{ Namespace: namespace, Context: v.GetString("context"), @@ -318,7 +308,8 @@ func InstallCmd() *cobra.Command { RequestedChannelSlug: preferredChannelSlug, AdditionalLabels: additionalLabels, AdditionalAnnotations: additionalAnnotations, - AdditionalTrustedCAs: caFiles, + TrustedCAsConfigmap: v.GetString("private-ca-configmap"), + TrustedCAsConfigmapNS: v.GetString("private-ca-configmap-namespace"), RegistryConfig: *registryConfig, @@ -562,7 +553,8 @@ func InstallCmd() *cobra.Command { cmd.Flags().Bool("exclude-admin-console", false, "set to true to exclude the admin console and only install the application") cmd.Flags().StringArray("additional-annotations", []string{}, "additional annotations to add to kotsadm pods") cmd.Flags().StringArray("additional-labels", []string{}, "additional labels to add to kotsadm pods") - cmd.Flags().StringArray("private-ca", []string{}, "Path to a trusted private CA certificate file") + cmd.Flags().String("private-ca-configmap", "", "the name of a configmap containing private CAs to add to the kotsadm deployment") + cmd.Flags().String("private-ca-configmap-namespace", "", "the namespace of a configmap containing private CAs to add to the kotsadm deployment") registryFlags(cmd.Flags()) diff --git a/pkg/kotsadm/types/deployoptions.go b/pkg/kotsadm/types/deployoptions.go index 89907170ce..cf6b6f2454 100644 --- a/pkg/kotsadm/types/deployoptions.go +++ b/pkg/kotsadm/types/deployoptions.go @@ -59,7 +59,8 @@ type DeployOptions struct { RequestedChannelSlug string AdditionalAnnotations map[string]string AdditionalLabels map[string]string - AdditionalTrustedCAs []string + TrustedCAsConfigmap string + TrustedCAsConfigmapNS string IdentityConfig kotsv1beta1.IdentityConfig IngressConfig kotsv1beta1.IngressConfig From 6cf094109b958bf4a412b052510a8391997c35dd Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 12 Sep 2024 13:54:03 -0400 Subject: [PATCH 03/16] add PrivateCACertNamespace function --- pkg/template/static_context.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/template/static_context.go b/pkg/template/static_context.go index ad2a3fcff1..4fe3abcfbf 100644 --- a/pkg/template/static_context.go +++ b/pkg/template/static_context.go @@ -111,6 +111,7 @@ func (ctx StaticCtx) FuncMap() template.FuncMap { funcMap["Lookup"] = ctx.lookup funcMap["PrivateCACert"] = ctx.privateCACert + funcMap["PrivateCACertNamespace"] = ctx.privateCACertNamespace return funcMap } @@ -683,3 +684,11 @@ func (ctx StaticCtx) privateCACert() string { // return the name of a configmap holding additional CA certificates provided by the end user at install time return os.Getenv("SSL_CERT_CONFIGMAP") } + +func (ctx StaticCtx) privateCACertNamespace() string { + // return the namespace of a configmap holding additional CA certificates provided by the end user at install time + if os.Getenv("SSL_CERT_CONFIGMAP_NAMESPACE") != "" { + return os.Getenv("SSL_CERT_CONFIGMAP_NAMESPACE") + } + return ctx.namespace() +} From 361f4350b520334e7e124d4cd6dd3797f4686ae3 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 12 Sep 2024 15:02:41 -0400 Subject: [PATCH 04/16] specify private CAs configmap via CLI --- cmd/kots/cli/install.go | 2 - pkg/kotsadm/objects/kotsadm_objects.go | 231 ++++++++++++++++--------- pkg/kotsadm/types/deployoptions.go | 1 - pkg/template/static_context.go | 8 - 4 files changed, 151 insertions(+), 91 deletions(-) diff --git a/cmd/kots/cli/install.go b/cmd/kots/cli/install.go index bd9a246e7c..b6515c5e9a 100644 --- a/cmd/kots/cli/install.go +++ b/cmd/kots/cli/install.go @@ -309,7 +309,6 @@ func InstallCmd() *cobra.Command { AdditionalLabels: additionalLabels, AdditionalAnnotations: additionalAnnotations, TrustedCAsConfigmap: v.GetString("private-ca-configmap"), - TrustedCAsConfigmapNS: v.GetString("private-ca-configmap-namespace"), RegistryConfig: *registryConfig, @@ -554,7 +553,6 @@ func InstallCmd() *cobra.Command { cmd.Flags().StringArray("additional-annotations", []string{}, "additional annotations to add to kotsadm pods") cmd.Flags().StringArray("additional-labels", []string{}, "additional labels to add to kotsadm pods") cmd.Flags().String("private-ca-configmap", "", "the name of a configmap containing private CAs to add to the kotsadm deployment") - cmd.Flags().String("private-ca-configmap-namespace", "", "the namespace of a configmap containing private CAs to add to the kotsadm deployment") registryFlags(cmd.Flags()) diff --git a/pkg/kotsadm/objects/kotsadm_objects.go b/pkg/kotsadm/objects/kotsadm_objects.go index f2741bea62..31ea4a93ad 100644 --- a/pkg/kotsadm/objects/kotsadm_objects.go +++ b/pkg/kotsadm/objects/kotsadm_objects.go @@ -344,6 +344,17 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }) } + if deployOptions.TrustedCAsConfigmap != "" { + env = append(env, corev1.EnvVar{ + Name: "SSL_CERT_DIR", + Value: "/certs", + }) + env = append(env, corev1.EnvVar{ + Name: "SSL_CERT_CONFIGMAP", + Value: deployOptions.TrustedCAsConfigmap, + }) + } + podAnnotations := map[string]string{ "backup.velero.io/backup-volumes": "backup", "pre.hook.backup.velero.io/command": `["/backup.sh"]`, @@ -359,6 +370,60 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e podLabels[k] = v } + volumes := []corev1.Volume{ + { + Name: "migrations", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + }, + { + Name: "backup", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + if deployOptions.TrustedCAsConfigmap != "" { + volumes = append(volumes, corev1.Volume{ + Name: "kotsadm-private-cas", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: deployOptions.TrustedCAsConfigmap, + }, + }, + }, + }) + } + + volumeMounts := []corev1.VolumeMount{ + { + Name: "backup", + MountPath: "/backup", + }, + { + Name: "tmp", + MountPath: "/tmp", + }, + } + + if deployOptions.TrustedCAsConfigmap != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "kotsadm-private-cas", + MountPath: "/certs", + }) + } + deployment := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", @@ -385,29 +450,8 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e Affinity: &corev1.Affinity{ NodeAffinity: defaultKOTSNodeAffinity(), }, - SecurityContext: securityContext, - Volumes: []corev1.Volume{ - { - Name: "migrations", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - { - Name: "backup", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "tmp", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, + SecurityContext: securityContext, + Volumes: volumes, ServiceAccountName: "kotsadm", RestartPolicy: corev1.RestartPolicyAlways, ImagePullSecrets: pullSecrets, @@ -631,17 +675,8 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }, }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "backup", - MountPath: "/backup", - }, - { - Name: "tmp", - MountPath: "/tmp", - }, - }, - Env: env, + VolumeMounts: volumeMounts, + Env: env, Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ "cpu": resource.MustParse("1"), @@ -694,6 +729,7 @@ func UpdateKotsadmStatefulSet(existingStatefulset *appsv1.StatefulSet, desiredSt return nil } +// TODO add configmap for additional CAs func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantity) (*appsv1.StatefulSet, error) { securityContext := k8sutil.SecurePodContext(1001, 1001, deployOptions.StrictSecurityContext) if deployOptions.IsOpenShift { @@ -846,6 +882,17 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }) } + if deployOptions.TrustedCAsConfigmap != "" { + env = append(env, corev1.EnvVar{ + Name: "SSL_CERT_DIR", + Value: "/certs", + }) + env = append(env, corev1.EnvVar{ + Name: "SSL_CERT_CONFIGMAP", + Value: deployOptions.TrustedCAsConfigmap, + }) + } + var storageClassName *string if deployOptions.StorageClassName != "" { storageClassName = &deployOptions.StorageClassName @@ -866,6 +913,72 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit podLabels[k] = v } + volumes := []corev1.Volume{ + { + Name: "kotsadmdata", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "kotsadmdata", + }, + }, + }, + { + Name: "migrations", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + }, + { + Name: "backup", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + if deployOptions.TrustedCAsConfigmap != "" { + volumes = append(volumes, corev1.Volume{ + Name: "kotsadm-private-cas", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: deployOptions.TrustedCAsConfigmap, + }, + }, + }, + }) + } + + volumeMounts := []corev1.VolumeMount{ + { + Name: "kotsadmdata", + MountPath: "/kotsadmdata", + }, + { + Name: "backup", + MountPath: "/backup", + }, + { + Name: "tmp", + MountPath: "/tmp", + }, + } + + if deployOptions.TrustedCAsConfigmap != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "kotsadm-private-cas", + MountPath: "/certs", + }) + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", @@ -893,37 +1006,8 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit Affinity: &corev1.Affinity{ NodeAffinity: defaultKOTSNodeAffinity(), }, - SecurityContext: securityContext, - Volumes: []corev1.Volume{ - { - Name: "kotsadmdata", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "kotsadmdata", - }, - }, - }, - { - Name: "migrations", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - { - Name: "backup", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "tmp", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, + SecurityContext: securityContext, + Volumes: volumes, ServiceAccountName: "kotsadm", RestartPolicy: corev1.RestartPolicyAlways, ImagePullSecrets: pullSecrets, @@ -1153,21 +1237,8 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }, }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "kotsadmdata", - MountPath: "/kotsadmdata", - }, - { - Name: "backup", - MountPath: "/backup", - }, - { - Name: "tmp", - MountPath: "/tmp", - }, - }, - Env: env, + VolumeMounts: volumeMounts, + Env: env, Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ "cpu": resource.MustParse("1"), diff --git a/pkg/kotsadm/types/deployoptions.go b/pkg/kotsadm/types/deployoptions.go index cf6b6f2454..6570dfcddd 100644 --- a/pkg/kotsadm/types/deployoptions.go +++ b/pkg/kotsadm/types/deployoptions.go @@ -60,7 +60,6 @@ type DeployOptions struct { AdditionalAnnotations map[string]string AdditionalLabels map[string]string TrustedCAsConfigmap string - TrustedCAsConfigmapNS string IdentityConfig kotsv1beta1.IdentityConfig IngressConfig kotsv1beta1.IngressConfig diff --git a/pkg/template/static_context.go b/pkg/template/static_context.go index 4fe3abcfbf..d31d0f17e3 100644 --- a/pkg/template/static_context.go +++ b/pkg/template/static_context.go @@ -684,11 +684,3 @@ func (ctx StaticCtx) privateCACert() string { // return the name of a configmap holding additional CA certificates provided by the end user at install time return os.Getenv("SSL_CERT_CONFIGMAP") } - -func (ctx StaticCtx) privateCACertNamespace() string { - // return the namespace of a configmap holding additional CA certificates provided by the end user at install time - if os.Getenv("SSL_CERT_CONFIGMAP_NAMESPACE") != "" { - return os.Getenv("SSL_CERT_CONFIGMAP_NAMESPACE") - } - return ctx.namespace() -} From 5255bfde61b621064fe3813307bc064720051e06 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 12 Sep 2024 15:11:25 -0400 Subject: [PATCH 05/16] f --- pkg/template/static_context.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/template/static_context.go b/pkg/template/static_context.go index d31d0f17e3..ad2a3fcff1 100644 --- a/pkg/template/static_context.go +++ b/pkg/template/static_context.go @@ -111,7 +111,6 @@ func (ctx StaticCtx) FuncMap() template.FuncMap { funcMap["Lookup"] = ctx.lookup funcMap["PrivateCACert"] = ctx.privateCACert - funcMap["PrivateCACertNamespace"] = ctx.privateCACertNamespace return funcMap } From 46c427bf7bcef369c3ee00290ae16ea8c1c4e683 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 12 Sep 2024 15:21:51 -0400 Subject: [PATCH 06/16] begin integration test for flag --- .github/workflows/build-test.yaml | 82 +++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index bb5134649d..9452358beb 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4209,6 +4209,87 @@ jobs: api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + validate-custom-cas: + runs-on: ubuntu-20.04 + needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + strategy: + fail-fast: false + matrix: + cluster: [ + {distribution: kind, version: v1.28.0} + ] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create Cluster + id: create-cluster + uses: replicatedhq/replicated-actions/create-cluster@v1 + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + kubernetes-distribution: ${{ matrix.cluster.distribution }} + kubernetes-version: ${{ matrix.cluster.version }} + cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + timeout-minutes: '120' + ttl: 2h + export-kubeconfig: true + + - name: download kots binary + uses: actions/download-artifact@v4 + with: + name: kots + path: bin/ + + - run: chmod +x bin/kots + + - name: create namespace and dockerhub secret + run: | + kubectl create ns "$APP_SLUG" + kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + - name: install yq + run: | + sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + + - name: run the test + run: | + set -e + echo ${{ secrets.GET_SET_CONFIG_LICENSE }} | base64 -d > license.yaml + + echo "test value" > ./ca.crt + kubectl create configmap -n automated-${{ github.run_id }} custom-cas --from-file=ca.crt=./ca.crt + + ./bin/kots \ + install "$APP_SLUG/automated" \ + --license-file license.yaml \ + --no-port-forward \ + --namespace "$APP_SLUG" \ + --shared-password password \ + --kotsadm-registry ttl.sh \ + --kotsadm-namespace automated-${{ github.run_id }} \ + --private-ca-configmap custom-cas \ + --kotsadm-tag 24h + + echo "check if the kotsadm deployment is using the configmap" + kubectl get deployment kotsadm -n automated-${{ github.run_id }} + + - name: Generate support bundle on failure + if: failure() + uses: ./.github/actions/generate-support-bundle + with: + kots-namespace: "$APP_SLUG" + artifact-name: ${{ github.job }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}-support-bundle + + - name: Remove Cluster + id: remove-cluster + uses: replicatedhq/replicated-actions/remove-cluster@v1 + if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + continue-on-error: true + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + validate-pr-tests: runs-on: ubuntu-20.04 @@ -4254,6 +4335,7 @@ jobs: - validate-replicated-sdk - validate-strict-preflight-checks - validate-get-set-config + - validate-custom-cas # cli-only tests - validate-kots-push-images-anonymous steps: From c7b1c51592e2fe0a7f34d0b145a2fb99a6c4e754 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 12 Sep 2024 16:03:06 -0400 Subject: [PATCH 07/16] set env var --- .github/workflows/build-test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 9452358beb..3f62e19a92 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4218,6 +4218,8 @@ jobs: cluster: [ {distribution: kind, version: v1.28.0} ] + env: + APP_SLUG: get-set-config steps: - name: Checkout uses: actions/checkout@v4 From 1e2df58a34971515f7c1763647112f1a72a1955f Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 12 Sep 2024 16:04:39 -0400 Subject: [PATCH 08/16] create ns --- .github/workflows/build-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 3f62e19a92..46d8ee88c5 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4260,6 +4260,7 @@ jobs: echo ${{ secrets.GET_SET_CONFIG_LICENSE }} | base64 -d > license.yaml echo "test value" > ./ca.crt + kubectl create ns automated-${{ github.run_id }} kubectl create configmap -n automated-${{ github.run_id }} custom-cas --from-file=ca.crt=./ca.crt ./bin/kots \ From b74fe5cfab3868e15d2ccd5a57fa87cef878f072 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Thu, 12 Sep 2024 18:48:57 -0400 Subject: [PATCH 09/16] use the right namespace --- .github/workflows/build-test.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 46d8ee88c5..7535fd51d1 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4260,8 +4260,7 @@ jobs: echo ${{ secrets.GET_SET_CONFIG_LICENSE }} | base64 -d > license.yaml echo "test value" > ./ca.crt - kubectl create ns automated-${{ github.run_id }} - kubectl create configmap -n automated-${{ github.run_id }} custom-cas --from-file=ca.crt=./ca.crt + kubectl create configmap -n "$APP_SLUG" custom-cas --from-file=ca.crt=./ca.crt ./bin/kots \ install "$APP_SLUG/automated" \ @@ -4275,7 +4274,7 @@ jobs: --kotsadm-tag 24h echo "check if the kotsadm deployment is using the configmap" - kubectl get deployment kotsadm -n automated-${{ github.run_id }} + kubectl get deployment kotsadm -n "$APP_SLUG" -o yaml - name: Generate support bundle on failure if: failure() From 27f0dad09f34c785b98e575977bdf1da9cbff21a Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 13 Sep 2024 11:25:36 -0400 Subject: [PATCH 10/16] add private-ca-configmap to generate-manifests --- cmd/kots/cli/admin-console-generate-manifests.go | 2 ++ pkg/upstream/admin-console.go | 4 ++++ pkg/upstream/types/types.go | 9 +++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/kots/cli/admin-console-generate-manifests.go b/cmd/kots/cli/admin-console-generate-manifests.go index 899690d94a..bcd97e2638 100644 --- a/cmd/kots/cli/admin-console-generate-manifests.go +++ b/cmd/kots/cli/admin-console-generate-manifests.go @@ -73,6 +73,7 @@ func AdminGenerateManifestsCmd() *cobra.Command { IsOpenShift: isOpenShift, IsGKEAutopilot: isGKEAutopilot, RegistryConfig: registryConfig, + TrustedCAsConfigmap: v.GetString("private-ca-configmap"), } adminConsoleFiles, err := upstream.GenerateAdminConsoleFiles(renderDir, options) if err != nil { @@ -104,6 +105,7 @@ func AdminGenerateManifestsCmd() *cobra.Command { cmd.Flags().String("https-proxy", "", "sets HTTPS_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("no-proxy", "", "sets NO_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("shared-password", "", "shared password to use when deploying the admin console") + cmd.Flags().String("private-ca-configmap", "", "the name of a configmap containing private CAs to add to the kotsadm deployment") cmd.Flags().Bool("with-minio", true, "set to true to include a local minio instance to be used for storage") cmd.Flags().Bool("minimal-rbac", false, "set to true to use the namespaced role and bindings instead of cluster-level permissions") cmd.Flags().StringSlice("additional-namespaces", []string{}, "Comma separate list to specify additional namespace(s) managed by KOTS outside where it is to be deployed. Ignored without with '--minimal-rbac=true'") diff --git a/pkg/upstream/admin-console.go b/pkg/upstream/admin-console.go index 70c9be8918..6483c20f76 100644 --- a/pkg/upstream/admin-console.go +++ b/pkg/upstream/admin-console.go @@ -37,6 +37,7 @@ type UpstreamSettings struct { MigrateToMinioXl bool CurrentMinioImage string AdditionalNamespaces []string + TrustedCAsConfigmap string RegistryConfig kotsadmtypes.RegistryConfig } @@ -63,6 +64,7 @@ func GenerateAdminConsoleFiles(renderDir string, options types.WriteOptions) ([] IsMinimalRBAC: options.IsMinimalRBAC, AdditionalNamespaces: options.AdditionalNamespaces, RegistryConfig: options.RegistryConfig, + TrustedCAsConfigmap: options.TrustedCAsConfigmap, } return generateNewAdminConsoleFiles(settings) } @@ -84,6 +86,7 @@ func GenerateAdminConsoleFiles(renderDir string, options types.WriteOptions) ([] IsMinimalRBAC: options.IsMinimalRBAC, AdditionalNamespaces: options.AdditionalNamespaces, RegistryConfig: options.RegistryConfig, + TrustedCAsConfigmap: options.TrustedCAsConfigmap, } if err := loadUpstreamSettingsFromFiles(settings, renderDir, existingFiles); err != nil { return nil, errors.Wrap(err, "failed to find existing settings") @@ -191,6 +194,7 @@ func generateNewAdminConsoleFiles(settings *UpstreamSettings) ([]types.UpstreamF IsMinimalRBAC: settings.IsMinimalRBAC, AdditionalNamespaces: settings.AdditionalNamespaces, RegistryConfig: settings.RegistryConfig, + TrustedCAsConfigmap: settings.TrustedCAsConfigmap, } if deployOptions.SharedPasswordBcrypt == "" && deployOptions.SharedPassword == "" { diff --git a/pkg/upstream/types/types.go b/pkg/upstream/types/types.go index 169d10a0fc..72c47560bc 100644 --- a/pkg/upstream/types/types.go +++ b/pkg/upstream/types/types.go @@ -79,10 +79,11 @@ type WriteOptions struct { // When true, the channel name in Installation yaml will not be changed. PreserveInstallation bool // Set to true on initial installation when an unencrypted config file is provided - EncryptConfig bool - SharedPassword string - IsOpenShift bool - IsGKEAutopilot bool + EncryptConfig bool + SharedPassword string + IsOpenShift bool + IsGKEAutopilot bool + TrustedCAsConfigmap string RegistryConfig kotsadmtypes.RegistryConfig } From f38d6735920088aa2ed6be32153a4a5bc56c3544 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 13 Sep 2024 11:57:25 -0400 Subject: [PATCH 11/16] check for cert file and env vars in deployment --- .github/workflows/build-test.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 0206f36733..075fd91195 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4276,6 +4276,30 @@ jobs: echo "check if the kotsadm deployment is using the configmap" kubectl get deployment kotsadm -n "$APP_SLUG" -o yaml + echo "exec into the deployment and check for the file and its contents" + if ! kubectl exec -n "$APP_SLUG" deployment/kotsadm -- cat /certs/ca.crt | grep "test value"; then + echo "expected /certs/ca.crt to contain 'test value'" + kubectl exec -n "$APP_SLUG" deployment/kotsadm -- cat /certs/ca.crt + exit 1 + fi + + echo "check that the deployment has an environment variable pointing to the file" + if ! kubectl exec -n "$APP_SLUG" deployment/kotsadm -- env | grep "SSL_CERT_DIR" | grep "/certs"; then + echo "expected env output to contain SSL_CERT_DIR=/certs" + kubectl exec -n "$APP_SLUG" deployment/kotsadm -- env + exit 1 + fi + + echo "check that the deployment has an environment variable with the configmap name" + if ! kubectl exec -n "$APP_SLUG" deployment/kotsadm -- env | grep "SSL_CERT_CONFIGMAP" | grep "custom-cas"; then + echo "expected env output to contain SSL_CERT_CONFIGMAP=custom-cas" + kubectl exec -n "$APP_SLUG" deployment/kotsadm -- env + exit 1 + fi + + ./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password --private-ca-configmap generated-custom-cas > generated-manifests.yaml + cat generated-manifests.yaml + - name: Generate support bundle on failure if: failure() uses: ./.github/actions/generate-support-bundle From 642e01bc06e02277446aadd50b7db30aae7a944c Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 13 Sep 2024 12:18:57 -0400 Subject: [PATCH 12/16] add basic generate-manifests test --- .github/workflows/build-test.yaml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 075fd91195..ccf61bd727 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4273,9 +4273,6 @@ jobs: --private-ca-configmap custom-cas \ --kotsadm-tag 24h - echo "check if the kotsadm deployment is using the configmap" - kubectl get deployment kotsadm -n "$APP_SLUG" -o yaml - echo "exec into the deployment and check for the file and its contents" if ! kubectl exec -n "$APP_SLUG" deployment/kotsadm -- cat /certs/ca.crt | grep "test value"; then echo "expected /certs/ca.crt to contain 'test value'" @@ -4297,8 +4294,18 @@ jobs: exit 1 fi - ./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password --private-ca-configmap generated-custom-cas > generated-manifests.yaml - cat generated-manifests.yaml + ./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password --private-ca-configmap generated-custom-cas + ls ./admin-console + if !cat ./admin-console/kotsadm-deployment.yaml | grep SSL_CERT_CONFIGMAP; then + echo "expected generated kotsadm-deployment.yaml to contain SSL_CERT_CONFIGMAP" + cat ./admin-console/kotsadm-deployment.yaml + exit 1 + fi + if !cat ./admin-console/kotsadm-deployment.yaml | grep generated-custom-cas; then + echo "expected generated kotsadm-deployment.yaml to contain generated-custom-cas" + cat ./admin-console/kotsadm-deployment.yaml + exit 1 + fi - name: Generate support bundle on failure if: failure() From 7080e9093461c5b8d771b1820f253cce9b63bc49 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 13 Sep 2024 12:21:47 -0400 Subject: [PATCH 13/16] manifest namespace --- .github/workflows/build-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index ccf61bd727..643502801e 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4294,7 +4294,7 @@ jobs: exit 1 fi - ./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password --private-ca-configmap generated-custom-cas + ./bin/kots admin-console generate-manifests -n "$APP_SLUG" --shared-password password --private-ca-configmap generated-custom-cas ls ./admin-console if !cat ./admin-console/kotsadm-deployment.yaml | grep SSL_CERT_CONFIGMAP; then echo "expected generated kotsadm-deployment.yaml to contain SSL_CERT_CONFIGMAP" From d01f9957fba7515f6cc5963e34dec27b3d89d05b Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 13 Sep 2024 12:25:48 -0400 Subject: [PATCH 14/16] fix cat --- .github/workflows/build-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 643502801e..9ae7d00c14 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4296,12 +4296,12 @@ jobs: ./bin/kots admin-console generate-manifests -n "$APP_SLUG" --shared-password password --private-ca-configmap generated-custom-cas ls ./admin-console - if !cat ./admin-console/kotsadm-deployment.yaml | grep SSL_CERT_CONFIGMAP; then + if ! cat ./admin-console/kotsadm-deployment.yaml | grep SSL_CERT_CONFIGMAP; then echo "expected generated kotsadm-deployment.yaml to contain SSL_CERT_CONFIGMAP" cat ./admin-console/kotsadm-deployment.yaml exit 1 fi - if !cat ./admin-console/kotsadm-deployment.yaml | grep generated-custom-cas; then + if ! cat ./admin-console/kotsadm-deployment.yaml | grep generated-custom-cas; then echo "expected generated kotsadm-deployment.yaml to contain generated-custom-cas" cat ./admin-console/kotsadm-deployment.yaml exit 1 From 249371c5c74aa3c8c140c5934faa8ab6d7c2db29 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 13 Sep 2024 12:26:36 -0400 Subject: [PATCH 15/16] remove cat --- .github/workflows/build-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 9ae7d00c14..1f29e4230f 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4296,12 +4296,12 @@ jobs: ./bin/kots admin-console generate-manifests -n "$APP_SLUG" --shared-password password --private-ca-configmap generated-custom-cas ls ./admin-console - if ! cat ./admin-console/kotsadm-deployment.yaml | grep SSL_CERT_CONFIGMAP; then + if ! grep SSL_CERT_CONFIGMAP < ./admin-console/kotsadm-deployment.yaml; then echo "expected generated kotsadm-deployment.yaml to contain SSL_CERT_CONFIGMAP" cat ./admin-console/kotsadm-deployment.yaml exit 1 fi - if ! cat ./admin-console/kotsadm-deployment.yaml | grep generated-custom-cas; then + if ! grep generated-custom-cas < ./admin-console/kotsadm-deployment.yaml; then echo "expected generated kotsadm-deployment.yaml to contain generated-custom-cas" cat ./admin-console/kotsadm-deployment.yaml exit 1 From cb48ec7515d4c083008dcdc28e017992b8bc7126 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 13 Sep 2024 13:39:01 -0400 Subject: [PATCH 16/16] rename TrustedCAsConfigmap to PrivateCAsConfigmap --- .../cli/admin-console-generate-manifests.go | 2 +- cmd/kots/cli/install.go | 2 +- pkg/kotsadm/objects/kotsadm_objects.go | 20 +++++++++---------- pkg/kotsadm/types/deployoptions.go | 2 +- pkg/upstream/admin-console.go | 8 ++++---- pkg/upstream/types/types.go | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/kots/cli/admin-console-generate-manifests.go b/cmd/kots/cli/admin-console-generate-manifests.go index bcd97e2638..ed4bc5dff0 100644 --- a/cmd/kots/cli/admin-console-generate-manifests.go +++ b/cmd/kots/cli/admin-console-generate-manifests.go @@ -73,7 +73,7 @@ func AdminGenerateManifestsCmd() *cobra.Command { IsOpenShift: isOpenShift, IsGKEAutopilot: isGKEAutopilot, RegistryConfig: registryConfig, - TrustedCAsConfigmap: v.GetString("private-ca-configmap"), + PrivateCAsConfigmap: v.GetString("private-ca-configmap"), } adminConsoleFiles, err := upstream.GenerateAdminConsoleFiles(renderDir, options) if err != nil { diff --git a/cmd/kots/cli/install.go b/cmd/kots/cli/install.go index b6515c5e9a..8690c056f5 100644 --- a/cmd/kots/cli/install.go +++ b/cmd/kots/cli/install.go @@ -308,7 +308,7 @@ func InstallCmd() *cobra.Command { RequestedChannelSlug: preferredChannelSlug, AdditionalLabels: additionalLabels, AdditionalAnnotations: additionalAnnotations, - TrustedCAsConfigmap: v.GetString("private-ca-configmap"), + PrivateCAsConfigmap: v.GetString("private-ca-configmap"), RegistryConfig: *registryConfig, diff --git a/pkg/kotsadm/objects/kotsadm_objects.go b/pkg/kotsadm/objects/kotsadm_objects.go index 31ea4a93ad..a1e7007e77 100644 --- a/pkg/kotsadm/objects/kotsadm_objects.go +++ b/pkg/kotsadm/objects/kotsadm_objects.go @@ -344,14 +344,14 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }) } - if deployOptions.TrustedCAsConfigmap != "" { + if deployOptions.PrivateCAsConfigmap != "" { env = append(env, corev1.EnvVar{ Name: "SSL_CERT_DIR", Value: "/certs", }) env = append(env, corev1.EnvVar{ Name: "SSL_CERT_CONFIGMAP", - Value: deployOptions.TrustedCAsConfigmap, + Value: deployOptions.PrivateCAsConfigmap, }) } @@ -393,13 +393,13 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }, } - if deployOptions.TrustedCAsConfigmap != "" { + if deployOptions.PrivateCAsConfigmap != "" { volumes = append(volumes, corev1.Volume{ Name: "kotsadm-private-cas", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: deployOptions.TrustedCAsConfigmap, + Name: deployOptions.PrivateCAsConfigmap, }, }, }, @@ -417,7 +417,7 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }, } - if deployOptions.TrustedCAsConfigmap != "" { + if deployOptions.PrivateCAsConfigmap != "" { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "kotsadm-private-cas", MountPath: "/certs", @@ -882,14 +882,14 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }) } - if deployOptions.TrustedCAsConfigmap != "" { + if deployOptions.PrivateCAsConfigmap != "" { env = append(env, corev1.EnvVar{ Name: "SSL_CERT_DIR", Value: "/certs", }) env = append(env, corev1.EnvVar{ Name: "SSL_CERT_CONFIGMAP", - Value: deployOptions.TrustedCAsConfigmap, + Value: deployOptions.PrivateCAsConfigmap, }) } @@ -944,13 +944,13 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }, } - if deployOptions.TrustedCAsConfigmap != "" { + if deployOptions.PrivateCAsConfigmap != "" { volumes = append(volumes, corev1.Volume{ Name: "kotsadm-private-cas", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: deployOptions.TrustedCAsConfigmap, + Name: deployOptions.PrivateCAsConfigmap, }, }, }, @@ -972,7 +972,7 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }, } - if deployOptions.TrustedCAsConfigmap != "" { + if deployOptions.PrivateCAsConfigmap != "" { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "kotsadm-private-cas", MountPath: "/certs", diff --git a/pkg/kotsadm/types/deployoptions.go b/pkg/kotsadm/types/deployoptions.go index 6570dfcddd..bdb58a877c 100644 --- a/pkg/kotsadm/types/deployoptions.go +++ b/pkg/kotsadm/types/deployoptions.go @@ -59,7 +59,7 @@ type DeployOptions struct { RequestedChannelSlug string AdditionalAnnotations map[string]string AdditionalLabels map[string]string - TrustedCAsConfigmap string + PrivateCAsConfigmap string IdentityConfig kotsv1beta1.IdentityConfig IngressConfig kotsv1beta1.IngressConfig diff --git a/pkg/upstream/admin-console.go b/pkg/upstream/admin-console.go index 6483c20f76..bad27dc38d 100644 --- a/pkg/upstream/admin-console.go +++ b/pkg/upstream/admin-console.go @@ -37,7 +37,7 @@ type UpstreamSettings struct { MigrateToMinioXl bool CurrentMinioImage string AdditionalNamespaces []string - TrustedCAsConfigmap string + PrivateCAsConfigmap string RegistryConfig kotsadmtypes.RegistryConfig } @@ -64,7 +64,7 @@ func GenerateAdminConsoleFiles(renderDir string, options types.WriteOptions) ([] IsMinimalRBAC: options.IsMinimalRBAC, AdditionalNamespaces: options.AdditionalNamespaces, RegistryConfig: options.RegistryConfig, - TrustedCAsConfigmap: options.TrustedCAsConfigmap, + PrivateCAsConfigmap: options.PrivateCAsConfigmap, } return generateNewAdminConsoleFiles(settings) } @@ -86,7 +86,7 @@ func GenerateAdminConsoleFiles(renderDir string, options types.WriteOptions) ([] IsMinimalRBAC: options.IsMinimalRBAC, AdditionalNamespaces: options.AdditionalNamespaces, RegistryConfig: options.RegistryConfig, - TrustedCAsConfigmap: options.TrustedCAsConfigmap, + PrivateCAsConfigmap: options.PrivateCAsConfigmap, } if err := loadUpstreamSettingsFromFiles(settings, renderDir, existingFiles); err != nil { return nil, errors.Wrap(err, "failed to find existing settings") @@ -194,7 +194,7 @@ func generateNewAdminConsoleFiles(settings *UpstreamSettings) ([]types.UpstreamF IsMinimalRBAC: settings.IsMinimalRBAC, AdditionalNamespaces: settings.AdditionalNamespaces, RegistryConfig: settings.RegistryConfig, - TrustedCAsConfigmap: settings.TrustedCAsConfigmap, + PrivateCAsConfigmap: settings.PrivateCAsConfigmap, } if deployOptions.SharedPasswordBcrypt == "" && deployOptions.SharedPassword == "" { diff --git a/pkg/upstream/types/types.go b/pkg/upstream/types/types.go index 72c47560bc..9b5c39589a 100644 --- a/pkg/upstream/types/types.go +++ b/pkg/upstream/types/types.go @@ -83,7 +83,7 @@ type WriteOptions struct { SharedPassword string IsOpenShift bool IsGKEAutopilot bool - TrustedCAsConfigmap string + PrivateCAsConfigmap string RegistryConfig kotsadmtypes.RegistryConfig }