diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 248875383f..1f29e4230f 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -4209,6 +4209,120 @@ jobs: api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + validate-custom-cas: + runs-on: ubuntu-20.04 + needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + strategy: + fail-fast: false + matrix: + cluster: [ + {distribution: kind, version: v1.28.0} + ] + env: + APP_SLUG: get-set-config + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create Cluster + id: create-cluster + uses: replicatedhq/replicated-actions/create-cluster@v1 + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + kubernetes-distribution: ${{ matrix.cluster.distribution }} + kubernetes-version: ${{ matrix.cluster.version }} + cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + timeout-minutes: '120' + ttl: 2h + export-kubeconfig: true + + - name: download kots binary + uses: actions/download-artifact@v4 + with: + name: kots + path: bin/ + + - run: chmod +x bin/kots + + - name: create namespace and dockerhub secret + run: | + kubectl create ns "$APP_SLUG" + kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + - name: install yq + run: | + sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + + - name: run the test + run: | + set -e + echo ${{ secrets.GET_SET_CONFIG_LICENSE }} | base64 -d > license.yaml + + echo "test value" > ./ca.crt + kubectl create configmap -n "$APP_SLUG" custom-cas --from-file=ca.crt=./ca.crt + + ./bin/kots \ + install "$APP_SLUG/automated" \ + --license-file license.yaml \ + --no-port-forward \ + --namespace "$APP_SLUG" \ + --shared-password password \ + --kotsadm-registry ttl.sh \ + --kotsadm-namespace automated-${{ github.run_id }} \ + --private-ca-configmap custom-cas \ + --kotsadm-tag 24h + + echo "exec into the deployment and check for the file and its contents" + if ! kubectl exec -n "$APP_SLUG" deployment/kotsadm -- cat /certs/ca.crt | grep "test value"; then + echo "expected /certs/ca.crt to contain 'test value'" + kubectl exec -n "$APP_SLUG" deployment/kotsadm -- cat /certs/ca.crt + exit 1 + fi + + echo "check that the deployment has an environment variable pointing to the file" + if ! kubectl exec -n "$APP_SLUG" deployment/kotsadm -- env | grep "SSL_CERT_DIR" | grep "/certs"; then + echo "expected env output to contain SSL_CERT_DIR=/certs" + kubectl exec -n "$APP_SLUG" deployment/kotsadm -- env + exit 1 + fi + + echo "check that the deployment has an environment variable with the configmap name" + if ! kubectl exec -n "$APP_SLUG" deployment/kotsadm -- env | grep "SSL_CERT_CONFIGMAP" | grep "custom-cas"; then + echo "expected env output to contain SSL_CERT_CONFIGMAP=custom-cas" + kubectl exec -n "$APP_SLUG" deployment/kotsadm -- env + exit 1 + fi + + ./bin/kots admin-console generate-manifests -n "$APP_SLUG" --shared-password password --private-ca-configmap generated-custom-cas + ls ./admin-console + if ! grep SSL_CERT_CONFIGMAP < ./admin-console/kotsadm-deployment.yaml; then + echo "expected generated kotsadm-deployment.yaml to contain SSL_CERT_CONFIGMAP" + cat ./admin-console/kotsadm-deployment.yaml + exit 1 + fi + if ! grep generated-custom-cas < ./admin-console/kotsadm-deployment.yaml; then + echo "expected generated kotsadm-deployment.yaml to contain generated-custom-cas" + cat ./admin-console/kotsadm-deployment.yaml + exit 1 + fi + + - name: Generate support bundle on failure + if: failure() + uses: ./.github/actions/generate-support-bundle + with: + kots-namespace: "$APP_SLUG" + artifact-name: ${{ github.job }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}-support-bundle + + - name: Remove Cluster + id: remove-cluster + uses: replicatedhq/replicated-actions/remove-cluster@v1 + if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + continue-on-error: true + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + validate-pr-tests: runs-on: ubuntu-20.04 @@ -4254,6 +4368,7 @@ jobs: - validate-replicated-sdk - validate-strict-preflight-checks - validate-get-set-config + - validate-custom-cas # cli-only tests - validate-kots-push-images-anonymous steps: diff --git a/cmd/kots/cli/admin-console-generate-manifests.go b/cmd/kots/cli/admin-console-generate-manifests.go index 899690d94a..ed4bc5dff0 100644 --- a/cmd/kots/cli/admin-console-generate-manifests.go +++ b/cmd/kots/cli/admin-console-generate-manifests.go @@ -73,6 +73,7 @@ func AdminGenerateManifestsCmd() *cobra.Command { IsOpenShift: isOpenShift, IsGKEAutopilot: isGKEAutopilot, RegistryConfig: registryConfig, + PrivateCAsConfigmap: v.GetString("private-ca-configmap"), } adminConsoleFiles, err := upstream.GenerateAdminConsoleFiles(renderDir, options) if err != nil { @@ -104,6 +105,7 @@ func AdminGenerateManifestsCmd() *cobra.Command { cmd.Flags().String("https-proxy", "", "sets HTTPS_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("no-proxy", "", "sets NO_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("shared-password", "", "shared password to use when deploying the admin console") + cmd.Flags().String("private-ca-configmap", "", "the name of a configmap containing private CAs to add to the kotsadm deployment") cmd.Flags().Bool("with-minio", true, "set to true to include a local minio instance to be used for storage") cmd.Flags().Bool("minimal-rbac", false, "set to true to use the namespaced role and bindings instead of cluster-level permissions") cmd.Flags().StringSlice("additional-namespaces", []string{}, "Comma separate list to specify additional namespace(s) managed by KOTS outside where it is to be deployed. Ignored without with '--minimal-rbac=true'") diff --git a/cmd/kots/cli/install.go b/cmd/kots/cli/install.go index d4f87d5ffa..8690c056f5 100644 --- a/cmd/kots/cli/install.go +++ b/cmd/kots/cli/install.go @@ -308,6 +308,7 @@ func InstallCmd() *cobra.Command { RequestedChannelSlug: preferredChannelSlug, AdditionalLabels: additionalLabels, AdditionalAnnotations: additionalAnnotations, + PrivateCAsConfigmap: v.GetString("private-ca-configmap"), RegistryConfig: *registryConfig, @@ -551,6 +552,7 @@ func InstallCmd() *cobra.Command { cmd.Flags().Bool("exclude-admin-console", false, "set to true to exclude the admin console and only install the application") cmd.Flags().StringArray("additional-annotations", []string{}, "additional annotations to add to kotsadm pods") cmd.Flags().StringArray("additional-labels", []string{}, "additional labels to add to kotsadm pods") + cmd.Flags().String("private-ca-configmap", "", "the name of a configmap containing private CAs to add to the kotsadm deployment") registryFlags(cmd.Flags()) diff --git a/pkg/kotsadm/objects/kotsadm_objects.go b/pkg/kotsadm/objects/kotsadm_objects.go index f2741bea62..a1e7007e77 100644 --- a/pkg/kotsadm/objects/kotsadm_objects.go +++ b/pkg/kotsadm/objects/kotsadm_objects.go @@ -344,6 +344,17 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }) } + if deployOptions.PrivateCAsConfigmap != "" { + env = append(env, corev1.EnvVar{ + Name: "SSL_CERT_DIR", + Value: "/certs", + }) + env = append(env, corev1.EnvVar{ + Name: "SSL_CERT_CONFIGMAP", + Value: deployOptions.PrivateCAsConfigmap, + }) + } + podAnnotations := map[string]string{ "backup.velero.io/backup-volumes": "backup", "pre.hook.backup.velero.io/command": `["/backup.sh"]`, @@ -359,6 +370,60 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e podLabels[k] = v } + volumes := []corev1.Volume{ + { + Name: "migrations", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + }, + { + Name: "backup", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + if deployOptions.PrivateCAsConfigmap != "" { + volumes = append(volumes, corev1.Volume{ + Name: "kotsadm-private-cas", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: deployOptions.PrivateCAsConfigmap, + }, + }, + }, + }) + } + + volumeMounts := []corev1.VolumeMount{ + { + Name: "backup", + MountPath: "/backup", + }, + { + Name: "tmp", + MountPath: "/tmp", + }, + } + + if deployOptions.PrivateCAsConfigmap != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "kotsadm-private-cas", + MountPath: "/certs", + }) + } + deployment := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", @@ -385,29 +450,8 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e Affinity: &corev1.Affinity{ NodeAffinity: defaultKOTSNodeAffinity(), }, - SecurityContext: securityContext, - Volumes: []corev1.Volume{ - { - Name: "migrations", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - { - Name: "backup", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "tmp", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, + SecurityContext: securityContext, + Volumes: volumes, ServiceAccountName: "kotsadm", RestartPolicy: corev1.RestartPolicyAlways, ImagePullSecrets: pullSecrets, @@ -631,17 +675,8 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }, }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "backup", - MountPath: "/backup", - }, - { - Name: "tmp", - MountPath: "/tmp", - }, - }, - Env: env, + VolumeMounts: volumeMounts, + Env: env, Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ "cpu": resource.MustParse("1"), @@ -694,6 +729,7 @@ func UpdateKotsadmStatefulSet(existingStatefulset *appsv1.StatefulSet, desiredSt return nil } +// TODO add configmap for additional CAs func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantity) (*appsv1.StatefulSet, error) { securityContext := k8sutil.SecurePodContext(1001, 1001, deployOptions.StrictSecurityContext) if deployOptions.IsOpenShift { @@ -846,6 +882,17 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }) } + if deployOptions.PrivateCAsConfigmap != "" { + env = append(env, corev1.EnvVar{ + Name: "SSL_CERT_DIR", + Value: "/certs", + }) + env = append(env, corev1.EnvVar{ + Name: "SSL_CERT_CONFIGMAP", + Value: deployOptions.PrivateCAsConfigmap, + }) + } + var storageClassName *string if deployOptions.StorageClassName != "" { storageClassName = &deployOptions.StorageClassName @@ -866,6 +913,72 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit podLabels[k] = v } + volumes := []corev1.Volume{ + { + Name: "kotsadmdata", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "kotsadmdata", + }, + }, + }, + { + Name: "migrations", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + }, + { + Name: "backup", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + if deployOptions.PrivateCAsConfigmap != "" { + volumes = append(volumes, corev1.Volume{ + Name: "kotsadm-private-cas", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: deployOptions.PrivateCAsConfigmap, + }, + }, + }, + }) + } + + volumeMounts := []corev1.VolumeMount{ + { + Name: "kotsadmdata", + MountPath: "/kotsadmdata", + }, + { + Name: "backup", + MountPath: "/backup", + }, + { + Name: "tmp", + MountPath: "/tmp", + }, + } + + if deployOptions.PrivateCAsConfigmap != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "kotsadm-private-cas", + MountPath: "/certs", + }) + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", @@ -893,37 +1006,8 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit Affinity: &corev1.Affinity{ NodeAffinity: defaultKOTSNodeAffinity(), }, - SecurityContext: securityContext, - Volumes: []corev1.Volume{ - { - Name: "kotsadmdata", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "kotsadmdata", - }, - }, - }, - { - Name: "migrations", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - { - Name: "backup", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "tmp", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, + SecurityContext: securityContext, + Volumes: volumes, ServiceAccountName: "kotsadm", RestartPolicy: corev1.RestartPolicyAlways, ImagePullSecrets: pullSecrets, @@ -1153,21 +1237,8 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }, }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "kotsadmdata", - MountPath: "/kotsadmdata", - }, - { - Name: "backup", - MountPath: "/backup", - }, - { - Name: "tmp", - MountPath: "/tmp", - }, - }, - Env: env, + VolumeMounts: volumeMounts, + Env: env, Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ "cpu": resource.MustParse("1"), diff --git a/pkg/kotsadm/types/deployoptions.go b/pkg/kotsadm/types/deployoptions.go index 35a061d600..bdb58a877c 100644 --- a/pkg/kotsadm/types/deployoptions.go +++ b/pkg/kotsadm/types/deployoptions.go @@ -59,6 +59,7 @@ type DeployOptions struct { RequestedChannelSlug string AdditionalAnnotations map[string]string AdditionalLabels map[string]string + PrivateCAsConfigmap string IdentityConfig kotsv1beta1.IdentityConfig IngressConfig kotsv1beta1.IngressConfig diff --git a/pkg/upstream/admin-console.go b/pkg/upstream/admin-console.go index 70c9be8918..bad27dc38d 100644 --- a/pkg/upstream/admin-console.go +++ b/pkg/upstream/admin-console.go @@ -37,6 +37,7 @@ type UpstreamSettings struct { MigrateToMinioXl bool CurrentMinioImage string AdditionalNamespaces []string + PrivateCAsConfigmap string RegistryConfig kotsadmtypes.RegistryConfig } @@ -63,6 +64,7 @@ func GenerateAdminConsoleFiles(renderDir string, options types.WriteOptions) ([] IsMinimalRBAC: options.IsMinimalRBAC, AdditionalNamespaces: options.AdditionalNamespaces, RegistryConfig: options.RegistryConfig, + PrivateCAsConfigmap: options.PrivateCAsConfigmap, } return generateNewAdminConsoleFiles(settings) } @@ -84,6 +86,7 @@ func GenerateAdminConsoleFiles(renderDir string, options types.WriteOptions) ([] IsMinimalRBAC: options.IsMinimalRBAC, AdditionalNamespaces: options.AdditionalNamespaces, RegistryConfig: options.RegistryConfig, + PrivateCAsConfigmap: options.PrivateCAsConfigmap, } if err := loadUpstreamSettingsFromFiles(settings, renderDir, existingFiles); err != nil { return nil, errors.Wrap(err, "failed to find existing settings") @@ -191,6 +194,7 @@ func generateNewAdminConsoleFiles(settings *UpstreamSettings) ([]types.UpstreamF IsMinimalRBAC: settings.IsMinimalRBAC, AdditionalNamespaces: settings.AdditionalNamespaces, RegistryConfig: settings.RegistryConfig, + PrivateCAsConfigmap: settings.PrivateCAsConfigmap, } if deployOptions.SharedPasswordBcrypt == "" && deployOptions.SharedPassword == "" { diff --git a/pkg/upstream/types/types.go b/pkg/upstream/types/types.go index 169d10a0fc..9b5c39589a 100644 --- a/pkg/upstream/types/types.go +++ b/pkg/upstream/types/types.go @@ -79,10 +79,11 @@ type WriteOptions struct { // When true, the channel name in Installation yaml will not be changed. PreserveInstallation bool // Set to true on initial installation when an unencrypted config file is provided - EncryptConfig bool - SharedPassword string - IsOpenShift bool - IsGKEAutopilot bool + EncryptConfig bool + SharedPassword string + IsOpenShift bool + IsGKEAutopilot bool + PrivateCAsConfigmap string RegistryConfig kotsadmtypes.RegistryConfig }