From 2c0d10eac8f85db959da3eae89fcb0e63640b044 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Wed, 26 Jun 2024 20:25:57 +0000 Subject: [PATCH 1/5] Allow specifying a custom storage class --- cmd/kots/cli/admin-console-generate-manifests.go | 2 ++ cmd/kots/cli/install.go | 4 ++-- pkg/kotsadm/objects/kotsadm_objects.go | 6 ++++++ pkg/kotsadm/objects/minio_objects.go | 6 ++++++ pkg/kotsadm/objects/rqlite_objects.go | 6 ++++++ pkg/kotsadm/types/deployoptions.go | 1 + pkg/upstream/admin-console.go | 4 ++++ pkg/upstream/types/types.go | 1 + 8 files changed, 28 insertions(+), 2 deletions(-) diff --git a/cmd/kots/cli/admin-console-generate-manifests.go b/cmd/kots/cli/admin-console-generate-manifests.go index feeb8aab42..899690d94a 100644 --- a/cmd/kots/cli/admin-console-generate-manifests.go +++ b/cmd/kots/cli/admin-console-generate-manifests.go @@ -61,6 +61,7 @@ func AdminGenerateManifestsCmd() *cobra.Command { options := upstreamtypes.WriteOptions{ Namespace: namespace, SharedPassword: v.GetString("shared-password"), + StorageClassName: v.GetString("storage-class"), HTTPProxyEnvValue: v.GetString("http-proxy"), HTTPSProxyEnvValue: v.GetString("https-proxy"), NoProxyEnvValue: v.GetString("no-proxy"), @@ -98,6 +99,7 @@ func AdminGenerateManifestsCmd() *cobra.Command { } cmd.Flags().String("rootdir", ".", "root directory that will be used to write the yaml to") + cmd.Flags().String("storage-class", "", "sets the storage class to use for the KOTS Admin Console components (if unset, the default storage class will be used)") cmd.Flags().String("http-proxy", "", "sets HTTP_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("https-proxy", "", "sets HTTPS_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("no-proxy", "", "sets NO_PROXY environment variable in all KOTS Admin Console components") diff --git a/cmd/kots/cli/install.go b/cmd/kots/cli/install.go index edf8d68371..51488ecb0d 100644 --- a/cmd/kots/cli/install.go +++ b/cmd/kots/cli/install.go @@ -261,6 +261,7 @@ func InstallCmd() *cobra.Command { Airgap: isAirgap, ProgressWriter: os.Stdout, Timeout: time.Minute * 2, + StorageClassName: v.GetString("storage-class"), HTTPProxyEnvValue: v.GetString("http-proxy"), HTTPSProxyEnvValue: v.GetString("https-proxy"), NoProxyEnvValue: v.GetString("no-proxy"), @@ -504,6 +505,7 @@ func InstallCmd() *cobra.Command { cmd.Flags().Int("port", 8800, "local port to listen on when port forwarding is enabled") cmd.Flags().String("wait-duration", "2m", "timeout to be used while waiting for individual components to be ready. must be in Go duration format (eg: 10s, 2m)") cmd.Flags().String("preflights-wait-duration", "15m", "timeout to be used while waiting for preflights to complete. must be in Go duration format (eg: 10s, 2m)") + cmd.Flags().String("storage-class", "", "sets the storage class to use for the KOTS Admin Console components (if unset, the default storage class will be used)") cmd.Flags().String("http-proxy", "", "sets HTTP_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("https-proxy", "", "sets HTTPS_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("no-proxy", "", "sets NO_PROXY environment variable in all KOTS Admin Console components") @@ -518,8 +520,6 @@ func InstallCmd() *cobra.Command { cmd.Flags().String("app-version-label", "", "the application version label to install. if not specified, the latest version will be installed") cmd.Flags().Bool("exclude-admin-console", false, "set to true to exclude the admin console and only install the application") - cmd.Flags().String("repo", "", "repo uri to use when installing a helm chart") - registryFlags(cmd.Flags()) // the following group of flags are experiemental and can be used to pull and push images during install time diff --git a/pkg/kotsadm/objects/kotsadm_objects.go b/pkg/kotsadm/objects/kotsadm_objects.go index a92397d08d..f091d31f02 100644 --- a/pkg/kotsadm/objects/kotsadm_objects.go +++ b/pkg/kotsadm/objects/kotsadm_objects.go @@ -836,6 +836,11 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }) } + var storageClassName *string + if deployOptions.StorageClassName != "" { + storageClassName = &deployOptions.StorageClassName + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", @@ -1165,6 +1170,7 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit Labels: types.GetKotsadmLabels(), }, Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: storageClassName, AccessModes: []corev1.PersistentVolumeAccessMode{ corev1.ReadWriteOnce, }, diff --git a/pkg/kotsadm/objects/minio_objects.go b/pkg/kotsadm/objects/minio_objects.go index 868a9c8147..9e8f4bea71 100644 --- a/pkg/kotsadm/objects/minio_objects.go +++ b/pkg/kotsadm/objects/minio_objects.go @@ -72,6 +72,11 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity) }, } + var storageClassName *string + if deployOptions.StorageClassName != "" { + storageClassName = &deployOptions.StorageClassName + } + initContainers := []corev1.Container{} if deployOptions.MigrateToMinioXl { initContainers = append(initContainers, migrateToMinioXlInitContainers(deployOptions, resourceRequirements)...) @@ -103,6 +108,7 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity) Labels: types.GetKotsadmLabels(), }, Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: storageClassName, AccessModes: []corev1.PersistentVolumeAccessMode{ corev1.ReadWriteOnce, }, diff --git a/pkg/kotsadm/objects/rqlite_objects.go b/pkg/kotsadm/objects/rqlite_objects.go index 4d77241064..80c3359ccb 100644 --- a/pkg/kotsadm/objects/rqlite_objects.go +++ b/pkg/kotsadm/objects/rqlite_objects.go @@ -45,6 +45,11 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity memoryRequest, memoryLimit = "512Mi", "1Gi" } + var storageClassName *string + if deployOptions.StorageClassName != "" { + storageClassName = &deployOptions.StorageClassName + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", @@ -70,6 +75,7 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity Labels: types.GetKotsadmLabels(), }, Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: storageClassName, AccessModes: []corev1.PersistentVolumeAccessMode{ corev1.ReadWriteOnce, }, diff --git a/pkg/kotsadm/types/deployoptions.go b/pkg/kotsadm/types/deployoptions.go index 3c2ed04aa7..5db3215c06 100644 --- a/pkg/kotsadm/types/deployoptions.go +++ b/pkg/kotsadm/types/deployoptions.go @@ -37,6 +37,7 @@ type DeployOptions struct { CurrentMinioImage string Timeout time.Duration PreflightsTimeout time.Duration + StorageClassName string HTTPProxyEnvValue string HTTPSProxyEnvValue string NoProxyEnvValue string diff --git a/pkg/upstream/admin-console.go b/pkg/upstream/admin-console.go index 77732acecc..70c9be8918 100644 --- a/pkg/upstream/admin-console.go +++ b/pkg/upstream/admin-console.go @@ -25,6 +25,7 @@ type UpstreamSettings struct { JWT string RqlitePassword string APIEncryptionKey string + StorageClassName string HTTPProxyEnvValue string HTTPSProxyEnvValue string NoProxyEnvValue string @@ -50,6 +51,7 @@ func GenerateAdminConsoleFiles(renderDir string, options types.WriteOptions) ([] Namespace: options.Namespace, SharedPassword: options.SharedPassword, AutoCreateClusterToken: uuid.New().String(), + StorageClassName: options.StorageClassName, HTTPProxyEnvValue: options.HTTPProxyEnvValue, HTTPSProxyEnvValue: options.HTTPSProxyEnvValue, NoProxyEnvValue: options.NoProxyEnvValue, @@ -73,6 +75,7 @@ func GenerateAdminConsoleFiles(renderDir string, options types.WriteOptions) ([] settings := &UpstreamSettings{ Namespace: options.Namespace, AutoCreateClusterToken: uuid.New().String(), + StorageClassName: options.StorageClassName, IsOpenShift: options.IsOpenShift, IsGKEAutopilot: options.IsGKEAutopilot, IncludeMinio: options.IncludeMinio, @@ -175,6 +178,7 @@ func generateNewAdminConsoleFiles(settings *UpstreamSettings) ([]types.UpstreamF RqlitePassword: settings.RqlitePassword, APIEncryptionKey: settings.APIEncryptionKey, AutoCreateClusterToken: settings.AutoCreateClusterToken, + StorageClassName: settings.StorageClassName, HTTPProxyEnvValue: settings.HTTPProxyEnvValue, HTTPSProxyEnvValue: settings.HTTPSProxyEnvValue, NoProxyEnvValue: settings.NoProxyEnvValue, diff --git a/pkg/upstream/types/types.go b/pkg/upstream/types/types.go index e7548cb19f..a6d72d7657 100644 --- a/pkg/upstream/types/types.go +++ b/pkg/upstream/types/types.go @@ -65,6 +65,7 @@ type WriteOptions struct { IncludeMinio bool MigrateToMinioXl bool CurrentMinioImage string + StorageClassName string HTTPProxyEnvValue string HTTPSProxyEnvValue string NoProxyEnvValue string From b914908ef100275c7f660c71d325973b9b42f597 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Wed, 26 Jun 2024 20:48:28 +0000 Subject: [PATCH 2/5] re-enable eks 1.30 and use custom storage class --- .github/actions/cmx-versions/dist/index.js | 4 +--- .github/actions/cmx-versions/index.js | 4 +--- .github/actions/kots-e2e/action.yml | 1 + cmd/kots/cli/pull.go | 2 ++ e2e/Makefile | 2 ++ e2e/e2e_test.go | 4 +++- e2e/kots/kots.go | 7 ++++++- pkg/pull/pull.go | 2 ++ 8 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/actions/cmx-versions/dist/index.js b/.github/actions/cmx-versions/dist/index.js index 8bfc9f6e47..24d351b8be 100644 --- a/.github/actions/cmx-versions/dist/index.js +++ b/.github/actions/cmx-versions/dist/index.js @@ -7656,9 +7656,7 @@ async function getClusterVersions() { latest_minor_versions: true, }, eks: { - // latest_version: true, - // TODO: re-enable latest_version once we have compatibility with 1.30 - versions: new Set(["1.29"]), + latest_version: true, instance_type: "m7g.large" // arm64 }, openshift: { diff --git a/.github/actions/cmx-versions/index.js b/.github/actions/cmx-versions/index.js index 39a950bdc6..c2ff291046 100644 --- a/.github/actions/cmx-versions/index.js +++ b/.github/actions/cmx-versions/index.js @@ -36,9 +36,7 @@ async function getClusterVersions() { latest_minor_versions: true, }, eks: { - // latest_version: true, - // TODO: re-enable latest_version once we have compatibility with 1.30 - versions: new Set(["1.29"]), + latest_version: true, instance_type: "m7g.large" // arm64 }, openshift: { diff --git a/.github/actions/kots-e2e/action.yml b/.github/actions/kots-e2e/action.yml index 33e185b0b7..465fc7ee92 100644 --- a/.github/actions/kots-e2e/action.yml +++ b/.github/actions/kots-e2e/action.yml @@ -110,6 +110,7 @@ runs: KOTS_DOCKERHUB_PASSWORD=${{ inputs.kots-dockerhub-password }} \ AIRGAP=${{ inputs.kots-airgap }} \ IS_OPENSHIFT=${{ inputs.k8s-distribution == 'openshift' && 'true' || 'false' }} \ + IS_EKS=${{ inputs.k8s-distribution == 'eks' && 'true' || 'false' }} \ SKIP_TEARDOWN=1 shell: bash diff --git a/cmd/kots/cli/pull.go b/cmd/kots/cli/pull.go index e93de9a7dd..a2df9b1f3d 100644 --- a/cmd/kots/cli/pull.go +++ b/cmd/kots/cli/pull.go @@ -74,6 +74,7 @@ func PullCmd() *cobra.Command { Username: v.GetString("registry-username"), Password: v.GetString("registry-password"), }, + StorageClassName: v.GetString("storage-class"), HTTPProxyEnvValue: v.GetString("http-proxy"), HTTPSProxyEnvValue: v.GetString("https-proxy"), NoProxyEnvValue: v.GetString("no-proxy"), @@ -127,6 +128,7 @@ func PullCmd() *cobra.Command { cmd.Flags().Bool("exclude-kots-kinds", true, "set to true to exclude rendering kots custom objects to the base directory") cmd.Flags().Bool("exclude-admin-console", false, "set to true to exclude the admin console (replicated apps only)") cmd.Flags().String("shared-password", "", "shared password to use when deploying the admin console") + cmd.Flags().String("storage-class", "", "sets the storage class to use for the KOTS Admin Console components (if unset, the default storage class will be used)") cmd.Flags().String("http-proxy", "", "sets HTTP_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("https-proxy", "", "sets HTTPS_PROXY environment variable in all KOTS Admin Console components") cmd.Flags().String("no-proxy", "", "sets NO_PROXY environment variable in all KOTS Admin Console components") diff --git a/e2e/Makefile b/e2e/Makefile index f97dae0c1a..846bcdb923 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -23,6 +23,7 @@ test: KOTSADM_IMAGE_TAG ?= alpha test: TESTIM_BRANCH ?= master test: AIRGAP ?= 0 test: IS_OPENSHIFT ?= 0 +test: IS_EKS ?= 0 test: SKIP_TEARDOWN ?= 0 ifneq ($(EXISTING_KUBECONFIG),) test: EXISTING_KUBECONFIG_VOLUME_MOUNT := "-v=$(EXISTING_KUBECONFIG):$(EXISTING_KUBECONFIG)" @@ -48,6 +49,7 @@ test: --kotsadm-image-tag=$(KOTSADM_IMAGE_TAG) \ --airgap=$(AIRGAP) \ --is-openshift=$(IS_OPENSHIFT) \ + --is-eks=$(IS_EKS) \ --kots-helm-chart-url=$(KOTS_HELM_CHART_URL) \ --kots-helm-chart-version=$(KOTS_HELM_CHART_VERSION) \ --kots-dockerhub-username=$(KOTS_DOCKERHUB_USERNAME) \ diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 250c399661..ea47699c88 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -44,6 +44,7 @@ var ( kotsadmImageTag string airgap bool isOpenShift bool + isEKS bool kotsadmForwardPort string kotsHelmChartURL string kotsHelmChartVersion string @@ -61,6 +62,7 @@ func init() { flag.StringVar(&kotsadmImageTag, "kotsadm-image-tag", "alpha", "override the kotsadm images tag") flag.BoolVar(&airgap, "airgap", false, "run install in airgapped mode") flag.BoolVar(&isOpenShift, "is-openshift", false, "the cluster is an openshift cluster") + flag.BoolVar(&isEKS, "is-eks", false, "the cluster is an eks cluster") flag.StringVar(&kotsadmForwardPort, "kotsadm-forward-port", "", "sets the port that the admin console will be exposed on instead of generating a random one") flag.StringVar(&kotsHelmChartURL, "kots-helm-chart-url", "", "kots helm chart url") flag.StringVar(&kotsHelmChartVersion, "kots-helm-chart-version", "", "kots helm chart version") @@ -96,7 +98,7 @@ var _ = BeforeSuite(func() { veleroCLI = velero.NewCLI(w.GetDir(), isOpenShift) - kotsInstaller = kots.NewInstaller(kotsadmImageRegistry, kotsadmImageNamespace, kotsadmImageTag, airgap, kotsDockerhubUsername, kotsDockerhubPassword) + kotsInstaller = kots.NewInstaller(kotsadmImageRegistry, kotsadmImageNamespace, kotsadmImageTag, airgap, kotsDockerhubUsername, kotsDockerhubPassword, isEKS) }) var _ = AfterSuite(func() { diff --git a/e2e/kots/kots.go b/e2e/kots/kots.go index a7e40a261e..53b62250ee 100644 --- a/e2e/kots/kots.go +++ b/e2e/kots/kots.go @@ -27,9 +27,10 @@ type Installer struct { airgap bool dockerhubUsername string dockerhubPassword string + isEKS bool } -func NewInstaller(imageRegistry, imageNamespace, imageTag string, airgap bool, dockerhubUsername string, dockerhubPassword string) *Installer { +func NewInstaller(imageRegistry, imageNamespace, imageTag string, airgap bool, dockerhubUsername string, dockerhubPassword string, isEKS bool) *Installer { return &Installer{ imageRegistry: imageRegistry, imageNamespace: imageNamespace, @@ -37,6 +38,7 @@ func NewInstaller(imageRegistry, imageNamespace, imageTag string, airgap bool, d airgap: airgap, dockerhubUsername: dockerhubUsername, dockerhubPassword: dockerhubPassword, + isEKS: isEKS, } } @@ -100,6 +102,9 @@ func (i *Installer) install(kubeconfig string, test inventory.Test) (*gexec.Sess fmt.Sprintf("--use-minimal-rbac=%t", test.UseMinimalRBAC), fmt.Sprintf("--skip-compatibility-check=%t", test.SkipCompatibilityCheck), } + if i.isEKS { + args = append(args, "--storage-class=gp2") + } return util.RunCommand(exec.Command("kots", args...)) } diff --git a/pkg/pull/pull.go b/pkg/pull/pull.go index 2feccaaca2..8370ec85a6 100644 --- a/pkg/pull/pull.go +++ b/pkg/pull/pull.go @@ -69,6 +69,7 @@ type PullOptions struct { AppSequence int64 AppVersionLabel string IsGitOps bool + StorageClassName string HTTPProxyEnvValue string HTTPSProxyEnvValue string NoProxyEnvValue string @@ -291,6 +292,7 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { IncludeAdminConsole: includeAdminConsole, SharedPassword: pullOptions.SharedPassword, EncryptConfig: encryptConfig, + StorageClassName: pullOptions.StorageClassName, HTTPProxyEnvValue: pullOptions.HTTPProxyEnvValue, HTTPSProxyEnvValue: pullOptions.HTTPSProxyEnvValue, NoProxyEnvValue: pullOptions.NoProxyEnvValue, From 49fc45e015e91cdabdadf07facefaf865c0da3bc Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Wed, 26 Jun 2024 21:21:10 +0000 Subject: [PATCH 3/5] fix multi-namespace test --- .github/workflows/build-test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 19ae937023..10ecb57534 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -1316,7 +1316,8 @@ jobs: --shared-password password \ --kotsadm-registry ttl.sh \ --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h + --kotsadm-tag 24h \ + --storage-class=${{ matrix.cluster.distribution == 'eks' && 'gp2' || '' }} EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then From 3d007863883b2a75713998f62865f3917554f2e6 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Thu, 27 Jun 2024 13:31:49 +0000 Subject: [PATCH 4/5] revert back to eks 1.29 --- .github/actions/cmx-versions/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/cmx-versions/index.js b/.github/actions/cmx-versions/index.js index c2ff291046..39a950bdc6 100644 --- a/.github/actions/cmx-versions/index.js +++ b/.github/actions/cmx-versions/index.js @@ -36,7 +36,9 @@ async function getClusterVersions() { latest_minor_versions: true, }, eks: { - latest_version: true, + // latest_version: true, + // TODO: re-enable latest_version once we have compatibility with 1.30 + versions: new Set(["1.29"]), instance_type: "m7g.large" // arm64 }, openshift: { From 891915b028fcf71db988abbc79864779541af4d8 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Thu, 27 Jun 2024 13:32:37 +0000 Subject: [PATCH 5/5] npm run build --- .github/actions/cmx-versions/dist/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/cmx-versions/dist/index.js b/.github/actions/cmx-versions/dist/index.js index 24d351b8be..8bfc9f6e47 100644 --- a/.github/actions/cmx-versions/dist/index.js +++ b/.github/actions/cmx-versions/dist/index.js @@ -7656,7 +7656,9 @@ async function getClusterVersions() { latest_minor_versions: true, }, eks: { - latest_version: true, + // latest_version: true, + // TODO: re-enable latest_version once we have compatibility with 1.30 + versions: new Set(["1.29"]), instance_type: "m7g.large" // arm64 }, openshift: {