From f4907f0f01071037207ce6b47e183bf6230d7731 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Fri, 6 Sep 2024 13:36:56 -0400 Subject: [PATCH] begin adding labels and annotations (#4864) * begin adding labels and annotations * labels/annotations not required here * labels/annotations not required here * add additional label and annotation check to minimal RBAC install test * actionlint? * ? * no comments * quotes * fix shellcheck * fix annotation/label values in test * force failure * return values * remove unneeded comments --- .github/workflows/build-test.yaml | 71 +++++++++++++++++++++ cmd/kots/cli/install.go | 21 ++++++ pkg/kotsadm/objects/configmaps_objects.go | 12 ++++ pkg/kotsadm/objects/distribution_objects.go | 19 ++++-- pkg/kotsadm/objects/kotsadm_objects.go | 64 ++++++++++++------- pkg/kotsadm/objects/minio_objects.go | 28 +++++--- pkg/kotsadm/objects/rqlite_objects.go | 19 ++++-- pkg/kotsadm/types/deployoptions.go | 2 + 8 files changed, 193 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 269db08ca7..bb5134649d 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -691,6 +691,10 @@ jobs: --shared-password password \ --kotsadm-registry ttl.sh \ --kotsadm-namespace automated-${{ github.run_id }} \ + --additional-labels testlabel1=testvalue1 \ + --additional-labels test.label/two=test.value.two \ + --additional-annotations testannotation1=testannotationvalue1 \ + --additional-annotations test.annotation/two=testannotation.value.two \ --kotsadm-tag 24h | tee output.txt if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then @@ -732,6 +736,73 @@ jobs: exit 1 fi + echo "check that the kotsadm, minio and rqlite pods have the correct labels and annotations" + echo "label one" + if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.labels.testlabel1}' | grep -q testvalue1; then + echo "kotsadm pod does not have the correct testlabel1 label" + kubectl get pods -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.labels}' + exit 1 + fi + if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.labels.testlabel1}' | grep -q testvalue1; then + echo "kotsadm-minio pod does not have the correct testlabel1 label" + kubectl get pods -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.labels}' + exit 1 + fi + if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.labels.testlabel1}' | grep -q testvalue1; then + echo "kotsadm-rqlite pod does not have the correct testlabel1 label" + kubectl get pods -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.labels}' + exit 1 + fi + echo "annotation one" + if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.annotations.testannotation1}' | grep -q testannotationvalue1; then + echo "kotsadm pod does not have the correct testannotation1 annotation" + kubectl get pods -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.annotations}' + exit 1 + fi + if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.annotations.testannotation1}' | grep -q testannotationvalue1; then + echo "kotsadm-minio pod does not have the correct testannotation1 annotation" + kubectl get pods -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.annotations}' + exit 1 + fi + if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.annotations.testannotation1}' | grep -q testannotationvalue1; then + echo "kotsadm-rqlite pod does not have the correct testannotation1 annotation" + kubectl get pods -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.annotations}' + exit 1 + fi + echo "label two" + if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.labels.test\.label/two}' | grep -q 'test.value.two'; then + echo "kotsadm pod does not have the correct test.label/two label" + kubectl get pod -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.labels}' + exit 1 + fi + if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.labels.test\.label/two}' | grep -q 'test.value.two'; then + echo "kotsadm-minio pod does not have the correct test.label/two label" + kubectl get pod -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.labels}' + exit 1 + fi + if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.labels.test\.label/two}' | grep -q 'test.value.two'; then + echo "kotsadm-rqlite pod does not have the correct test.label/two label" + kubectl get pod -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.labels}' + exit 1 + fi + echo "annotation two" + if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.annotations.test\.annotation/two}' | grep -q 'testannotation.value.two'; then + echo "kotsadm pod does not have the correct test.annotation/two annotation" + kubectl get pod -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.annotations}' + exit 1 + fi + if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.annotations.test\.annotation/two}' | grep -q 'testannotation.value.two'; then + echo "kotsadm-minio pod does not have the correct test.annotation/two annotation" + kubectl get pod -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.annotations}' + exit 1 + fi + if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.annotations.test\.annotation/two}' | grep -q 'testannotation.value.two'; then + echo "kotsadm-rqlite pod does not have the correct test.annotation/two annotation" + kubectl get pod -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.annotations}' + exit 1 + fi + echo "additional pod labels and annotations are present" + - name: Generate support bundle on failure if: failure() uses: ./.github/actions/generate-support-bundle diff --git a/cmd/kots/cli/install.go b/cmd/kots/cli/install.go index 7d700e7ff7..d4f87d5ffa 100644 --- a/cmd/kots/cli/install.go +++ b/cmd/kots/cli/install.go @@ -260,6 +260,23 @@ func InstallCmd() *cobra.Command { simultaneousUploads, _ := strconv.Atoi(v.GetString("airgap-upload-parallelism")) + additionalLabels := map[string]string{} + additionalAnnotations := map[string]string{} + for _, label := range v.GetStringSlice("additional-labels") { + parts := strings.Split(label, "=") + if len(parts) != 2 { + return errors.Errorf("additional-labels flag is not in the correct format. Must be key=value") + } + additionalLabels[parts[0]] = parts[1] + } + for _, annotation := range v.GetStringSlice("additional-annotations") { + parts := strings.Split(annotation, "=") + if len(parts) != 2 { + return errors.Errorf("additional-annotations flag is not in the correct format. Must be key=value") + } + additionalAnnotations[parts[0]] = parts[1] + } + deployOptions := kotsadmtypes.DeployOptions{ Namespace: namespace, Context: v.GetString("context"), @@ -289,6 +306,8 @@ func InstallCmd() *cobra.Command { IncludeMinioSnapshots: v.GetBool("with-minio"), StrictSecurityContext: v.GetBool("strict-security-context"), RequestedChannelSlug: preferredChannelSlug, + AdditionalLabels: additionalLabels, + AdditionalAnnotations: additionalAnnotations, RegistryConfig: *registryConfig, @@ -530,6 +549,8 @@ func InstallCmd() *cobra.Command { cmd.Flags().Bool("skip-compatibility-check", false, "set to true to skip compatibility checks between the current kots version and the app") cmd.Flags().String("app-version-label", "", "the application version label to install. if not specified, the latest version will be installed") cmd.Flags().Bool("exclude-admin-console", false, "set to true to exclude the admin console and only install the application") + cmd.Flags().StringArray("additional-annotations", []string{}, "additional annotations to add to kotsadm pods") + cmd.Flags().StringArray("additional-labels", []string{}, "additional labels to add to kotsadm pods") registryFlags(cmd.Flags()) diff --git a/pkg/kotsadm/objects/configmaps_objects.go b/pkg/kotsadm/objects/configmaps_objects.go index 2082570f6b..20446b7175 100644 --- a/pkg/kotsadm/objects/configmaps_objects.go +++ b/pkg/kotsadm/objects/configmaps_objects.go @@ -3,6 +3,7 @@ package kotsadm import ( _ "embed" "fmt" + "strings" "github.com/replicatedhq/kots/pkg/kotsadm/types" kotsadmversion "github.com/replicatedhq/kots/pkg/kotsadm/version" @@ -11,6 +12,15 @@ import ( ) func KotsadmConfigMap(deployOptions types.DeployOptions) *corev1.ConfigMap { + additionalAnnotationsArray := []string{} + for key, value := range deployOptions.AdditionalAnnotations { + additionalAnnotationsArray = append(additionalAnnotationsArray, fmt.Sprintf("%s=%s", key, value)) + } + additionalLabelsArray := []string{} + for key, value := range deployOptions.AdditionalLabels { + additionalLabelsArray = append(additionalLabelsArray, fmt.Sprintf("%s=%s", key, value)) + } + data := map[string]string{ "kots-install-id": fmt.Sprintf("%v", deployOptions.InstallID), "initial-app-images-pushed": fmt.Sprintf("%v", deployOptions.AppImagesPushed), @@ -26,6 +36,8 @@ func KotsadmConfigMap(deployOptions types.DeployOptions) *corev1.ConfigMap { "with-minio": fmt.Sprintf("%v", deployOptions.IncludeMinio), "app-version-label": deployOptions.AppVersionLabel, "requested-channel-slug": deployOptions.RequestedChannelSlug, + "additional-annotations": strings.Join(additionalAnnotationsArray, ","), + "additional-labels": strings.Join(additionalLabelsArray, ","), } if kotsadmversion.KotsadmPullSecret(deployOptions.Namespace, deployOptions.RegistryConfig) != nil { diff --git a/pkg/kotsadm/objects/distribution_objects.go b/pkg/kotsadm/objects/distribution_objects.go index 147f7b67d2..f3f09f2272 100644 --- a/pkg/kotsadm/objects/distribution_objects.go +++ b/pkg/kotsadm/objects/distribution_objects.go @@ -84,15 +84,23 @@ func DistributionStatefulset(deployOptions types.DeployOptions, size resource.Qu securityContext = k8sutil.SecurePodContext(1000, 1000, deployOptions.StrictSecurityContext) } + podLabels := map[string]string{ + "app": "kotsadm-storage-registry", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "StatefulSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm-storage-registry", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm-storage-registry", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ @@ -124,9 +132,8 @@ func DistributionStatefulset(deployOptions types.DeployOptions, size resource.Qu ServiceName: "kotsadm-storage-registry", Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm-storage-registry", - }), + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(podLabels), }, Spec: corev1.PodSpec{ SecurityContext: securityContext, diff --git a/pkg/kotsadm/objects/kotsadm_objects.go b/pkg/kotsadm/objects/kotsadm_objects.go index f091d31f02..348ab9a139 100644 --- a/pkg/kotsadm/objects/kotsadm_objects.go +++ b/pkg/kotsadm/objects/kotsadm_objects.go @@ -344,15 +344,31 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }) } + podAnnotations := map[string]string{ + "backup.velero.io/backup-volumes": "backup", + "pre.hook.backup.velero.io/command": `["/backup.sh"]`, + "pre.hook.backup.velero.io/timeout": "10m", + } + for k, v := range deployOptions.AdditionalAnnotations { + podAnnotations[k] = v + } + podLabels := map[string]string{ + "app": "kotsadm", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + deployment := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "Deployment", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -362,14 +378,8 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm", - }), - Annotations: map[string]string{ - "backup.velero.io/backup-volumes": "backup", - "pre.hook.backup.velero.io/command": `["/backup.sh"]`, - "pre.hook.backup.velero.io/timeout": "10m", - }, + Labels: types.GetKotsadmLabels(podLabels), + Annotations: podAnnotations, }, Spec: corev1.PodSpec{ Affinity: &corev1.Affinity{ @@ -841,15 +851,31 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit storageClassName = &deployOptions.StorageClassName } + podAnnotations := map[string]string{ + "backup.velero.io/backup-volumes": "backup", + "pre.hook.backup.velero.io/command": `["/backup.sh"]`, + "pre.hook.backup.velero.io/timeout": "10m", + } + for k, v := range deployOptions.AdditionalAnnotations { + podAnnotations[k] = v + } + podLabels := map[string]string{ + "app": "kotsadm", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "StatefulSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.StatefulSetSpec{ ServiceName: "kotsadm", @@ -860,14 +886,8 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm", - }), - Annotations: map[string]string{ - "backup.velero.io/backup-volumes": "backup", - "pre.hook.backup.velero.io/command": `["/backup.sh"]`, - "pre.hook.backup.velero.io/timeout": "10m", - }, + Labels: types.GetKotsadmLabels(podLabels), + Annotations: podAnnotations, }, Spec: corev1.PodSpec{ Affinity: &corev1.Affinity{ diff --git a/pkg/kotsadm/objects/minio_objects.go b/pkg/kotsadm/objects/minio_objects.go index 9e8f4bea71..26f5a19620 100644 --- a/pkg/kotsadm/objects/minio_objects.go +++ b/pkg/kotsadm/objects/minio_objects.go @@ -82,15 +82,29 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity) initContainers = append(initContainers, migrateToMinioXlInitContainers(deployOptions, resourceRequirements)...) } + podAnnotations := map[string]string{ + "backup.velero.io/backup-volumes": "kotsadm-minio,minio-config-dir,minio-cert-dir", + } + for k, v := range deployOptions.AdditionalAnnotations { + podAnnotations[k] = v + } + podLabels := map[string]string{ + "app": "kotsadm-minio", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "StatefulSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm-minio", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm-minio", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ @@ -122,12 +136,8 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity) }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm-minio", - }), - Annotations: map[string]string{ - "backup.velero.io/backup-volumes": "kotsadm-minio,minio-config-dir,minio-cert-dir", - }, + Labels: types.GetKotsadmLabels(podLabels), + Annotations: podAnnotations, }, Spec: corev1.PodSpec{ Affinity: &corev1.Affinity{ diff --git a/pkg/kotsadm/objects/rqlite_objects.go b/pkg/kotsadm/objects/rqlite_objects.go index 80c3359ccb..a3d8cefed4 100644 --- a/pkg/kotsadm/objects/rqlite_objects.go +++ b/pkg/kotsadm/objects/rqlite_objects.go @@ -50,15 +50,23 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity storageClassName = &deployOptions.StorageClassName } + podLabels := map[string]string{ + "app": "kotsadm-rqlite", + } + for k, v := range deployOptions.AdditionalLabels { + podLabels[k] = v + } + statefulset := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "StatefulSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm-rqlite", - Namespace: deployOptions.Namespace, - Labels: types.GetKotsadmLabels(), + Name: "kotsadm-rqlite", + Namespace: deployOptions.Namespace, + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(deployOptions.AdditionalLabels), }, Spec: appsv1.StatefulSetSpec{ ServiceName: "kotsadm-rqlite-headless", @@ -89,9 +97,8 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: types.GetKotsadmLabels(map[string]string{ - "app": "kotsadm-rqlite", - }), + Annotations: deployOptions.AdditionalAnnotations, + Labels: types.GetKotsadmLabels(podLabels), }, Spec: corev1.PodSpec{ SecurityContext: securityContext, diff --git a/pkg/kotsadm/types/deployoptions.go b/pkg/kotsadm/types/deployoptions.go index 14d5be2404..35a061d600 100644 --- a/pkg/kotsadm/types/deployoptions.go +++ b/pkg/kotsadm/types/deployoptions.go @@ -57,6 +57,8 @@ type DeployOptions struct { AdditionalNamespaces []string IsGKEAutopilot bool RequestedChannelSlug string + AdditionalAnnotations map[string]string + AdditionalLabels map[string]string IdentityConfig kotsv1beta1.IdentityConfig IngressConfig kotsv1beta1.IngressConfig