begin adding labels and annotations

.github/workflows/build-test.yaml

@@ -691,6 +691,10 @@ jobs:
             --shared-password password \
             --kotsadm-registry ttl.sh \
             --kotsadm-namespace automated-${{ github.run_id }} \
+            --additional-labels testlabel1=testvalue1 \
+            --additional-labels test.label/two=test.value.two \
+            --additional-annotations testannotation1=testannotationvalue1 \
+            --additional-annotations test.annotation/two=testannotation.value.two \
             --kotsadm-tag 24h | tee output.txt
 
           if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
@@ -732,6 +736,73 @@ jobs:
             exit 1
           fi
 
+          echo "check that the kotsadm, minio and rqlite pods have the correct labels and annotations"
+          echo "label one"
+          if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.labels.testlabel1}' | grep -q testvalue1; then
+              echo "kotsadm pod does not have the correct testlabel1 label"
+              kubectl get pods -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.labels}'
+              exit 1
+          fi
+          if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.labels.testlabel1}' | grep -q testvalue1; then
+              echo "kotsadm-minio pod does not have the correct testlabel1 label"
+              kubectl get pods -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.labels}'
+              exit 1
+          fi
+          if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.labels.testlabel1}' | grep -q testvalue1; then
+              echo "kotsadm-rqlite pod does not have the correct testlabel1 label"
+              kubectl get pods -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.labels}'
+              exit 1
+          fi
+          echo "annotation one"
+          if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.annotations.testannotation1}' | grep -q testannotationvalue1; then
+              echo "kotsadm pod does not have the correct testannotation1 annotation"
+              kubectl get pods -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.annotations}'
+              exit 1
+          fi
+          if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.annotations.testannotation1}' | grep -q testannotationvalue1; then
+              echo "kotsadm-minio pod does not have the correct testannotation1 annotation"
+              kubectl get pods -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.annotations}'
+              exit 1
+          fi
+          if ! kubectl get pods -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.annotations.testannotation1}' | grep -q testannotationvalue1; then
+              echo "kotsadm-rqlite pod does not have the correct testannotation1 annotation"
+              kubectl get pods -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.annotations}'
+              exit 1
+          fi
+          echo "label two"
+          if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.labels.test\.label/two}' | grep -q 'test.value.two'; then
+              echo "kotsadm pod does not have the correct test.label/two label"
+              kubectl get pod -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.labels}'
+              exit 1
+          fi
+          if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.labels.test\.label/two}' | grep -q 'test.value.two'; then
+              echo "kotsadm-minio pod does not have the correct test.label/two label"
+              kubectl get pod -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.labels}'
+              exit 1
+          fi
+          if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.labels.test\.label/two}' | grep -q 'test.value.two'; then
+              echo "kotsadm-rqlite pod does not have the correct test.label/two label"
+              kubectl get pod -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.labels}'
+              exit 1
+          fi
+          echo "annotation two"
+          if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.annotations.test\.annotation/two}' | grep -q 'testannotation.value.two'; then
+              echo "kotsadm pod does not have the correct test.annotation/two annotation"
+              kubectl get pod -n "$APP_SLUG" -l app=kotsadm -o jsonpath='{.items[0].metadata.annotations}'
+              exit 1
+          fi
+          if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.annotations.test\.annotation/two}' | grep -q 'testannotation.value.two'; then
+              echo "kotsadm-minio pod does not have the correct test.annotation/two annotation"
+              kubectl get pod -n "$APP_SLUG" -l app=kotsadm-minio -o jsonpath='{.items[0].metadata.annotations}'
+              exit 1
+          fi
+          if ! kubectl get pod -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.annotations.test\.annotation/two}' | grep -q 'testannotation.value.two'; then
+              echo "kotsadm-rqlite pod does not have the correct test.annotation/two annotation"
+              kubectl get pod -n "$APP_SLUG" -l app=kotsadm-rqlite -o jsonpath='{.items[0].metadata.annotations}'
+              exit 1
+          fi
+          echo "additional pod labels and annotations are present"
+
       - name: Generate support bundle on failure
         if: failure()
         uses: ./.github/actions/generate-support-bundle

cmd/kots/cli/install.go

@@ -260,6 +260,23 @@ func InstallCmd() *cobra.Command {
 
 			simultaneousUploads, _ := strconv.Atoi(v.GetString("airgap-upload-parallelism"))
 
+			additionalLabels := map[string]string{}
+			additionalAnnotations := map[string]string{}
+			for _, label := range v.GetStringSlice("additional-labels") {
+				parts := strings.Split(label, "=")
+				if len(parts) != 2 {
+					return errors.Errorf("additional-labels flag is not in the correct format.  Must be key=value")
+				}
+				additionalLabels[parts[0]] = parts[1]
+			}
+			for _, annotation := range v.GetStringSlice("additional-annotations") {
+				parts := strings.Split(annotation, "=")
+				if len(parts) != 2 {
+					return errors.Errorf("additional-annotations flag is not in the correct format.  Must be key=value")
+				}
+				additionalAnnotations[parts[0]] = parts[1]
+			}
+
 			deployOptions := kotsadmtypes.DeployOptions{
 				Namespace:              namespace,
 				Context:                v.GetString("context"),
@@ -289,6 +306,8 @@ func InstallCmd() *cobra.Command {
 				IncludeMinioSnapshots:  v.GetBool("with-minio"),
 				StrictSecurityContext:  v.GetBool("strict-security-context"),
 				RequestedChannelSlug:   preferredChannelSlug,
+				AdditionalLabels:       additionalLabels,
+				AdditionalAnnotations:  additionalAnnotations,
 
 				RegistryConfig: *registryConfig,
 
@@ -530,6 +549,8 @@ func InstallCmd() *cobra.Command {
 	cmd.Flags().Bool("skip-compatibility-check", false, "set to true to skip compatibility checks between the current kots version and the app")
 	cmd.Flags().String("app-version-label", "", "the application version label to install. if not specified, the latest version will be installed")
 	cmd.Flags().Bool("exclude-admin-console", false, "set to true to exclude the admin console and only install the application")
+	cmd.Flags().StringArray("additional-annotations", []string{}, "additional annotations to add to kotsadm pods")
+	cmd.Flags().StringArray("additional-labels", []string{}, "additional labels to add to kotsadm pods")
 
 	registryFlags(cmd.Flags())
 

pkg/kotsadm/objects/configmaps_objects.go

@@ -3,6 +3,7 @@ package kotsadm
 import (
 	_ "embed"
 	"fmt"
+	"strings"
 
 	"github.com/replicatedhq/kots/pkg/kotsadm/types"
 	kotsadmversion "github.com/replicatedhq/kots/pkg/kotsadm/version"
@@ -11,6 +12,15 @@ import (
 )
 
 func KotsadmConfigMap(deployOptions types.DeployOptions) *corev1.ConfigMap {
+	additionalAnnotationsArray := []string{}
+	for key, value := range deployOptions.AdditionalAnnotations {
+		additionalAnnotationsArray = append(additionalAnnotationsArray, fmt.Sprintf("%s=%s", key, value))
+	}
+	additionalLabelsArray := []string{}
+	for key, value := range deployOptions.AdditionalLabels {
+		additionalLabelsArray = append(additionalLabelsArray, fmt.Sprintf("%s=%s", key, value))
+	}
+
 	data := map[string]string{
 		"kots-install-id":           fmt.Sprintf("%v", deployOptions.InstallID),
 		"initial-app-images-pushed": fmt.Sprintf("%v", deployOptions.AppImagesPushed),
@@ -26,6 +36,8 @@ func KotsadmConfigMap(deployOptions types.DeployOptions) *corev1.ConfigMap {
 		"with-minio":                fmt.Sprintf("%v", deployOptions.IncludeMinio),
 		"app-version-label":         deployOptions.AppVersionLabel,
 		"requested-channel-slug":    deployOptions.RequestedChannelSlug,
+		"additional-annotations":    strings.Join(additionalAnnotationsArray, ","),
+		"additional-labels":         strings.Join(additionalLabelsArray, ","),
 	}
 
 	if kotsadmversion.KotsadmPullSecret(deployOptions.Namespace, deployOptions.RegistryConfig) != nil {

pkg/kotsadm/objects/distribution_objects.go

@@ -84,15 +84,23 @@ func DistributionStatefulset(deployOptions types.DeployOptions, size resource.Qu
 		securityContext = k8sutil.SecurePodContext(1000, 1000, deployOptions.StrictSecurityContext)
 	}
 
+	podLabels := map[string]string{
+		"app": "kotsadm-storage-registry",
+	}
+	for k, v := range deployOptions.AdditionalLabels {
+		podLabels[k] = v
+	}
+
 	statefulset := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "apps/v1",
 			Kind:       "StatefulSet",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "kotsadm-storage-registry",
-			Namespace: deployOptions.Namespace,
-			Labels:    types.GetKotsadmLabels(),
+			Name:        "kotsadm-storage-registry",
+			Namespace:   deployOptions.Namespace,
+			Annotations: deployOptions.AdditionalAnnotations,
+			Labels:      types.GetKotsadmLabels(deployOptions.AdditionalLabels),
 		},
 		Spec: appsv1.StatefulSetSpec{
 			Selector: &metav1.LabelSelector{
@@ -124,9 +132,8 @@ func DistributionStatefulset(deployOptions types.DeployOptions, size resource.Qu
 			ServiceName: "kotsadm-storage-registry",
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: types.GetKotsadmLabels(map[string]string{
-						"app": "kotsadm-storage-registry",
-					}),
+					Annotations: deployOptions.AdditionalAnnotations,
+					Labels:      types.GetKotsadmLabels(podLabels),
 				},
 				Spec: corev1.PodSpec{
 					SecurityContext: securityContext,

pkg/kotsadm/objects/kotsadm_objects.go

@@ -344,15 +344,31 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e
 		})
 	}
 
+	podAnnotations := map[string]string{
+		"backup.velero.io/backup-volumes":   "backup",
+		"pre.hook.backup.velero.io/command": `["/backup.sh"]`,
+		"pre.hook.backup.velero.io/timeout": "10m",
+	}
+	for k, v := range deployOptions.AdditionalAnnotations {
+		podAnnotations[k] = v
+	}
+	podLabels := map[string]string{
+		"app": "kotsadm",
+	}
+	for k, v := range deployOptions.AdditionalLabels {
+		podLabels[k] = v
+	}
+
 	deployment := &appsv1.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "apps/v1",
 			Kind:       "Deployment",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "kotsadm",
-			Namespace: deployOptions.Namespace,
-			Labels:    types.GetKotsadmLabels(),
+			Name:        "kotsadm",
+			Namespace:   deployOptions.Namespace,
+			Annotations: deployOptions.AdditionalAnnotations,
+			Labels:      types.GetKotsadmLabels(deployOptions.AdditionalLabels),
 		},
 		Spec: appsv1.DeploymentSpec{
 			Selector: &metav1.LabelSelector{
@@ -362,14 +378,8 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: types.GetKotsadmLabels(map[string]string{
-						"app": "kotsadm",
-					}),
-					Annotations: map[string]string{
-						"backup.velero.io/backup-volumes":   "backup",
-						"pre.hook.backup.velero.io/command": `["/backup.sh"]`,
-						"pre.hook.backup.velero.io/timeout": "10m",
-					},
+					Labels:      types.GetKotsadmLabels(podLabels),
+					Annotations: podAnnotations,
 				},
 				Spec: corev1.PodSpec{
 					Affinity: &corev1.Affinity{
@@ -841,15 +851,31 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit
 		storageClassName = &deployOptions.StorageClassName
 	}
 
+	podAnnotations := map[string]string{
+		"backup.velero.io/backup-volumes":   "backup",
+		"pre.hook.backup.velero.io/command": `["/backup.sh"]`,
+		"pre.hook.backup.velero.io/timeout": "10m",
+	}
+	for k, v := range deployOptions.AdditionalAnnotations {
+		podAnnotations[k] = v
+	}
+	podLabels := map[string]string{
+		"app": "kotsadm",
+	}
+	for k, v := range deployOptions.AdditionalLabels {
+		podLabels[k] = v
+	}
+
 	statefulset := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "apps/v1",
 			Kind:       "StatefulSet",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "kotsadm",
-			Namespace: deployOptions.Namespace,
-			Labels:    types.GetKotsadmLabels(),
+			Name:        "kotsadm",
+			Namespace:   deployOptions.Namespace,
+			Annotations: deployOptions.AdditionalAnnotations,
+			Labels:      types.GetKotsadmLabels(deployOptions.AdditionalLabels),
 		},
 		Spec: appsv1.StatefulSetSpec{
 			ServiceName: "kotsadm",
@@ -860,14 +886,8 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: types.GetKotsadmLabels(map[string]string{
-						"app": "kotsadm",
-					}),
-					Annotations: map[string]string{
-						"backup.velero.io/backup-volumes":   "backup",
-						"pre.hook.backup.velero.io/command": `["/backup.sh"]`,
-						"pre.hook.backup.velero.io/timeout": "10m",
-					},
+					Labels:      types.GetKotsadmLabels(podLabels),
+					Annotations: podAnnotations,
 				},
 				Spec: corev1.PodSpec{
 					Affinity: &corev1.Affinity{

pkg/kotsadm/objects/minio_objects.go

@@ -82,15 +82,29 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity)
 		initContainers = append(initContainers, migrateToMinioXlInitContainers(deployOptions, resourceRequirements)...)
 	}
 
+	podAnnotations := map[string]string{
+		"backup.velero.io/backup-volumes": "kotsadm-minio,minio-config-dir,minio-cert-dir",
+	}
+	for k, v := range deployOptions.AdditionalAnnotations {
+		podAnnotations[k] = v
+	}
+	podLabels := map[string]string{
+		"app": "kotsadm-minio",
+	}
+	for k, v := range deployOptions.AdditionalLabels {
+		podLabels[k] = v
+	}
+
 	statefulset := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "apps/v1",
 			Kind:       "StatefulSet",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "kotsadm-minio",
-			Namespace: deployOptions.Namespace,
-			Labels:    types.GetKotsadmLabels(),
+			Name:        "kotsadm-minio",
+			Namespace:   deployOptions.Namespace,
+			Annotations: deployOptions.AdditionalAnnotations,
+			Labels:      types.GetKotsadmLabels(deployOptions.AdditionalLabels),
 		},
 		Spec: appsv1.StatefulSetSpec{
 			Selector: &metav1.LabelSelector{
@@ -122,12 +136,8 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity)
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: types.GetKotsadmLabels(map[string]string{
-						"app": "kotsadm-minio",
-					}),
-					Annotations: map[string]string{
-						"backup.velero.io/backup-volumes": "kotsadm-minio,minio-config-dir,minio-cert-dir",
-					},
+					Labels:      types.GetKotsadmLabels(podLabels),
+					Annotations: podAnnotations,
 				},
 				Spec: corev1.PodSpec{
 					Affinity: &corev1.Affinity{

pkg/kotsadm/objects/rqlite_objects.go

@@ -50,15 +50,23 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity
 		storageClassName = &deployOptions.StorageClassName
 	}
 
+	podLabels := map[string]string{
+		"app": "kotsadm-rqlite",
+	}
+	for k, v := range deployOptions.AdditionalLabels {
+		podLabels[k] = v
+	}
+
 	statefulset := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "apps/v1",
 			Kind:       "StatefulSet",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "kotsadm-rqlite",
-			Namespace: deployOptions.Namespace,
-			Labels:    types.GetKotsadmLabels(),
+			Name:        "kotsadm-rqlite",
+			Namespace:   deployOptions.Namespace,
+			Annotations: deployOptions.AdditionalAnnotations,
+			Labels:      types.GetKotsadmLabels(deployOptions.AdditionalLabels),
 		},
 		Spec: appsv1.StatefulSetSpec{
 			ServiceName:         "kotsadm-rqlite-headless",
@@ -89,9 +97,8 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: types.GetKotsadmLabels(map[string]string{
-						"app": "kotsadm-rqlite",
-					}),
+					Annotations: deployOptions.AdditionalAnnotations,
+					Labels:      types.GetKotsadmLabels(podLabels),
 				},
 				Spec: corev1.PodSpec{
 					SecurityContext:  securityContext,

pkg/kotsadm/types/deployoptions.go

@@ -57,6 +57,8 @@ type DeployOptions struct {
 	AdditionalNamespaces   []string
 	IsGKEAutopilot         bool
 	RequestedChannelSlug   string
+	AdditionalAnnotations  map[string]string
+	AdditionalLabels       map[string]string
 
 	IdentityConfig kotsv1beta1.IdentityConfig
 	IngressConfig  kotsv1beta1.IngressConfig