Skip to content

Commit

Permalink
supply additional labels and annotations to fs-minio pods (#4866)
Browse files Browse the repository at this point in the history
* supply additional labels and annotations to fs-minio pods

* use correct namespace

* add GetInstallationParamsWithClientset unit test

* add unit test for kotsadm-fs-minio creation

* update imports
  • Loading branch information
laverya authored Sep 6, 2024
1 parent f4907f0 commit a5ecf2c
Show file tree
Hide file tree
Showing 4 changed files with 278 additions and 14 deletions.
36 changes: 35 additions & 1 deletion pkg/kotsutil/kots.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
serializer "k8s.io/apimachinery/pkg/runtime/serializer/json"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
applicationv1beta1 "sigs.k8s.io/application/api/v1beta1"
)
Expand Down Expand Up @@ -1140,6 +1141,8 @@ type InstallationParams struct {
WithMinio bool
AppVersionLabel string
RequestedChannelSlug string
AdditionalAnnotations map[string]string
AdditionalLabels map[string]string
}

func GetInstallationParams(configMapName string) (InstallationParams, error) {
Expand All @@ -1150,12 +1153,18 @@ func GetInstallationParams(configMapName string) (InstallationParams, error) {
return autoConfig, errors.Wrap(err, "failed to get k8s clientset")
}

return GetInstallationParamsWithClientset(clientset, configMapName, util.PodNamespace)
}

func GetInstallationParamsWithClientset(clientset kubernetes.Interface, configMapName string, namespace string) (InstallationParams, error) {
autoConfig := InstallationParams{}

isKurl, err := kurl.IsKurl(clientset)
if err != nil {
return autoConfig, errors.Wrap(err, "failed to check if cluster is kurl")
}

kotsadmConfigMap, err := clientset.CoreV1().ConfigMaps(util.PodNamespace).Get(context.TODO(), configMapName, metav1.GetOptions{})
kotsadmConfigMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{})
if err != nil {
if kuberneteserrors.IsNotFound(err) {
return autoConfig, nil
Expand Down Expand Up @@ -1183,6 +1192,31 @@ func GetInstallationParams(configMapName string) (InstallationParams, error) {
autoConfig.EnableImageDeletion = isKurl
}

autoConfig.AdditionalAnnotations = make(map[string]string)
allAnnotations := strings.Split(kotsadmConfigMap.Data["additional-annotations"], ",")
for _, annotation := range allAnnotations {
if annotation == "" {
continue
}
parts := strings.Split(annotation, "=")
if len(parts) != 2 {
return autoConfig, errors.Errorf("invalid additional annotation %q", annotation)
}
autoConfig.AdditionalAnnotations[parts[0]] = parts[1]
}
autoConfig.AdditionalLabels = make(map[string]string)
allLabels := strings.Split(kotsadmConfigMap.Data["additional-labels"], ",")
for _, label := range allLabels {
if label == "" {
continue
}
parts := strings.Split(label, "=")
if len(parts) != 2 {
return autoConfig, errors.Errorf("invalid additional label %q", label)
}
autoConfig.AdditionalLabels[parts[0]] = parts[1]
}

return autoConfig, nil
}

Expand Down
122 changes: 122 additions & 0 deletions pkg/kotsutil/kots_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"testing"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
Expand All @@ -23,6 +24,8 @@ import (
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
applicationv1beta1 "sigs.k8s.io/application/api/v1beta1"
)

Expand Down Expand Up @@ -1255,3 +1258,122 @@ func TestFindChannelInLicense(t *testing.T) {
})
}
}

func TestGetInstallationParamsWithClientset(t *testing.T) {
type args struct {
configMapName string
namespace string
clientSet kubernetes.Interface
}
tests := []struct {
name string
args args
want kotsutil.InstallationParams
wantErr assert.ErrorAssertionFunc
}{
{
name: "basic test",
args: args{
configMapName: "kotsadm",
namespace: "test-namespace",
clientSet: fake.NewClientset(&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kotsadm",
Namespace: "test-namespace",
},
Data: map[string]string{
"additional-annotations": "abc/xyz=test-annotation1,test.annotation/two=test.value/two/test",
"additional-labels": "xyz=label2,abc=123",
"app-version-label": "",
"ensure-rbac": "true",
"initial-app-images-pushed": "false",
"kots-install-id": "2liAJUuyAi3Gnyhvi5Arv5BJRZ4",
"minio-enabled-snapshots": "true",
"registry-is-read-only": "false",
"requested-channel-slug": "stable",
"skip-compatibility-check": "false",
"skip-preflights": "false",
"skip-rbac-check": "false",
"strict-security-context": "false",
"use-minimal-rbac": "false",
"wait-duration": "2m0s",
"with-minio": "true",
},
}),
},
want: kotsutil.InstallationParams{
AdditionalAnnotations: map[string]string{"abc/xyz": "test-annotation1", "test.annotation/two": "test.value/two/test"},
AdditionalLabels: map[string]string{"abc": "123", "xyz": "label2"},
AppVersionLabel: "",
EnsureRBAC: true,
KotsadmRegistry: "",
SkipImagePush: false,
SkipPreflights: false,
SkipCompatibilityCheck: false,
RegistryIsReadOnly: false,
EnableImageDeletion: false,
SkipRBACCheck: false,
UseMinimalRBAC: false,
StrictSecurityContext: false,
WaitDuration: time.Minute * 2,
WithMinio: true,
RequestedChannelSlug: "stable",
},
},
{
name: "no labels or annotations",
args: args{
configMapName: "kotsadm",
namespace: "test-namespace",
clientSet: fake.NewClientset(&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kotsadm",
Namespace: "test-namespace",
},
Data: map[string]string{
"app-version-label": "",
"ensure-rbac": "true",
"initial-app-images-pushed": "false",
"kots-install-id": "2liAJUuyAi3Gnyhvi5Arv5BJRZ4",
"minio-enabled-snapshots": "true",
"registry-is-read-only": "false",
"requested-channel-slug": "stable",
"skip-compatibility-check": "false",
"skip-preflights": "false",
"skip-rbac-check": "false",
"strict-security-context": "false",
"use-minimal-rbac": "false",
"wait-duration": "2m0s",
"with-minio": "true",
},
}),
},
want: kotsutil.InstallationParams{
AdditionalAnnotations: map[string]string{},
AdditionalLabels: map[string]string{},
AppVersionLabel: "",
EnsureRBAC: true,
KotsadmRegistry: "",
SkipImagePush: false,
SkipPreflights: false,
SkipCompatibilityCheck: false,
RegistryIsReadOnly: false,
EnableImageDeletion: false,
SkipRBACCheck: false,
UseMinimalRBAC: false,
StrictSecurityContext: false,
WaitDuration: time.Minute * 2,
WithMinio: true,
RequestedChannelSlug: "stable",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := require.New(t)
got, err := kotsutil.GetInstallationParamsWithClientset(tt.args.clientSet, tt.args.configMapName, tt.args.namespace)
req.NoError(err)
req.Equal(tt.want, got)
})
}
}
53 changes: 40 additions & 13 deletions pkg/snapshot/filesystem_minio.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
kotsadmresources "github.com/replicatedhq/kots/pkg/kotsadm/resources"
kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types"
kotsadmversion "github.com/replicatedhq/kots/pkg/kotsadm/version"
"github.com/replicatedhq/kots/pkg/kotsutil"
"github.com/replicatedhq/kots/pkg/kurl"
"github.com/replicatedhq/kots/pkg/logger"
kotss3 "github.com/replicatedhq/kots/pkg/s3"
Expand Down Expand Up @@ -300,6 +301,11 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe
}
}

globalOptions, err := kotsutil.GetInstallationParamsWithClientset(clientset, kotsadmtypes.KotsadmConfigMap, deployOptions.Namespace)
if err != nil {
return nil, errors.Wrap(err, "failed to get global options")
}

env := []corev1.EnvVar{
{
Name: "MINIO_UPDATE",
Expand Down Expand Up @@ -329,13 +335,28 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe
},
}

podAnnotations := map[string]string{
"kots.io/fs-minio-creds-secret-checksum": secretChecksum,
}
for k, v := range globalOptions.AdditionalAnnotations {
podAnnotations[k] = v
}
podLabels := map[string]string{
"app": "kotsadm-fs-minio",
}
for k, v := range globalOptions.AdditionalLabels {
podLabels[k] = v
}

return &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: "apps/v1",
Kind: "Deployment",
},
ObjectMeta: metav1.ObjectMeta{
Name: FileSystemMinioDeploymentName,
Name: FileSystemMinioDeploymentName,
Labels: globalOptions.AdditionalLabels,
Annotations: globalOptions.AdditionalAnnotations,
},
Spec: appsv1.DeploymentSpec{
Replicas: pointer.Int32Ptr(1),
Expand All @@ -349,12 +370,8 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "kotsadm-fs-minio",
},
Annotations: map[string]string{
"kots.io/fs-minio-creds-secret-checksum": secretChecksum,
},
Labels: podLabels,
Annotations: podAnnotations,
},
Spec: corev1.PodSpec{
SecurityContext: &securityContext,
Expand Down Expand Up @@ -837,18 +854,28 @@ func fileSystemMinioConfigPod(clientset kubernetes.Interface, deployOptions File
}
}

globalOptions, err := kotsutil.GetInstallationParamsWithClientset(clientset, kotsadmtypes.KotsadmConfigMap, deployOptions.Namespace)
if err != nil {
return nil, errors.Wrap(err, "failed to get global options")
}
podLabels := map[string]string{
"app": "kotsadm-fs-minio",
"check": podCheckTag,
}
for k, v := range globalOptions.AdditionalLabels {
podLabels[k] = v
}

pod := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Pod",
},
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: deployOptions.Namespace,
Labels: map[string]string{
"app": "kotsadm-fs-minio",
"check": podCheckTag,
},
Name: podName,
Namespace: deployOptions.Namespace,
Labels: podLabels,
Annotations: globalOptions.AdditionalAnnotations,
},
Spec: corev1.PodSpec{
SecurityContext: &securityContext,
Expand Down
81 changes: 81 additions & 0 deletions pkg/snapshot/filesystem_minio_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package snapshot

import (
"context"
"testing"

"github.com/replicatedhq/kots/pkg/kotsadm/types"
kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types"
kotssnapshottypes "github.com/replicatedhq/kots/pkg/snapshot/types"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
)

func Test_ensureFileSystemMinioDeployment(t *testing.T) {
hostPathString := "/test/hostpath"
type args struct {
clientSet kubernetes.Interface
deployOptions FileSystemDeployOptions
registryConfig types.RegistryConfig
marshalledSecret []byte
}
tests := []struct {
name string
args args
validate func(t *testing.T, clientset kubernetes.Interface)
}{
{
name: "new hostpath deployment",
args: args{
clientSet: fake.NewClientset(
&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kotsadmtypes.KotsadmConfigMap,
Namespace: "test-namespace",
},
Data: map[string]string{
"additional-annotations": "abc/xyz=test-annotation1,test.annotation/two=test.value/two/test",
"additional-labels": "xyz=label2,abc=123",
},
},
),
deployOptions: FileSystemDeployOptions{
Namespace: "test-namespace",
IsOpenShift: false,
ForceReset: false,
FileSystemConfig: kotssnapshottypes.FileSystemConfig{
HostPath: &hostPathString,
},
},
},
validate: func(t *testing.T, clientset kubernetes.Interface) {
createdDeployment, err := clientset.AppsV1().Deployments("test-namespace").Get(context.Background(), "kotsadm-fs-minio", metav1.GetOptions{})
require.NoError(t, err)
require.NotNil(t, createdDeployment)

// check annotations
require.Equal(t, map[string]string{"abc/xyz": "test-annotation1", "test.annotation/two": "test.value/two/test"}, createdDeployment.Annotations)
// check labels
require.Equal(t, map[string]string{"xyz": "label2", "abc": "123"}, createdDeployment.Labels)

// check pod template annotations
require.Equal(t, map[string]string{"abc/xyz": "test-annotation1", "test.annotation/two": "test.value/two/test", "kots.io/fs-minio-creds-secret-checksum": "d41d8cd98f00b204e9800998ecf8427e"}, createdDeployment.Spec.Template.Annotations)
// check pod template labels
require.Equal(t, map[string]string{"xyz": "label2", "abc": "123", "app": "kotsadm-fs-minio"}, createdDeployment.Spec.Template.Labels)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := context.Background()
req := require.New(t)

err := ensureFileSystemMinioDeployment(ctx, tt.args.clientSet, tt.args.deployOptions, tt.args.registryConfig, tt.args.marshalledSecret)
req.NoError(err)
tt.validate(t, tt.args.clientSet)
})
}
}

0 comments on commit a5ecf2c

Please sign in to comment.