From 7ee38183069359378e62c643eed0576d00823173 Mon Sep 17 00:00:00 2001 From: Jan Bouska Date: Tue, 12 Nov 2024 14:27:27 +0100 Subject: [PATCH] PoC rework ensure --- .../controller/common/action/base_action.go | 24 ++++ .../common/utils/kubernetes/common.go | 27 ++++ .../common/utils/kubernetes/deployment.go | 18 +++ .../common/utils/kubernetes/ensure/common.go | 45 +++++++ .../utils/kubernetes/ensure/deployment.go | 13 ++ internal/controller/common/utils/set_proxy.go | 21 ++- .../controller/common/utils/set_proxy_test.go | 19 ++- internal/controller/tuf/actions/deployment.go | 120 +++++++++++++++--- .../controller/tuf/utils/tuf_deployment.go | 95 -------------- 9 files changed, 261 insertions(+), 121 deletions(-) create mode 100644 internal/controller/common/utils/kubernetes/ensure/common.go create mode 100644 internal/controller/common/utils/kubernetes/ensure/deployment.go delete mode 100644 internal/controller/tuf/utils/tuf_deployment.go diff --git a/internal/controller/common/action/base_action.go b/internal/controller/common/action/base_action.go index 5e62a2051..590213332 100644 --- a/internal/controller/common/action/base_action.go +++ b/internal/controller/common/action/base_action.go @@ -10,9 +10,12 @@ import ( "strings" "time" + "github.com/securesign/operator/internal/apis" "github.com/securesign/operator/internal/controller/annotations" + "github.com/securesign/operator/internal/controller/constants" "k8s.io/apimachinery/pkg/api/equality" apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -62,6 +65,7 @@ func (action *BaseAction) StatusUpdate(ctx context.Context, obj client2.Object) return &Result{Result: reconcile.Result{Requeue: false}} } +// Deprecated: Use Error function func (action *BaseAction) Failed(err error) *Result { action.Logger.Error(err, "error during action execution") return &Result{ @@ -70,6 +74,25 @@ func (action *BaseAction) Failed(err error) *Result { } } +func (action *BaseAction) Error(ctx context.Context, err error, instance apis.ConditionsAwareObject) *Result { + if errors.Is(err, reconcile.TerminalError(err)) { + instance.SetCondition(metav1.Condition{ + Type: constants.Ready, + Status: metav1.ConditionFalse, + Reason: constants.Failure, + Message: err.Error(), + }) + if updateErr := action.Client.Status().Update(ctx, instance); updateErr != nil { + err = errors.Join(err, updateErr) + } + } + action.Logger.Error(err, "error during action execution") + return &Result{ + Err: err, + } +} + +// Deprecated: Use Error function with TerminalError passed as an argument func (action *BaseAction) FailedWithStatusUpdate(ctx context.Context, err error, instance client2.Object) *Result { if e := action.Client.Status().Update(ctx, instance); e != nil { if strings.Contains(err.Error(), OptimisticLockErrorMsg) { @@ -96,6 +119,7 @@ func (action *BaseAction) Requeue() *Result { } } +// Deprecated: Use kubernetes.CreateOrUpdate function func (action *BaseAction) Ensure(ctx context.Context, obj client2.Object, opts ...EnsureOption) (bool, error) { var ( expected client2.Object diff --git a/internal/controller/common/utils/kubernetes/common.go b/internal/controller/common/utils/kubernetes/common.go index 29bac7968..98e0b0882 100644 --- a/internal/controller/common/utils/kubernetes/common.go +++ b/internal/controller/common/utils/kubernetes/common.go @@ -9,7 +9,11 @@ import ( "strconv" "strings" + "github.com/securesign/operator/internal/controller/annotations" + apiErrors "k8s.io/apimachinery/pkg/api/errors" k8sLabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" v13 "github.com/openshift/api/operator/v1" "github.com/securesign/operator/internal/controller/common/utils" @@ -133,3 +137,26 @@ func FindByLabelSelector(ctx context.Context, c client.Client, list client.Objec return c.List(ctx, list, client.InNamespace(namespace), listOptions) } + +func CreateOrUpdate[T client.Object](ctx context.Context, cli client.Client, obj T, fn ...func(object T) error) (result controllerutil.OperationResult, err error) { + err = retry.OnError(retry.DefaultRetry, func(err error) bool { + return apiErrors.IsConflict(err) || apiErrors.IsAlreadyExists(err) + }, func() error { + var createUpdateError error + result, createUpdateError = controllerutil.CreateOrUpdate(ctx, cli, obj, func() (fnError error) { + annoStr, find := obj.GetAnnotations()[annotations.PausedReconciliation] + if find { + annoBool, _ := strconv.ParseBool(annoStr) + if annoBool { + return + } + } + for _, f := range fn { + fnError = errors.Join(fnError, f(obj)) + } + return + }) + return createUpdateError + }) + return +} diff --git a/internal/controller/common/utils/kubernetes/deployment.go b/internal/controller/common/utils/kubernetes/deployment.go index b1450b674..a55bf7823 100644 --- a/internal/controller/common/utils/kubernetes/deployment.go +++ b/internal/controller/common/utils/kubernetes/deployment.go @@ -117,3 +117,21 @@ func getDeploymentCondition(status v1.DeploymentStatus, condType v1.DeploymentCo } return nil } + +func FindContainerByName(dp *v1.Deployment, containerName string) *corev1.Container { + for i, c := range dp.Spec.Template.Spec.Containers { + if c.Name == containerName { + return &dp.Spec.Template.Spec.Containers[i] + } + } + return nil +} + +func FindVolumeByName(dp *v1.Deployment, volumeName string) *corev1.Volume { + for i, v := range dp.Spec.Template.Spec.Volumes { + if v.Name == volumeName { + return &dp.Spec.Template.Spec.Volumes[i] + } + } + return nil +} diff --git a/internal/controller/common/utils/kubernetes/ensure/common.go b/internal/controller/common/utils/kubernetes/ensure/common.go new file mode 100644 index 000000000..19056cdc7 --- /dev/null +++ b/internal/controller/common/utils/kubernetes/ensure/common.go @@ -0,0 +1,45 @@ +package ensure + +import ( + "slices" + + "golang.org/x/exp/maps" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func managedDeleteFunction(managed []string) func(string, string) bool { + return func(key, _ string) bool { + return slices.Contains(managed, key) + } +} + +func Labels[T client.Object](managedLabels []string, labels map[string]string) func(controlled T) error { + return func(obj T) (e error) { + if obj.GetLabels() == nil { + obj.SetLabels(labels) + return + } + maps.DeleteFunc(obj.GetLabels(), managedDeleteFunction(managedLabels)) + maps.Copy(obj.GetLabels(), labels) + return + } +} + +func Annotations[T client.Object](managedAnnotations []string, annotations map[string]string) func(controlled T) error { + return func(obj T) (e error) { + if obj.GetAnnotations() == nil { + obj.SetAnnotations(annotations) + return + } + maps.DeleteFunc(obj.GetAnnotations(), managedDeleteFunction(managedAnnotations)) + maps.Copy(obj.GetAnnotations(), annotations) + return + } +} + +func ControllerReference[T client.Object](owner client.Object, cli client.Client) func(controlled T) error { + return func(controlled T) error { + return controllerutil.SetControllerReference(owner, controlled, cli.Scheme()) + } +} diff --git a/internal/controller/common/utils/kubernetes/ensure/deployment.go b/internal/controller/common/utils/kubernetes/ensure/deployment.go new file mode 100644 index 000000000..8e609945c --- /dev/null +++ b/internal/controller/common/utils/kubernetes/ensure/deployment.go @@ -0,0 +1,13 @@ +package ensure + +import ( + "github.com/securesign/operator/internal/controller/common/utils" + v1 "k8s.io/api/apps/v1" +) + +func Proxy() func(*v1.Deployment) error { + return func(dp *v1.Deployment) error { + utils.SetProxyEnvs(dp) + return nil + } +} diff --git a/internal/controller/common/utils/set_proxy.go b/internal/controller/common/utils/set_proxy.go index 6071b1f8a..a76300de8 100644 --- a/internal/controller/common/utils/set_proxy.go +++ b/internal/controller/common/utils/set_proxy.go @@ -1,13 +1,32 @@ package utils import ( + "reflect" + "slices" + "github.com/operator-framework/operator-lib/proxy" appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" ) // SetProxyEnvs set the standard environment variables for proxys "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY" func SetProxyEnvs(dep *appsv1.Deployment) { + proxyEnvs := proxy.ReadProxyVarsFromEnv() for i, container := range dep.Spec.Template.Spec.Containers { - dep.Spec.Template.Spec.Containers[i].Env = append(container.Env, proxy.ReadProxyVarsFromEnv()...) + for _, e := range proxyEnvs { + if index := slices.IndexFunc(container.Env, + func(envVar v1.EnvVar) bool { return e.Name == envVar.Name }, + ); index > -1 { + if reflect.DeepEqual(e, container.Env[index]) { + // variable already present + continue + } else { + // overwrite + dep.Spec.Template.Spec.Containers[i].Env[index] = e + } + } else { + dep.Spec.Template.Spec.Containers[i].Env = append(dep.Spec.Template.Spec.Containers[i].Env, e) + } + } } } diff --git a/internal/controller/common/utils/set_proxy_test.go b/internal/controller/common/utils/set_proxy_test.go index a16f65dbf..d29f5ad07 100644 --- a/internal/controller/common/utils/set_proxy_test.go +++ b/internal/controller/common/utils/set_proxy_test.go @@ -4,7 +4,6 @@ import ( "testing" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) @@ -28,6 +27,10 @@ func TestSetProxyEnvs(t *testing.T) { Name: "answer", Value: "42", }, + { + Name: "no_proxy", + Value: "toBeOverwritten", + }, } // Define a mock deployment @@ -49,7 +52,7 @@ func TestSetProxyEnvs(t *testing.T) { SetProxyEnvs(dep) g.Expect(dep.Spec.Template.Spec.Containers).ShouldNot(BeNil()) - g.Expect(dep.Spec.Template.Spec.Containers[0].Env).Should(HaveLen(1)) + g.Expect(dep.Spec.Template.Spec.Containers[0].Env).Should(HaveLen(2)) g.Expect(dep.Spec.Template.Spec.Containers[0].Env).Should(BeEquivalentTo(defaultEnv)) for _, e := range mockReadProxyVarsFromEnv() { @@ -58,9 +61,17 @@ func TestSetProxyEnvs(t *testing.T) { SetProxyEnvs(dep) - expectedEnvVars := append(defaultEnv, mockReadProxyVarsFromEnv()...) + expectedEnvVars := append(mockReadProxyVarsFromEnv(), corev1.EnvVar{ + Name: "answer", + Value: "42", + }) g.Expect(dep.Spec.Template.Spec.Containers).ShouldNot(BeNil()) g.Expect(dep.Spec.Template.Spec.Containers[0].Env).Should(HaveLen(7)) - g.Expect(dep.Spec.Template.Spec.Containers[0].Env).Should(BeEquivalentTo(expectedEnvVars)) + g.Expect(dep.Spec.Template.Spec.Containers[0].Env).Should(ConsistOf(expectedEnvVars)) + + // ensure no duplicates + SetProxyEnvs(dep) + g.Expect(dep.Spec.Template.Spec.Containers).ShouldNot(BeNil()) + g.Expect(dep.Spec.Template.Spec.Containers[0].Env).Should(HaveLen(7)) } diff --git a/internal/controller/tuf/actions/deployment.go b/internal/controller/tuf/actions/deployment.go index e1d39d5b4..4a68a3388 100644 --- a/internal/controller/tuf/actions/deployment.go +++ b/internal/controller/tuf/actions/deployment.go @@ -6,14 +6,22 @@ import ( rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" "github.com/securesign/operator/internal/controller/common/action" + "github.com/securesign/operator/internal/controller/common/utils" + "github.com/securesign/operator/internal/controller/common/utils/kubernetes" + "github.com/securesign/operator/internal/controller/common/utils/kubernetes/ensure" "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/labels" - tufutils "github.com/securesign/operator/internal/controller/tuf/utils" + "golang.org/x/exp/maps" + v1 "k8s.io/api/apps/v1" + core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) +const volumeName, containerName = "repository", "tuf-server" + func NewDeployAction() action.Action[*rhtasv1alpha1.Tuf] { return &deployAction{} } @@ -32,30 +40,28 @@ func (i deployAction) CanHandle(_ context.Context, tuf *rhtasv1alpha1.Tuf) bool } func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) *action.Result { - var ( - updated bool - err error - ) - labels := labels.For(ComponentName, DeploymentName, instance.Name) - dp := tufutils.CreateTufDeployment(instance, DeploymentName, RBACName, labels) - - if err = controllerutil.SetControllerReference(instance, dp, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Deployment: %w", err)) - } - - if updated, err = i.Ensure(ctx, dp); err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create TUF: %w", err), instance) + var ( + result controllerutil.OperationResult + err error + ) + if result, err = kubernetes.CreateOrUpdate(ctx, i.Client, + &v1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: DeploymentName, + Namespace: instance.Namespace, + }, + }, + i.createTufDeployment(instance, RBACName, labels), + ensure.ControllerReference[*v1.Deployment](instance, i.Client), + ensure.Labels[*v1.Deployment](maps.Keys(labels), labels), + ensure.Proxy(), + ); err != nil { + return i.Error(ctx, fmt.Errorf("could not create TUF: %w", err), instance) } - if updated { + if result != controllerutil.OperationResultNone { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{Type: constants.Ready, Status: metav1.ConditionFalse, Reason: constants.Creating, Message: "Deployment created"}) return i.StatusUpdate(ctx, instance) @@ -63,3 +69,75 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) * return i.Continue() } } + +func (i deployAction) createTufDeployment(instance *rhtasv1alpha1.Tuf, sa string, labels map[string]string) func(*v1.Deployment) error { + return func(dp *v1.Deployment) error { + + spec := &dp.Spec + spec.Replicas = utils.Pointer[int32](1) + spec.Selector = &metav1.LabelSelector{ + MatchLabels: labels, + } + spec.Strategy = v1.DeploymentStrategy{ + Type: "Recreate", + } + + template := &spec.Template + template.Labels = labels + template.Spec.ServiceAccountName = sa + + volume := kubernetes.FindVolumeByName(dp, volumeName) + if volume == nil { + template.Spec.Volumes = append(template.Spec.Volumes, core.Volume{Name: volumeName}) + volume = &template.Spec.Volumes[len(template.Spec.Volumes)-1] + } + volume.VolumeSource = core.VolumeSource{ + PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ + ClaimName: instance.Status.PvcName, + }, + } + + container := kubernetes.FindContainerByName(dp, containerName) + if container == nil { + template.Spec.Containers = append(template.Spec.Containers, core.Container{Name: containerName}) + container = &template.Spec.Containers[len(template.Spec.Containers)-1] + } + container.Image = constants.HttpServerImage + container.Ports = []core.ContainerPort{ + { + Protocol: core.ProtocolTCP, + ContainerPort: 8080, + }, + } + + container.VolumeMounts = []core.VolumeMount{ + { + Name: volumeName, + MountPath: "/var/www/html", + // let user upload manual update using `oc rsync` command + ReadOnly: false, + }} + + if container.LivenessProbe == nil { + container.LivenessProbe = &core.Probe{} + } + // server is running returning any status code (including 403 - noindex.html) + container.LivenessProbe.Exec = &core.ExecAction{Command: []string{"curl", "localhost:8080"}} + container.LivenessProbe.InitialDelaySeconds = 30 + container.LivenessProbe.PeriodSeconds = 10 + + if container.ReadinessProbe == nil { + container.ReadinessProbe = &core.Probe{} + } + container.ReadinessProbe.HTTPGet = &core.HTTPGetAction{ + Path: "/root.json", + Port: intstr.FromInt32(8080), + Scheme: "HTTP", + } + container.ReadinessProbe.InitialDelaySeconds = 10 + container.ReadinessProbe.PeriodSeconds = 10 + container.ReadinessProbe.FailureThreshold = 10 + + return nil + } +} diff --git a/internal/controller/tuf/utils/tuf_deployment.go b/internal/controller/tuf/utils/tuf_deployment.go deleted file mode 100644 index c0afaf964..000000000 --- a/internal/controller/tuf/utils/tuf_deployment.go +++ /dev/null @@ -1,95 +0,0 @@ -package utils - -import ( - "github.com/securesign/operator/api/v1alpha1" - "github.com/securesign/operator/internal/controller/common/utils" - "github.com/securesign/operator/internal/controller/constants" - apps "k8s.io/api/apps/v1" - core "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -func CreateTufDeployment(instance *v1alpha1.Tuf, dpName string, sa string, labels map[string]string) *apps.Deployment { - replicas := int32(1) - dep := &apps.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: dpName, - Namespace: instance.Namespace, - Labels: labels, - }, - Spec: apps.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: core.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: core.PodSpec{ - ServiceAccountName: sa, - Volumes: []core.Volume{ - { - Name: "repository", - VolumeSource: core.VolumeSource{ - PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ - ClaimName: instance.Status.PvcName, - }, - }, - }, - }, - Containers: []core.Container{ - { - Name: "tuf-server", - Image: constants.HttpServerImage, - Ports: []core.ContainerPort{ - { - Protocol: core.ProtocolTCP, - ContainerPort: 8080, - }, - }, - VolumeMounts: []core.VolumeMount{ - { - Name: "repository", - MountPath: "/var/www/html", - // let user to upload manual update using `oc rsync` command - ReadOnly: false, - }, - }, - LivenessProbe: &core.Probe{ - InitialDelaySeconds: 30, - PeriodSeconds: 10, - TimeoutSeconds: 1, - FailureThreshold: 3, - SuccessThreshold: 1, - ProbeHandler: core.ProbeHandler{ - // server is running returning any status code (including 403 - noindex.html) - Exec: &core.ExecAction{Command: []string{"curl", "localhost:8080"}}, - }, - }, - ReadinessProbe: &core.Probe{ - InitialDelaySeconds: 10, - PeriodSeconds: 10, - TimeoutSeconds: 1, - FailureThreshold: 10, - SuccessThreshold: 1, - ProbeHandler: core.ProbeHandler{ - HTTPGet: &core.HTTPGetAction{ - Port: intstr.FromInt32(8080), - Path: "/root.json", - }, - }, - }, - }, - }, - }, - }, - Strategy: apps.DeploymentStrategy{ - Type: "Recreate", - }, - }, - } - utils.SetProxyEnvs(dep) - return dep -}