From aeac8af99d9bdbb7637227b791425750357c15f3 Mon Sep 17 00:00:00 2001 From: Brandon Dunne Date: Tue, 17 Oct 2023 15:40:13 -0400 Subject: [PATCH 1/4] Refactor pod/deployment lookup logic so that it can be reused --- .../helpers/miq-components/operator.go | 40 ++++--------------- .../api/v1alpha1/miqutils/find.go | 33 +++++++++++++++ 2 files changed, 41 insertions(+), 32 deletions(-) create mode 100644 manageiq-operator/api/v1alpha1/miqutils/find.go diff --git a/manageiq-operator/api/v1alpha1/helpers/miq-components/operator.go b/manageiq-operator/api/v1alpha1/helpers/miq-components/operator.go index 341fce21f..7a1db031d 100644 --- a/manageiq-operator/api/v1alpha1/helpers/miq-components/operator.go +++ b/manageiq-operator/api/v1alpha1/helpers/miq-components/operator.go @@ -4,6 +4,7 @@ import ( "context" miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1" + miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -14,7 +15,10 @@ import ( ) func ManageOperator(cr *miqv1alpha1.ManageIQ, client client.Client) (*appsv1.Deployment, controllerutil.MutateFn) { - deployment := operatorDeployment(cr, client) + podName := os.Getenv("POD_NAME") + pod := miqutilsv1alpha1.FindPodByName(client, cr.Namespace, podName) + replicaSet := miqutilsv1alpha1.FindReplicaSetByName(client, cr.Namespace, pod.ObjectMeta.OwnerReferences[0].Name) + deployment := miqutilsv1alpha1.FindDeploymentByName(client, cr.Namespace, replicaSet.ObjectMeta.OwnerReferences[0].Name) f := func() error { addAppLabel(cr.Spec.AppName, &deployment.ObjectMeta) @@ -106,38 +110,10 @@ func ManageOperatorRoleBinding(cr *miqv1alpha1.ManageIQ, client client.Client) ( return operatorRoleBinding, f } -func operatorPod(cr *miqv1alpha1.ManageIQ, client client.Client) *corev1.Pod { - operatorPodName := os.Getenv("POD_NAME") - podKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorPodName} - pod := &corev1.Pod{} - client.Get(context.TODO(), podKey, pod) - - return pod -} - -func operatorReplicaSet(cr *miqv1alpha1.ManageIQ, client client.Client) *appsv1.ReplicaSet { - pod := operatorPod(cr, client) - operatorReplicaSetName := pod.ObjectMeta.OwnerReferences[0].Name - replicaSetKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorReplicaSetName} - replicaSet := &appsv1.ReplicaSet{} - client.Get(context.TODO(), replicaSetKey, replicaSet) - - return replicaSet -} - -func operatorDeployment(cr *miqv1alpha1.ManageIQ, client client.Client) *appsv1.Deployment { - replicaSet := operatorReplicaSet(cr, client) - operatorDeploymentName := replicaSet.ObjectMeta.OwnerReferences[0].Name - deploymentKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorDeploymentName} - deployment := &appsv1.Deployment{} - client.Get(context.TODO(), deploymentKey, deployment) - - return deployment -} - func operatorServiceAccount(cr *miqv1alpha1.ManageIQ, client client.Client) *corev1.ServiceAccount { - deployment := operatorDeployment(cr, client) - operatorServiceAccountName := deployment.Spec.Template.Spec.ServiceAccountName + podName := os.Getenv("POD_NAME") + pod := miqutilsv1alpha1.FindPodByName(client, cr.Namespace, podName) + operatorServiceAccountName := pod.Spec.ServiceAccountName serviceAccountKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorServiceAccountName} serviceAccount := &corev1.ServiceAccount{} client.Get(context.TODO(), serviceAccountKey, serviceAccount) diff --git a/manageiq-operator/api/v1alpha1/miqutils/find.go b/manageiq-operator/api/v1alpha1/miqutils/find.go new file mode 100644 index 000000000..e76a5926c --- /dev/null +++ b/manageiq-operator/api/v1alpha1/miqutils/find.go @@ -0,0 +1,33 @@ +package miqutils + +import ( + "context" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func FindPodByName(client client.Client, namespace string, name string) *corev1.Pod { + podKey := types.NamespacedName{Namespace: namespace, Name: name} + pod := &corev1.Pod{} + client.Get(context.TODO(), podKey, pod) + + return pod +} + +func FindReplicaSetByName(client client.Client, namespace string, name string) *appsv1.ReplicaSet { + replicaSetKey := types.NamespacedName{Namespace: namespace, Name: name} + replicaSet := &appsv1.ReplicaSet{} + client.Get(context.TODO(), replicaSetKey, replicaSet) + + return replicaSet +} + +func FindDeploymentByName(client client.Client, namespace string, name string) *appsv1.Deployment { + deploymentKey := types.NamespacedName{Namespace: namespace, Name: name} + deployment := &appsv1.Deployment{} + client.Get(context.TODO(), deploymentKey, deployment) + + return deployment +} From f030f700b7c6a4fa17cd00257123515a7528414e Mon Sep 17 00:00:00 2001 From: Brandon Dunne Date: Tue, 17 Oct 2023 18:35:18 -0400 Subject: [PATCH 2/4] Use values from the operator to set Arch NodeAffinities on deployments --- .../v1alpha1/helpers/miq-components/httpd.go | 3 + .../v1alpha1/helpers/miq-components/kafka.go | 9 ++- .../helpers/miq-components/memcached.go | 3 + .../helpers/miq-components/orchestrator.go | 3 + .../helpers/miq-components/postgresql.go | 3 + .../api/v1alpha1/miqutils/node_affinity.go | 60 +++++++++++++++++++ .../controller/manageiq_controller.go | 4 +- 7 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 manageiq-operator/api/v1alpha1/miqutils/node_affinity.go diff --git a/manageiq-operator/api/v1alpha1/helpers/miq-components/httpd.go b/manageiq-operator/api/v1alpha1/helpers/miq-components/httpd.go index 6cc50db3c..d9567d8c1 100644 --- a/manageiq-operator/api/v1alpha1/helpers/miq-components/httpd.go +++ b/manageiq-operator/api/v1alpha1/helpers/miq-components/httpd.go @@ -8,6 +8,7 @@ import ( miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1" tlstools "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/helpers/tlstools" + miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils" routev1 "github.com/openshift/api/route/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -470,6 +471,8 @@ func HttpdDeployment(client client.Client, cr *miqv1alpha1.ManageIQ, scheme *run deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: volumeName, VolumeSource: corev1.VolumeSource{Secret: &secretVolumeSource}}) } + miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client) + return nil } diff --git a/manageiq-operator/api/v1alpha1/helpers/miq-components/kafka.go b/manageiq-operator/api/v1alpha1/helpers/miq-components/kafka.go index ba023c96e..2fa01646b 100644 --- a/manageiq-operator/api/v1alpha1/helpers/miq-components/kafka.go +++ b/manageiq-operator/api/v1alpha1/helpers/miq-components/kafka.go @@ -4,6 +4,7 @@ import ( "context" miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1" + miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" resource "k8s.io/apimachinery/pkg/api/resource" @@ -197,7 +198,7 @@ func ZookeeperService(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*corev1 return service, f } -func KafkaDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) { +func KafkaDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) { deploymentLabels := map[string]string{ "name": "kafka", "app": cr.Spec.AppName, @@ -315,13 +316,15 @@ func KafkaDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1. }, }, } + miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client) + return nil } return deployment, f, nil } -func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) { +func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) { deploymentLabels := map[string]string{ "name": "zookeeper", "app": cr.Spec.AppName, @@ -398,6 +401,8 @@ func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*app }, }, } + miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client) + return nil } diff --git a/manageiq-operator/api/v1alpha1/helpers/miq-components/memcached.go b/manageiq-operator/api/v1alpha1/helpers/miq-components/memcached.go index 3d369567a..cbecfb164 100644 --- a/manageiq-operator/api/v1alpha1/helpers/miq-components/memcached.go +++ b/manageiq-operator/api/v1alpha1/helpers/miq-components/memcached.go @@ -2,6 +2,7 @@ package miqtools import ( miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1" + miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -103,6 +104,8 @@ func NewMemcachedDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl deployment.Spec.Template.Spec.Containers[0].Env = addOrUpdateEnvVar(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "MEMCACHED_EXTRA_PARAMETERS", Value: "-Z -o ssl_chain_cert=/root/server.crt -o ssl_key=/root/server.key -p 11211"}) } + miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client) + return nil } diff --git a/manageiq-operator/api/v1alpha1/helpers/miq-components/orchestrator.go b/manageiq-operator/api/v1alpha1/helpers/miq-components/orchestrator.go index 099ccb60a..7fc124acb 100644 --- a/manageiq-operator/api/v1alpha1/helpers/miq-components/orchestrator.go +++ b/manageiq-operator/api/v1alpha1/helpers/miq-components/orchestrator.go @@ -4,6 +4,7 @@ import ( "context" miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1" + miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -322,6 +323,8 @@ func OrchestratorDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl }} deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: "database-secret", VolumeSource: corev1.VolumeSource{Secret: &databaseSecretVolumeSource}}) + miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client) + return nil } diff --git a/manageiq-operator/api/v1alpha1/helpers/miq-components/postgresql.go b/manageiq-operator/api/v1alpha1/helpers/miq-components/postgresql.go index 0b21ff303..78925e211 100644 --- a/manageiq-operator/api/v1alpha1/helpers/miq-components/postgresql.go +++ b/manageiq-operator/api/v1alpha1/helpers/miq-components/postgresql.go @@ -4,6 +4,7 @@ import ( "context" miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1" + miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" resource "k8s.io/apimachinery/pkg/api/resource" @@ -286,6 +287,8 @@ func PostgresqlDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme addInternalCertificate(cr, deployment, client, "postgresql", "/opt/app-root/src/certificates") + miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client) + return nil } diff --git a/manageiq-operator/api/v1alpha1/miqutils/node_affinity.go b/manageiq-operator/api/v1alpha1/miqutils/node_affinity.go new file mode 100644 index 000000000..e5455315b --- /dev/null +++ b/manageiq-operator/api/v1alpha1/miqutils/node_affinity.go @@ -0,0 +1,60 @@ +package miqutils + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "os" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func OperatorNodeAffinityArchValues(deployment *appsv1.Deployment, client client.Client) []string { + podName := os.Getenv("POD_NAME") + pod := FindPodByName(client, deployment.ObjectMeta.Namespace, podName) + + if pod.Spec.Affinity == nil { + // In case we don't find the operator pod (local testing) or it doesn't have affinities + return []string{} + } + nodeSelectorTerms := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + + for _, selector := range nodeSelectorTerms { + for _, matchExpression := range selector.MatchExpressions { + if matchExpression.Key == "kubernetes.io/arch" { + return matchExpression.Values + } + } + } + + // We should never get here, but the compiler requires it + return []string{} +} + +func SetDeploymentNodeAffinity(deployment *appsv1.Deployment, client client.Client) { + operatorNodeAffinityArchValues := OperatorNodeAffinityArchValues(deployment, client) + if len(operatorNodeAffinityArchValues) == 0 { + // We're running local, can't find the operator pod, or it doesn't have any affinities to use as a template. Skip it. + return + } + + matchExpression := corev1.NodeSelectorRequirement{ + Key: "kubernetes.io/arch", + Operator: corev1.NodeSelectorOpIn, + Values: operatorNodeAffinityArchValues, + } + + matchExpressions := []corev1.NodeSelectorRequirement{matchExpression} + + nodeSelectorTerm := corev1.NodeSelectorTerm{ + MatchExpressions: matchExpressions, + } + + nodeSelectionTerms := []corev1.NodeSelectorTerm{nodeSelectorTerm} + + deployment.Spec.Template.Spec.Affinity = &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: nodeSelectionTerms, + }, + }, + } +} diff --git a/manageiq-operator/internal/controller/manageiq_controller.go b/manageiq-operator/internal/controller/manageiq_controller.go index ee10f4ccb..9bf358328 100644 --- a/manageiq-operator/internal/controller/manageiq_controller.go +++ b/manageiq-operator/internal/controller/manageiq_controller.go @@ -563,7 +563,7 @@ func (r *ManageIQReconciler) generateKafkaResources(cr *miqv1alpha1.ManageIQ) er logger.Info("Service has been reconciled", "component", "zookeeper", "result", result) } - kafkaDeployment, mutateFunc, err := miqtool.KafkaDeployment(cr, r.Scheme) + kafkaDeployment, mutateFunc, err := miqtool.KafkaDeployment(cr, r.Client, r.Scheme) if err != nil { return err } @@ -574,7 +574,7 @@ func (r *ManageIQReconciler) generateKafkaResources(cr *miqv1alpha1.ManageIQ) er logger.Info("Deployment has been reconciled", "component", "kafka", "result", result) } - zookeeperDeployment, mutateFunc, err := miqtool.ZookeeperDeployment(cr, r.Scheme) + zookeeperDeployment, mutateFunc, err := miqtool.ZookeeperDeployment(cr, r.Client, r.Scheme) if err != nil { return err } From 7d0c8f175e8fc8b08beb8e4db7f79d07270cdadc Mon Sep 17 00:00:00 2001 From: Brandon Dunne Date: Tue, 17 Oct 2023 18:38:17 -0400 Subject: [PATCH 3/4] Add NodeAffinities to the operator deployment --- manageiq-operator/config/manager/manager.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/manageiq-operator/config/manager/manager.yaml b/manageiq-operator/config/manager/manager.yaml index 9c2dd36ea..ff1bdd827 100644 --- a/manageiq-operator/config/manager/manager.yaml +++ b/manageiq-operator/config/manager/manager.yaml @@ -12,6 +12,15 @@ spec: labels: name: manageiq-operator spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 serviceAccountName: manageiq-operator containers: - name: manageiq-operator From 70cad9d952b1515021516c7e33a73b76d9d05dcb Mon Sep 17 00:00:00 2001 From: Brandon Dunne Date: Mon, 23 Oct 2023 14:38:02 -0400 Subject: [PATCH 4/4] default to amd64 only --- manageiq-operator/api/v1alpha1/miqutils/node_affinity.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/manageiq-operator/api/v1alpha1/miqutils/node_affinity.go b/manageiq-operator/api/v1alpha1/miqutils/node_affinity.go index e5455315b..4b955a8ab 100644 --- a/manageiq-operator/api/v1alpha1/miqutils/node_affinity.go +++ b/manageiq-operator/api/v1alpha1/miqutils/node_affinity.go @@ -10,23 +10,24 @@ import ( func OperatorNodeAffinityArchValues(deployment *appsv1.Deployment, client client.Client) []string { podName := os.Getenv("POD_NAME") pod := FindPodByName(client, deployment.ObjectMeta.Namespace, podName) + values := []string{"amd64"} if pod.Spec.Affinity == nil { // In case we don't find the operator pod (local testing) or it doesn't have affinities - return []string{} + return values } + nodeSelectorTerms := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms for _, selector := range nodeSelectorTerms { for _, matchExpression := range selector.MatchExpressions { if matchExpression.Key == "kubernetes.io/arch" { - return matchExpression.Values + values = matchExpression.Values } } } - // We should never get here, but the compiler requires it - return []string{} + return values } func SetDeploymentNodeAffinity(deployment *appsv1.Deployment, client client.Client) {