Skip to content

Commit

Permalink
Merge pull request ManageIQ#1003 from bdunne/node_affinity
Browse files Browse the repository at this point in the history
Add Node affinity
  • Loading branch information
Fryguy authored Oct 26, 2023
2 parents afd62fe + 70cad9d commit 13812b3
Show file tree
Hide file tree
Showing 10 changed files with 132 additions and 36 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
tlstools "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/helpers/tlstools"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
routev1 "github.com/openshift/api/route/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -470,6 +471,8 @@ func HttpdDeployment(client client.Client, cr *miqv1alpha1.ManageIQ, scheme *run
deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: volumeName, VolumeSource: corev1.VolumeSource{Secret: &secretVolumeSource}})
}

miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
resource "k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -197,7 +198,7 @@ func ZookeeperService(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*corev1
return service, f
}

func KafkaDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
func KafkaDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
deploymentLabels := map[string]string{
"name": "kafka",
"app": cr.Spec.AppName,
Expand Down Expand Up @@ -315,13 +316,15 @@ func KafkaDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.
},
},
}
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

return deployment, f, nil
}

func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
deploymentLabels := map[string]string{
"name": "zookeeper",
"app": cr.Spec.AppName,
Expand Down Expand Up @@ -398,6 +401,8 @@ func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*app
},
},
}
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package miqtools

import (
miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -103,6 +104,8 @@ func NewMemcachedDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl
deployment.Spec.Template.Spec.Containers[0].Env = addOrUpdateEnvVar(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "MEMCACHED_EXTRA_PARAMETERS", Value: "-Z -o ssl_chain_cert=/root/server.crt -o ssl_key=/root/server.key -p 11211"})
}

miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
40 changes: 8 additions & 32 deletions manageiq-operator/api/v1alpha1/helpers/miq-components/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
Expand All @@ -14,7 +15,10 @@ import (
)

func ManageOperator(cr *miqv1alpha1.ManageIQ, client client.Client) (*appsv1.Deployment, controllerutil.MutateFn) {
deployment := operatorDeployment(cr, client)
podName := os.Getenv("POD_NAME")
pod := miqutilsv1alpha1.FindPodByName(client, cr.Namespace, podName)
replicaSet := miqutilsv1alpha1.FindReplicaSetByName(client, cr.Namespace, pod.ObjectMeta.OwnerReferences[0].Name)
deployment := miqutilsv1alpha1.FindDeploymentByName(client, cr.Namespace, replicaSet.ObjectMeta.OwnerReferences[0].Name)

f := func() error {
addAppLabel(cr.Spec.AppName, &deployment.ObjectMeta)
Expand Down Expand Up @@ -106,38 +110,10 @@ func ManageOperatorRoleBinding(cr *miqv1alpha1.ManageIQ, client client.Client) (
return operatorRoleBinding, f
}

func operatorPod(cr *miqv1alpha1.ManageIQ, client client.Client) *corev1.Pod {
operatorPodName := os.Getenv("POD_NAME")
podKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorPodName}
pod := &corev1.Pod{}
client.Get(context.TODO(), podKey, pod)

return pod
}

func operatorReplicaSet(cr *miqv1alpha1.ManageIQ, client client.Client) *appsv1.ReplicaSet {
pod := operatorPod(cr, client)
operatorReplicaSetName := pod.ObjectMeta.OwnerReferences[0].Name
replicaSetKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorReplicaSetName}
replicaSet := &appsv1.ReplicaSet{}
client.Get(context.TODO(), replicaSetKey, replicaSet)

return replicaSet
}

func operatorDeployment(cr *miqv1alpha1.ManageIQ, client client.Client) *appsv1.Deployment {
replicaSet := operatorReplicaSet(cr, client)
operatorDeploymentName := replicaSet.ObjectMeta.OwnerReferences[0].Name
deploymentKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorDeploymentName}
deployment := &appsv1.Deployment{}
client.Get(context.TODO(), deploymentKey, deployment)

return deployment
}

func operatorServiceAccount(cr *miqv1alpha1.ManageIQ, client client.Client) *corev1.ServiceAccount {
deployment := operatorDeployment(cr, client)
operatorServiceAccountName := deployment.Spec.Template.Spec.ServiceAccountName
podName := os.Getenv("POD_NAME")
pod := miqutilsv1alpha1.FindPodByName(client, cr.Namespace, podName)
operatorServiceAccountName := pod.Spec.ServiceAccountName
serviceAccountKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorServiceAccountName}
serviceAccount := &corev1.ServiceAccount{}
client.Get(context.TODO(), serviceAccountKey, serviceAccount)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
Expand Down Expand Up @@ -322,6 +323,8 @@ func OrchestratorDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl
}}
deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: "database-secret", VolumeSource: corev1.VolumeSource{Secret: &databaseSecretVolumeSource}})

miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
resource "k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -286,6 +287,8 @@ func PostgresqlDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme

addInternalCertificate(cr, deployment, client, "postgresql", "/opt/app-root/src/certificates")

miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
33 changes: 33 additions & 0 deletions manageiq-operator/api/v1alpha1/miqutils/find.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package miqutils

import (
"context"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)

func FindPodByName(client client.Client, namespace string, name string) *corev1.Pod {
podKey := types.NamespacedName{Namespace: namespace, Name: name}
pod := &corev1.Pod{}
client.Get(context.TODO(), podKey, pod)

return pod
}

func FindReplicaSetByName(client client.Client, namespace string, name string) *appsv1.ReplicaSet {
replicaSetKey := types.NamespacedName{Namespace: namespace, Name: name}
replicaSet := &appsv1.ReplicaSet{}
client.Get(context.TODO(), replicaSetKey, replicaSet)

return replicaSet
}

func FindDeploymentByName(client client.Client, namespace string, name string) *appsv1.Deployment {
deploymentKey := types.NamespacedName{Namespace: namespace, Name: name}
deployment := &appsv1.Deployment{}
client.Get(context.TODO(), deploymentKey, deployment)

return deployment
}
61 changes: 61 additions & 0 deletions manageiq-operator/api/v1alpha1/miqutils/node_affinity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package miqutils

import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"os"
"sigs.k8s.io/controller-runtime/pkg/client"
)

func OperatorNodeAffinityArchValues(deployment *appsv1.Deployment, client client.Client) []string {
podName := os.Getenv("POD_NAME")
pod := FindPodByName(client, deployment.ObjectMeta.Namespace, podName)
values := []string{"amd64"}

if pod.Spec.Affinity == nil {
// In case we don't find the operator pod (local testing) or it doesn't have affinities
return values
}

nodeSelectorTerms := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms

for _, selector := range nodeSelectorTerms {
for _, matchExpression := range selector.MatchExpressions {
if matchExpression.Key == "kubernetes.io/arch" {
values = matchExpression.Values
}
}
}

return values
}

func SetDeploymentNodeAffinity(deployment *appsv1.Deployment, client client.Client) {
operatorNodeAffinityArchValues := OperatorNodeAffinityArchValues(deployment, client)
if len(operatorNodeAffinityArchValues) == 0 {
// We're running local, can't find the operator pod, or it doesn't have any affinities to use as a template. Skip it.
return
}

matchExpression := corev1.NodeSelectorRequirement{
Key: "kubernetes.io/arch",
Operator: corev1.NodeSelectorOpIn,
Values: operatorNodeAffinityArchValues,
}

matchExpressions := []corev1.NodeSelectorRequirement{matchExpression}

nodeSelectorTerm := corev1.NodeSelectorTerm{
MatchExpressions: matchExpressions,
}

nodeSelectionTerms := []corev1.NodeSelectorTerm{nodeSelectorTerm}

deployment.Spec.Template.Spec.Affinity = &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: nodeSelectionTerms,
},
},
}
}
9 changes: 9 additions & 0 deletions manageiq-operator/config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@ spec:
labels:
name: manageiq-operator
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values:
- amd64
serviceAccountName: manageiq-operator
containers:
- name: manageiq-operator
Expand Down
4 changes: 2 additions & 2 deletions manageiq-operator/internal/controller/manageiq_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -563,7 +563,7 @@ func (r *ManageIQReconciler) generateKafkaResources(cr *miqv1alpha1.ManageIQ) er
logger.Info("Service has been reconciled", "component", "zookeeper", "result", result)
}

kafkaDeployment, mutateFunc, err := miqtool.KafkaDeployment(cr, r.Scheme)
kafkaDeployment, mutateFunc, err := miqtool.KafkaDeployment(cr, r.Client, r.Scheme)
if err != nil {
return err
}
Expand All @@ -574,7 +574,7 @@ func (r *ManageIQReconciler) generateKafkaResources(cr *miqv1alpha1.ManageIQ) er
logger.Info("Deployment has been reconciled", "component", "kafka", "result", result)
}

zookeeperDeployment, mutateFunc, err := miqtool.ZookeeperDeployment(cr, r.Scheme)
zookeeperDeployment, mutateFunc, err := miqtool.ZookeeperDeployment(cr, r.Client, r.Scheme)
if err != nil {
return err
}
Expand Down

0 comments on commit 13812b3

Please sign in to comment.