Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move messaging configuration to file and deprecate deployMessagingService flag #1041

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Move messaging configuration to file and deprecate deployMessagingService flag #1041

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6c880dd
messaging config from file
nasark Jan 21, 2024
21cd053
deploy kafka depending on appname/hostname
nasark Jan 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions images/manageiq-base/container-assets/container_env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,35 @@ cat > ${APP_ROOT}/certs/v2_key << KEY
:key: ${encryption_key}
KEY


[[ -f /run/secrets/messaging/MESSAGING_HOSTNAME ]] && messaging_hostname_file=$(cat /run/secrets/messaging/MESSAGING_HOSTNAME)
[[ -f /run/secrets/messaging/MESSAGING_USERNAME ]] && messaging_username_file=$(cat /run/secrets/messaging/MESSAGING_USERNAME)
[[ -f /run/secrets/messaging/MESSAGING_PASSWORD ]] && messaging_password_file=$(cat /run/secrets/messaging/MESSAGING_PASSWORD)
[[ -f /run/secrets/messaging/MESSAGING_PORT ]] && messaging_port_file=$(cat /run/secrets/messaging/MESSAGING_PORT)
[[ -f /run/secrets/messaging/MESSAGING_SASL_MECHANISM ]] && messaging_sasl_mechanism_file=$(cat /run/secrets/messaging/MESSAGING_SASL_MECHANISM)
[[ -f /etc/pki/ca-trust/source/anchors/root.crt ]] && messaging_ca_path=/etc/pki/ca-trust/source/anchors/root.crt
messaging_hostname=${MESSAGING_HOSTNAME:-$messaging_hostname_file}
messaging_hostname=${messaging_hostname:-localhost}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this right? I thought localhost didn't work?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should probably default to the name of the service used to reach kafka

messaging_username=${MESSAGING_USERNAME:-$messaging_username_file}
messaging_password=${MESSAGING_PASSWORD:-$messaging_password_file}
messaging_port=${MESSAGING_PORT:-$messaging_port_file}
messaging_port=${messaging_port:-9093}
messaging_sasl_mechanism=${MESSAGING_SASL_MECHANISM:-$messaging_sasl_mechanism_file}
Comment on lines +56 to +62
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since the env vars won't be used, should they be removed from these conditional assignments? I see that the database env vars are still checked in these conditional assignments https://github.com/ManageIQ/manageiq-pods/blob/master/images/manageiq-base/container-assets/container_env#L12-L20 so I followed suite but didn't understand the reasoning

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We kept them more as a convenience for developers - if you run it locally, it's just a lot easier to pass them as -e on the command line

messaging_ca_path=${messaging_ca_path:-/etc/pki/ca-trust/source/anchors/ca.crt}

echo "== Writing messaging config =="
cat > ${APP_ROOT}/config/messaging.yml << KEY
---
production:
host: ${messaging_hostname}
port: ${messaging_port}
protocol: Kafka
encoding: json
username: ${messaging_username}
password: ${messaging_password}
sasl_mechanism: ${messaging_sasl_mechanism}
ssl: true
ca_file: ${messaging_ca_path}
KEY

echo "${GUID}" > ${APP_ROOT}/GUID
5 changes: 1 addition & 4 deletions images/manageiq-orchestrator/container-assets/entrypoint
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,10 +116,7 @@ EOS

check_svc_status ${MEMCACHED_SERVICE_HOST} ${MEMCACHED_SERVICE_PORT}
check_svc_status ${database_hostname} ${database_port}

if [ -n "$MESSAGING_HOSTNAME" ] && [ -n "$MESSAGING_PORT" ]; then
check_svc_status ${MESSAGING_HOSTNAME} ${MESSAGING_PORT}
fi
check_svc_status ${messaging_hostname} ${messaging_port}

check_deployment_status || exit 1

Expand Down
33 changes: 32 additions & 1 deletion manageiq-operator/api/v1alpha1/helpers/miq-components/kafka/kafka.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,37 @@ func renewKafkaCASecret(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *
return nil
}

func MessagingEnvSecret(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *runtime.Scheme) (*corev1.Secret, controllerutil.MutateFn) {
secretData := make(map[string]string)

secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "messaging-env-secret",
Namespace: cr.Namespace,
},
StringData: secretData,
}

f := func() error {
if err := controllerutil.SetControllerReference(cr, secret, scheme); err != nil {
return err
}

miqtool.AddLabels(map[string]string{"app": cr.Spec.AppName}, &secret.ObjectMeta)

secretData["hostname"] = cr.Spec.AppName + "-kafka-bootstrap"
secretData["username"] = cr.Spec.AppName + "-kafka-admin"
secretData["port"] = "9093"
secretData["sasl_mechanism"] = "SCRAM-SHA-512"

secret.StringData = secretData

return nil
}

return secret, f
}

func KafkaClusterSpec() map[string]interface{} {
return map[string]interface{}{
"kafka": map[string]interface{}{
Expand Down Expand Up @@ -440,7 +471,7 @@ func KafkaUser(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*unstructured.
Kind: "KafkaUser",
Version: "v1beta2",
})
kafkaUserCR.SetName(cr.Spec.AppName + "-user")
kafkaUserCR.SetName(cr.Spec.AppName + "-kafka-admin")
kafkaUserCR.SetNamespace(cr.Namespace)
kafkaUserCR.SetLabels(map[string]string{"strimzi.io/cluster": cr.Spec.AppName})

Expand Down
80 changes: 26 additions & 54 deletions manageiq-operator/api/v1alpha1/helpers/miq-components/orchestrator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,50 +111,6 @@ func orchestratorObjectName(cr *miqv1alpha1.ManageIQ) string {
return cr.Spec.AppName + "-orchestrator"
}

func addMessagingEnv(cr *miqv1alpha1.ManageIQ, c *corev1.Container, client client.Client) {
if !*cr.Spec.DeployMessagingService {
return
}

messagingEnv := []corev1.EnvVar{
corev1.EnvVar{
Name: "MESSAGING_HOSTNAME",
Value: cr.Spec.AppName + "-kafka-bootstrap",
},
corev1.EnvVar{
Name: "MESSAGING_PASSWORD",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: cr.Spec.AppName + "-user"},
Key: "password",
},
},
},
corev1.EnvVar{
Name: "MESSAGING_PORT",
Value: "9093",
},
corev1.EnvVar{
Name: "MESSAGING_TYPE",
Value: "kafka",
},
corev1.EnvVar{
Name: "MESSAGING_USERNAME",
Value: cr.Spec.AppName + "-user",
},
corev1.EnvVar{
Name: "MESSAGING_SASL_MECHANISM",
Value: "SCRAM-SHA-512",
},
}

for _, env := range messagingEnv {
c.Env = append(c.Env, env)
}

return
}

func addPostgresConfig(cr *miqv1alpha1.ManageIQ, d *appsv1.Deployment, client client.Client) {
d.Spec.Template.Spec.Containers[0].Env = addOrUpdateEnvVar(d.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "DATABASE_REGION", Value: cr.Spec.DatabaseRegion})
}
Expand Down Expand Up @@ -239,7 +195,6 @@ func OrchestratorDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl
},
}

addMessagingEnv(cr, &container, client)
err = addResourceReqs(cr.Spec.OrchestratorMemoryLimit, cr.Spec.OrchestratorMemoryRequest, cr.Spec.OrchestratorCpuLimit, cr.Spec.OrchestratorCpuRequest, &container)
if err != nil {
return nil, nil, err
Expand Down Expand Up @@ -299,15 +254,6 @@ func OrchestratorDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl
deployment.Spec.Template.Spec.Containers[0].Env = addOrUpdateEnvVar(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "UI_SSL_SECRET_NAME", Value: cr.Spec.InternalCertificatesSecret})
}

messagingCAPath := ""
if certSecret := InternalCertificatesSecret(cr, client); certSecret.Data["root_crt"] != nil && certSecret.Data["root_key"] != nil {
messagingCAPath = "/etc/pki/ca-trust/source/anchors/root.crt"
} else {
messagingCAPath = "/etc/pki/ca-trust/source/anchors/ca.crt"
}

deployment.Spec.Template.Spec.Containers[0].Env = addOrUpdateEnvVar(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "MESSAGING_SSL_CA", Value: messagingCAPath})

volumeMount := corev1.VolumeMount{Name: "encryption-key", MountPath: "/run/secrets/manageiq/application", ReadOnly: true}
deployment.Spec.Template.Spec.Containers[0].VolumeMounts = addOrUpdateVolumeMount(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, volumeMount)

Expand All @@ -326,6 +272,32 @@ func OrchestratorDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl
}}
deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: "database-secret", VolumeSource: corev1.VolumeSource{Secret: &databaseSecretVolumeSource}})

messagingVolumeMount := corev1.VolumeMount{Name: "messaging-env-secret", MountPath: "/run/secrets/messaging", ReadOnly: true}
deployment.Spec.Template.Spec.Containers[0].VolumeMounts = addOrUpdateVolumeMount(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, messagingVolumeMount)

messagingSecretProjection := &corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{Name: "messaging-env-secret"},
Items: []corev1.KeyToPath{
corev1.KeyToPath{Key: "hostname", Path: "MESSAGING_HOSTNAME"},
corev1.KeyToPath{Key: "port", Path: "MESSAGING_PORT"},
corev1.KeyToPath{Key: "sasl_mechanism", Path: "MESSAGING_SASL_MECHANISM"},
corev1.KeyToPath{Key: "username", Path: "MESSAGING_USERNAME"},
},
},
}
messagingSecretVolumeSource := addOrUpdateProjectedSecretVolumeSource("messaging-env-secret", deployment.Spec.Template.Spec.Volumes, messagingSecretProjection)
deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: "messaging-env-secret", VolumeSource: corev1.VolumeSource{Projected: &messagingSecretVolumeSource}})

kafkaUserSecretProjection := &corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{Name: cr.Spec.AppName + "-kafka-admin"},
Items: []corev1.KeyToPath{corev1.KeyToPath{Key: "password", Path: "MESSAGING_PASSWORD"}},
},
}
kafkaUserSecretVolumeSource := addOrUpdateProjectedSecretVolumeSource("messaging-env-secret", deployment.Spec.Template.Spec.Volumes, kafkaUserSecretProjection)
deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: "messaging-env-secret", VolumeSource: corev1.VolumeSource{Projected: &kafkaUserSecretVolumeSource}})

miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
Expand Down
23 changes: 17 additions & 6 deletions manageiq-operator/internal/controller/manageiq_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,11 +134,9 @@ func (r *ManageIQReconciler) Reconcile(ctx context.Context, request ctrl.Request
if e := r.generateMemcachedResources(miqInstance); e != nil {
return reconcile.Result{}, e
}
if *miqInstance.Spec.DeployMessagingService {
logger.Info("Reconciling the Kafka resources...")
if e := r.generateKafkaResources(miqInstance); e != nil {
return reconcile.Result{}, e
}
logger.Info("Reconciling the Kafka resources...")
if e := r.generateKafkaResources(miqInstance); e != nil {
return reconcile.Result{}, e
}
logger.Info("Reconciling the Orchestrator resources...")
if e := r.generateOrchestratorResources(miqInstance); e != nil {
Expand Down Expand Up @@ -526,6 +524,19 @@ func (r *ManageIQReconciler) generatePostgresqlResources(cr *miqv1alpha1.ManageI
}

func (r *ManageIQReconciler) generateKafkaResources(cr *miqv1alpha1.ManageIQ) error {
secret, mutateFunc := miqkafka.MessagingEnvSecret(cr, r.Client, r.Scheme)
if result, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, secret, mutateFunc); err != nil {
return err
} else if result != controllerutil.OperationResultNone {
logger.Info("Secret has been reconciled", "component", "kafka", "result", result)
}

hostName := string(secret.Data["hostname"])
if hostName != "manageiq-kafka-bootstrap" {
Comment on lines +534 to +535
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I followed the same methodology we use for postgresql but in this case the hostname also depends on cr.Spec.AppName. Would this be an issue? For example if a user decides to change the default app name for some reason then this wouldn't work, however this allows us to not deploy upstream kafka in the downstream scenario

logger.Info("External Kafka selected, skipping Kafka service reconciliation", "hostname", hostName)
return nil
}

if miqutilsv1alpha1.FindCatalogSourceByName(r.Client, "openshift-marketplace", "community-operators") != nil {
kafkaOperatorGroup, mutateFunc := miqkafka.KafkaOperatorGroup(cr, r.Scheme)
if result, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, kafkaOperatorGroup, mutateFunc); err != nil {
Expand Down Expand Up @@ -671,7 +682,7 @@ func (r *ManageIQReconciler) generateNetworkPolicies(cr *miqv1alpha1.ManageIQ) e
logger.Info("NetworkPolicy allow postgres has been reconciled", "component", "network_policy", "result", result)
}

if *cr.Spec.DeployMessagingService == true {
if cr.Spec.AppName == "manageiq" {
networkPolicyAllowKafka, mutateFunc := miqtool.NetworkPolicyAllowKafka(cr, r.Scheme, &r.Client)
if result, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, networkPolicyAllowKafka, mutateFunc); err != nil {
return err
Expand Down
Loading