Merge pull request #926 from olliewalsh/rabbitipv6_18.0.0

[18.0.0-proposed] Fix rabbitmq IPv6 with TLS/FIPS

config/rbac/role.yaml

@@ -205,6 +205,14 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - config.openshift.io
+  resources:
+  - networks
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

controllers/core/openstackcontrolplane_controller.go

@@ -111,6 +111,7 @@ func (r *OpenStackControlPlaneReconciler) GetLogger(ctx context.Context) logr.Lo
 //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete;
 //+kubebuilder:rbac:groups=cert-manager.io,resources=issuers,verbs=get;list;watch;create;update;patch;delete;
 //+kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete;
+//+kubebuilder:rbac:groups=config.openshift.io,resources=networks,verbs=get;list;watch;
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20240624132705-6c8da3c0bbfd
 	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240709171418-83ff4f73c986
 	github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240624132705-6c8da3c0bbfd
-	github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240624132705-6c8da3c0bbfd
+	github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240709171418-83ff4f73c986
 	github.com/openstack-k8s-operators/manila-operator/api v0.3.1-0.20240709222938-272b1b93e719
 	github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240709194146-eb1cfc2518c5
 	github.com/openstack-k8s-operators/neutron-operator/api v0.3.1-0.20240710004943-45c853971543

go.sum

@@ -126,8 +126,8 @@ github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.2024062
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.20240624132705-6c8da3c0bbfd/go.mod h1:zuPcZ5Kopr15AdfxvA0xqKIIGCZ0XbSe/0VHNKuvbEE=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240624132705-6c8da3c0bbfd h1:MY3MDe11c9R/kp0ALVeaWHIdRpbQh9Xs3ym/Z/KBBlU=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240624132705-6c8da3c0bbfd/go.mod h1:v9iFrR8J5fZACS9W5pZau/4lwyWs/YmO4ezpDeoEFKU=
-github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240624132705-6c8da3c0bbfd h1:FDN/wK2+B+9IwIpuY8K1CCLjqrzSLVXuqn9PFWPX+LM=
-github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240624132705-6c8da3c0bbfd/go.mod h1:0h76CxD9g0z2Hk7fGFOZcjnzT1tQQ/yRNv3OXng+S/A=
+github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240709171418-83ff4f73c986 h1:DS6K5o+Mb3ghNsf/6als1+LjpqSAknvqetuDSUdxV9M=
+github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240709171418-83ff4f73c986/go.mod h1:0h76CxD9g0z2Hk7fGFOZcjnzT1tQQ/yRNv3OXng+S/A=
 github.com/openstack-k8s-operators/manila-operator/api v0.3.1-0.20240709222938-272b1b93e719 h1:SZqwffeJXG73gYiMab7yPtrMJkA2mtOatMw8hsSpjGg=
 github.com/openstack-k8s-operators/manila-operator/api v0.3.1-0.20240709222938-272b1b93e719/go.mod h1:Vc61/6I9y+fBCw6k0HVi29mStMEvq0G1IMquFQJfGhM=
 github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240709194146-eb1cfc2518c5 h1:kRO+Q9xd4YChb4WZtGmbNoaU8dkYsAHNCqDsSWXAA5A=

main.go

@@ -52,6 +52,7 @@ import (
 	placementv1 "github.com/openstack-k8s-operators/placement-operator/api/v1beta1"
 	swiftv1 "github.com/openstack-k8s-operators/swift-operator/api/v1beta1"
 	telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1"
+
 	// Note(lpiwowar): Please, do not remove! This import is necessary in order
 	// to make the test-operator part of the openstack-operator-index.
 	_ "github.com/openstack-k8s-operators/test-operator/api/v1beta1"
@@ -75,6 +76,7 @@ import (
 	corev1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1"
 	dataplanev1 "github.com/openstack-k8s-operators/openstack-operator/apis/dataplane/v1beta1"
 
+	ocp_configv1 "github.com/openshift/api/config/v1"
 	clientcontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/client"
 	corecontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/core"
 	dataplanecontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/dataplane"
@@ -117,6 +119,7 @@ func init() {
 	utilruntime.Must(routev1.AddToScheme(scheme))
 	utilruntime.Must(certmgrv1.AddToScheme(scheme))
 	utilruntime.Must(barbicanv1.AddToScheme(scheme))
+	utilruntime.Must(ocp_configv1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 }
 

pkg/openstack/rabbitmq.go

@@ -155,24 +155,43 @@ func reconcileRabbitMQ(
 		},
 	}
 
+	IPv6Enabled, err := ocp.HasIPv6ClusterNetwork(ctx, helper)
+	if err != nil {
+		return mqFailed, ctrl.Result{}, err
+	}
+	inetFamily := "inet"
+	inetProtocol := "tcp"
+	tlsArgs := ""
+	fipsArgs := ""
+	if IPv6Enabled {
+		inetFamily = "inet6"
+	}
+	erlangInetConfig := fmt.Sprintf("{%s,true}.\n", inetFamily)
+
 	if instance.Spec.TLS.PodLevel.Enabled {
+		inetProtocol = "tls"
+		tlsArgs = "-ssl_dist_optfile /etc/rabbitmq/inter-node-tls.config"
 		fipsEnabled, err := ocp.IsFipsCluster(ctx, helper)
 		if err != nil {
 			return mqFailed, ctrl.Result{}, err
 		}
-		clusterNodeTLSArgs := "-proto_dist inet_tls -ssl_dist_optfile /etc/rabbitmq/inter-node-tls.config"
 		if fipsEnabled {
-			clusterNodeTLSArgs += " -crypto fips_mode true"
+			fipsArgs = "-crypto fips_mode true"
 		}
-
-		envVars = append(envVars, corev1.EnvVar{
-			Name:  "RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS",
-			Value: clusterNodeTLSArgs,
-		}, corev1.EnvVar{
-			Name:  "RABBITMQ_CTL_ERL_ARGS",
-			Value: clusterNodeTLSArgs,
-		})
 	}
+	envVars = append(envVars, corev1.EnvVar{
+		Name: "RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS",
+		Value: fmt.Sprintf(
+			"-kernel inetrc '/etc/rabbitmq/erl_inetrc' -proto_dist %s_%s %s %s",
+			inetFamily,
+			inetProtocol,
+			tlsArgs,
+			fipsArgs,
+		),
+	}, corev1.EnvVar{
+		Name:  "RABBITMQ_CTL_ERL_ARGS",
+		Value: fmt.Sprintf("-proto_dist %s_%s %s", inetFamily, inetProtocol, tlsArgs),
+	})
 
 	cms := []util.Template{
 		{
@@ -206,7 +225,7 @@ func reconcileRabbitMQ(
 		},
 	}
 
-	err := configmap.EnsureConfigMaps(ctx, helper, instance, cms, nil)
+	err = configmap.EnsureConfigMaps(ctx, helper, instance, cms, nil)
 	if err != nil {
 		Log.Error(err, "Unable to create rabbitmq config maps")
 		return mqFailed, ctrl.Result{}, err
@@ -344,6 +363,8 @@ func reconcileRabbitMQ(
 			rabbitmq.Spec.Rabbitmq.AdditionalConfig = strings.Join(settings, "\n")
 		}
 
+		rabbitmq.Spec.Rabbitmq.ErlangInetConfig = erlangInetConfig
+		rabbitmq.Spec.Rabbitmq.AdvancedConfig = ""
 		if tlsCert != "" {
 			rabbitmq.Spec.TLS.CaSecretName = tlsCert
 			rabbitmq.Spec.TLS.SecretName = tlsCert

tests/functional/ctlplane/suite_test.go

@@ -59,6 +59,7 @@ import (
 	client_ctrl "github.com/openstack-k8s-operators/openstack-operator/controllers/client"
 	core_ctrl "github.com/openstack-k8s-operators/openstack-operator/controllers/core"
 
+	ocp_configv1 "github.com/openshift/api/config/v1"
 	infra_test "github.com/openstack-k8s-operators/infra-operator/apis/test/helpers"
 	keystone_test "github.com/openstack-k8s-operators/keystone-operator/api/test/helpers"
 	certmanager_test "github.com/openstack-k8s-operators/lib-common/modules/certmanager/test/helpers"
@@ -173,6 +174,8 @@ var _ = BeforeSuite(func() {
 	Expect(err).ShouldNot(HaveOccurred())
 	certmgrv1CRDs, err := test.GetOpenShiftCRDDir("cert-manager/v1", gomod)
 	Expect(err).ShouldNot(HaveOccurred())
+	ocpconfigv1CRDs, err := test.GetOpenShiftCRDDir("config/v1", gomod)
+	Expect(err).ShouldNot(HaveOccurred())
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
@@ -199,6 +202,7 @@ var _ = BeforeSuite(func() {
 			barbicanv1CRDs,
 			rabbitmqv2CRDs,
 			certmgrv1CRDs,
+			ocpconfigv1CRDs,
 		},
 		ErrorIfCRDPathMissing: true,
 		WebhookInstallOptions: envtest.WebhookInstallOptions{
@@ -265,6 +269,8 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	err = networkv1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
+	err = ocp_configv1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
 
 	//+kubebuilder:scaffold:scheme
 
@@ -286,6 +292,8 @@ var _ = BeforeSuite(func() {
 	ovn = ovn_test.NewTestHelper(ctx, k8sClient, timeout, interval, logger)
 	Expect(ovn).NotTo(BeNil())
 
+	th.CreateClusterNetworkConfig()
+
 	// Start the controller-manager if goroutine
 	webhookInstallOptions := &testEnv.WebhookInstallOptions
 	k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{