Merge pull request #135 from cschwede/dataplane-devices

Add Dataplane devices to rings

api/go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

api/go.sum

@@ -2,9 +2,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -44,11 +41,10 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 h1:2XF1Vzq06X+inNqgJ9tRnGuw+ZVCB3FazXODD6JE1R8=
+github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -126,7 +122,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

api/v1beta1/swiftring_types.go

@@ -94,6 +94,15 @@ type SwiftRingList struct {
 	Items           []SwiftRing `json:"items"`
 }
 
+
+type SwiftDisk struct {
+	Device string `json:"device"`
+	Path   string `json:"path"`
+	Weight int32  `json:"weight"`
+	Region int32  `json:"region"`
+	Zone   int32  `json:"zone"`
+}
+
 func init() {
 	SchemeBuilder.Register(&SwiftRing{}, &SwiftRingList{})
 }

config/rbac/role.yaml

@@ -119,6 +119,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - dataplane.openstack.org
+  resources:
+  - openstackdataplanenodesets
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - k8s.cni.cncf.io
   resources:

controllers/swiftring_controller.go

@@ -35,6 +35,7 @@ import (
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/job"
 
+	dataplanev1 "github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1"
 	swiftv1beta1 "github.com/openstack-k8s-operators/swift-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/swift-operator/pkg/swift"
 	"github.com/openstack-k8s-operators/swift-operator/pkg/swiftring"
@@ -151,49 +152,55 @@ func (r *SwiftRingReconciler) reconcileNormal(ctx context.Context, instance *swi
 
 	serviceLabels := swiftring.Labels()
 
-	// Create a ConfigMap populated with content from templates/
-	envVars := make(map[string]env.Setter)
-	tpl := swiftring.ConfigMapTemplates(instance, serviceLabels)
-	err := configmap.EnsureConfigMaps(ctx, helper, instance, tpl, &envVars)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
-
 	// Swift ring init job - start
 	if instance.Status.Hash == nil {
 		instance.Status.Hash = map[string]string{}
 	}
-	ringCreateHash := instance.Status.Hash[swiftv1beta1.RingCreateHash]
 
-	// Check if the device list ConfigMap did change and if so, delete the
-	// rebalance Job. This will result in a new Job that rebalances with
-	// the updated device list
-	_, deviceListHash, err := configmap.GetConfigMapAndHashWithName(ctx, helper, swift.DeviceConfigMapName, instance.Namespace)
+	deviceList, deviceListHash, err := swiftring.DeviceList(ctx, helper, instance)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
+
 	if instance.Status.Hash[swiftv1beta1.DeviceListHash] != deviceListHash {
-		if err := job.DeleteJob(ctx, helper, instance.Name+"-rebalance", instance.Namespace); err != nil {
+		// Create or update the devicelist ConfigMap
+		envVars := make(map[string]env.Setter)
+		tpl := swiftring.ConfigMapTemplates(instance, serviceLabels, deviceList)
+		err = configmap.EnsureConfigMaps(ctx, helper, instance, tpl, &envVars)
+		if err != nil {
 			return ctrl.Result{}, err
 		}
+
+		// Delete a possibly still existing job that finished to re-run the job
+		j, err := job.GetJobWithName(ctx, helper, instance.Name+"-rebalance", instance.Namespace)
+		if err != nil {
+			if !apierrors.IsNotFound(err) {
+				return ctrl.Result{}, err
+			}
+		} else {
+			if j.Status.Active == 0 {
+				err = job.DeleteJob(ctx, helper, instance.Name+"-rebalance", instance.Namespace)
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+			}
+		}
+
 		instance.Status.Hash[swiftv1beta1.RingCreateHash] = ""
 		instance.Status.Hash[swiftv1beta1.DeviceListHash] = deviceListHash
 		if err := r.Status().Update(ctx, instance); err != nil {
 			return ctrl.Result{}, err
 		}
 	}
 
-	ringCreateJob := job.NewJob(swiftring.GetRingJob(instance, serviceLabels), swiftv1beta1.RingCreateHash, false, 5*time.Second, ringCreateHash)
+	ringCreateJob := job.NewJob(swiftring.GetRingJob(instance, serviceLabels), "rebalance", false, 5*time.Second, instance.Status.Hash[swiftv1beta1.RingCreateHash])
 	ctrlResult, err := ringCreateJob.DoJob(ctx, helper)
 	if (ctrlResult != ctrl.Result{}) {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.ReadyCondition,
 			condition.RequestedReason,
 			condition.SeverityInfo,
 			condition.ReadyInitMessage))
-		if err := r.Status().Update(ctx, instance); err != nil {
-			return ctrl.Result{}, err
-		}
 		return ctrlResult, nil
 	}
 	if err != nil {
@@ -210,7 +217,6 @@ func (r *SwiftRingReconciler) reconcileNormal(ctx context.Context, instance *swi
 
 	if ringCreateJob.HasChanged() {
 		instance.Status.Hash[swiftv1beta1.RingCreateHash] = ringCreateJob.GetHash()
-		instance.Status.Hash[swiftv1beta1.DeviceListHash] = deviceListHash
 		if err := r.Status().Update(ctx, instance); err != nil {
 			return ctrl.Result{}, err
 		}
@@ -255,24 +261,20 @@ func (r *SwiftRingReconciler) reconcileDelete(ctx context.Context, instance *swi
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *SwiftRingReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	deviceConfigMapFilter := func(ctx context.Context, o client.Object) []reconcile.Request {
+	swiftRingFilter := func(ctx context.Context, o client.Object) []reconcile.Request {
 		result := []reconcile.Request{}
-		if o.GetName() == swift.DeviceConfigMapName {
-			// There should be only one SwiftRing instance within
-			// the Namespace - that needs to be reconciled
-			swiftRings := &swiftv1beta1.SwiftRingList{}
-			listOpts := []client.ListOption{client.InNamespace(o.GetNamespace())}
-			err := r.Client.List(context.Background(), swiftRings, listOpts...)
-			if err != nil {
-				return nil
-			}
-			for _, cr := range swiftRings.Items {
-				name := client.ObjectKey{
-					Namespace: o.GetNamespace(),
-					Name:      cr.Name,
-				}
-				result = append(result, reconcile.Request{NamespacedName: name})
+		swiftRings := &swiftv1beta1.SwiftRingList{}
+		listOpts := []client.ListOption{client.InNamespace(o.GetNamespace())}
+		err := r.Client.List(context.Background(), swiftRings, listOpts...)
+		if err != nil {
+			return nil
+		}
+		for _, cr := range swiftRings.Items {
+			name := client.ObjectKey{
+				Namespace: o.GetNamespace(),
+				Name:      cr.Name,
 			}
+			result = append(result, reconcile.Request{NamespacedName: name})
 		}
 		return result
 	}
@@ -283,6 +285,7 @@ func (r *SwiftRingReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.ConfigMap{}).
 		Owns(&rbacv1.ClusterRole{}).
 		Owns(&rbacv1.ClusterRoleBinding{}).
-		Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(deviceConfigMapFilter)).
+		Watches(&swiftv1beta1.SwiftStorage{}, handler.EnqueueRequestsFromMapFunc(swiftRingFilter)).
+		Watches(&dataplanev1.OpenStackDataPlaneNodeSet{}, handler.EnqueueRequestsFromMapFunc(swiftRingFilter)).
 		Complete(r)
 }

controllers/swiftstorage_controller.go

@@ -41,7 +41,6 @@ import (
 	"k8s.io/client-go/kubernetes"
 
 	swiftv1beta1 "github.com/openstack-k8s-operators/swift-operator/api/v1beta1"
-	"github.com/openstack-k8s-operators/swift-operator/pkg/swift"
 	"github.com/openstack-k8s-operators/swift-operator/pkg/swiftstorage"
 
 	"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
@@ -141,22 +140,6 @@ func (r *SwiftStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	serviceLabels := swiftstorage.Labels()
 	envVars := make(map[string]env.Setter)
 
-	// Check if there is already an existing ConfigMap and device list. If
-	// not, create an initial device list to bootstrap the cluster with The
-	// weights are simply set to the requested size, this will be changed
-	// once all StatefulSets are running
-	_, ctrlResult, err := configmap.GetConfigMap(ctx, helper, instance, swift.DeviceConfigMapName, 5*time.Second)
-	if err != nil {
-		return ctrlResult, err
-	} else if (ctrlResult != ctrl.Result{}) {
-		devices := swiftstorage.DeviceList(ctx, helper, instance)
-		tpl := swiftstorage.DeviceConfigMapTemplates(instance, devices)
-		err = configmap.EnsureConfigMaps(ctx, helper, instance, tpl, &envVars)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
-	}
-
 	// Create a ConfigMap populated with content from templates/
 	tpl := swiftstorage.ConfigMapTemplates(instance, serviceLabels, instance.Spec.MemcachedServers)
 	err = configmap.EnsureConfigMaps(ctx, helper, instance, tpl, &envVars)
@@ -169,7 +152,7 @@ func (r *SwiftStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	ctrlResult, err = svc.CreateOrPatch(ctx, helper)
+	ctrlResult, err := svc.CreateOrPatch(ctx, helper)
 	if err != nil {
 		return ctrlResult, err
 	} else if (ctrlResult != ctrl.Result{}) {
@@ -236,14 +219,6 @@ func (r *SwiftStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request
 
 	instance.Status.ReadyCount = sset.GetStatefulSet().Status.ReadyReplicas
 	if instance.Status.ReadyCount == *instance.Spec.Replicas {
-		envVars := make(map[string]env.Setter)
-		devices := swiftstorage.DeviceList(ctx, helper, instance)
-		tpl = swiftstorage.DeviceConfigMapTemplates(instance, devices)
-		err = configmap.EnsureConfigMaps(ctx, helper, instance, tpl, &envVars)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
-
 		// When the cluster is attached to an external network, create DNS record for every
 		// cluster member so it can be resolved from outside cluster (edpm nodes)
 		podList, err := pod.GetPodListWithLabel(ctx, helper, instance.Namespace, serviceLabels)

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
 	github.com/onsi/ginkgo/v2 v2.15.0
 	github.com/onsi/gomega v1.31.1
+	github.com/openstack-k8s-operators/dataplane-operator/api v0.3.1-0.20240219024506-c76c45e69237
 	github.com/openstack-k8s-operators/infra-operator/apis v0.3.1-0.20240218132212-ad757a2f5bab
 	github.com/openstack-k8s-operators/keystone-operator/api v0.3.1-0.20240216173228-eec429bcc776
 	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240216173409-86913e6d5885
@@ -36,18 +37,22 @@ require (
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gophercloud/gophercloud v1.9.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/metal3-io/baremetal-operator/apis v0.5.0 // indirect
+	github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.4.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/openshift/api v3.9.0+incompatible // indirect
 	github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.20240216173409-86913e6d5885 // indirect
+	github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240216173409-86913e6d5885 // indirect
+	github.com/openstack-k8s-operators/openstack-baremetal-operator/api v0.3.1-0.20240218185734-5d372a0fd380 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect