From 86f294a32fd1ee7b64051bfbb6d79b8895add9a6 Mon Sep 17 00:00:00 2001 From: Sunny Date: Mon, 8 Apr 2019 18:34:45 +0530 Subject: [PATCH] storageos: Add compute-only deployment support --- .../storageos/v1/storageoscluster_types.go | 5 + .../storageos/v1/zz_generated.deepcopy.go | 17 ++- pkg/storageos/daemonset.go | 50 +++++++- pkg/storageos/deploy.go | 45 ++++++- pkg/storageos/deploy_test.go | 121 ++++++++++++++++++ pkg/storageos/podspec.go | 6 +- pkg/storageos/statefulset.go | 2 +- 7 files changed, 229 insertions(+), 17 deletions(-) diff --git a/pkg/apis/storageos/v1/storageoscluster_types.go b/pkg/apis/storageos/v1/storageoscluster_types.go index e249d0a93..3e7d3fb39 100644 --- a/pkg/apis/storageos/v1/storageoscluster_types.go +++ b/pkg/apis/storageos/v1/storageoscluster_types.go @@ -115,6 +115,11 @@ type StorageOSClusterSpec struct { // node affinity requiredDuringSchedulingIgnoredDuringExecution. NodeSelectorTerms []corev1.NodeSelectorTerm `json:"nodeSelectorTerms"` + // ComputeOnlyNodeSelectorTerms is to set the placement of storageos compute + // only pods using node affinity + // requiredDuringSchedulingIgnoredDuringExecution. + ComputeOnlyNodeSelectorTerms []corev1.NodeSelectorTerm `json:"computeOnlyNodeSelectorTerms"` + // Tolerations is to set the placement of storageos pods using // pod toleration. Tolerations []corev1.Toleration `json:"tolerations"` diff --git a/pkg/apis/storageos/v1/zz_generated.deepcopy.go b/pkg/apis/storageos/v1/zz_generated.deepcopy.go index 29a90612a..7cc1d93f0 100644 --- a/pkg/apis/storageos/v1/zz_generated.deepcopy.go +++ b/pkg/apis/storageos/v1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1 import ( - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -112,14 +112,14 @@ func (in *JobSpec) DeepCopyInto(out *JobSpec) { } if in.NodeSelectorTerms != nil { in, out := &in.NodeSelectorTerms, &out.NodeSelectorTerms - *out = make([]v1.NodeSelectorTerm, len(*in)) + *out = make([]corev1.NodeSelectorTerm, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) + *out = make([]corev1.Toleration, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -344,14 +344,21 @@ func (in *StorageOSClusterSpec) DeepCopyInto(out *StorageOSClusterSpec) { out.KVBackend = in.KVBackend if in.NodeSelectorTerms != nil { in, out := &in.NodeSelectorTerms, &out.NodeSelectorTerms - *out = make([]v1.NodeSelectorTerm, len(*in)) + *out = make([]corev1.NodeSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ComputeOnlyNodeSelectorTerms != nil { + in, out := &in.ComputeOnlyNodeSelectorTerms, &out.ComputeOnlyNodeSelectorTerms + *out = make([]corev1.NodeSelectorTerm, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) + *out = make([]corev1.Toleration, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/pkg/storageos/daemonset.go b/pkg/storageos/daemonset.go index 3be5df75d..b1969a38d 100644 --- a/pkg/storageos/daemonset.go +++ b/pkg/storageos/daemonset.go @@ -9,7 +9,49 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +const ( + // Names of the storageos daemonset resources. + daemonsetName = "storageos-daemonset" + computeOnlyDaemonsetName = "storageos-compute-only" +) + +// createDaemonSet creates storageos storage daemonset. func (s *Deployment) createDaemonSet() error { + dset, err := s.getBasicDaemonSetConfiguration(daemonsetName) + if err != nil { + return err + } + + s.addNodeAffinity(&dset.Spec.Template.Spec, s.stos.Spec.NodeSelectorTerms) + + return s.createOrUpdateObject(dset) +} + +// createComputeOnlyDaemonSet creates storageos compute only daemonset. +func (s *Deployment) createComputeOnlyDaemonSet() error { + // Check if node selector terms for compute only is specified. + if len(s.stos.Spec.ComputeOnlyNodeSelectorTerms) < 1 { + return nil + } + + dset, err := s.getBasicDaemonSetConfiguration(computeOnlyDaemonsetName) + if err != nil { + return err + } + + podSpec := &dset.Spec.Template.Spec + nodeContainer := &podSpec.Containers[0] + + // Pass compute-only label. + nodeContainer.Env = s.addStorageOSLabelsEnvVars(nodeContainer.Env, computeOnlyLabelVal) + + s.addNodeAffinity(podSpec, s.stos.Spec.ComputeOnlyNodeSelectorTerms) + + return s.createOrUpdateObject(dset) +} + +// getBasicDaemonSet creates a basic daemonset configuration for storageos. +func (s *Deployment) getBasicDaemonSetConfiguration(name string) (*appsv1.DaemonSet, error) { ls := labelsForDaemonSet(s.stos.Name) privileged := true mountPropagationBidirectional := corev1.MountPropagationBidirectional @@ -21,7 +63,7 @@ func (s *Deployment) createDaemonSet() error { Kind: "DaemonSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: daemonsetName, + Name: name, Namespace: s.stos.Spec.GetResourceNS(), Labels: map[string]string{ "app": "storageos", @@ -217,10 +259,8 @@ func (s *Deployment) createDaemonSet() error { podSpec := &dset.Spec.Template.Spec nodeContainer := &podSpec.Containers[0] - s.addNodeAffinity(podSpec) - if err := s.addTolerations(podSpec); err != nil { - return err + return nil, err } nodeContainer.Env = s.addKVBackendEnvVars(nodeContainer.Env) @@ -233,7 +273,7 @@ func (s *Deployment) createDaemonSet() error { s.addCSI(podSpec) - return s.createOrUpdateObject(dset) + return dset, nil } func (s *Deployment) deleteDaemonSet(name string) error { diff --git a/pkg/storageos/deploy.go b/pkg/storageos/deploy.go index b62b75c38..e2721da36 100644 --- a/pkg/storageos/deploy.go +++ b/pkg/storageos/deploy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "strings" "github.com/blang/semver" storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" @@ -24,7 +25,6 @@ const ( daemonsetKind = "daemonset" statefulsetKind = "statefulset" - daemonsetName = "storageos-daemonset" statefulsetName = "storageos-statefulset" tlsSecretType = "kubernetes.io/tls" @@ -36,6 +36,7 @@ const ( hostnameEnvVar = "HOSTNAME" adminUsernameEnvVar = "ADMIN_USERNAME" adminPasswordEnvVar = "ADMIN_PASSWORD" + labelsEnvVar = "LABELS" joinEnvVar = "JOIN" advertiseIPEnvVar = "ADVERTISE_IP" namespaceEnvVar = "NAMESPACE" @@ -61,8 +62,9 @@ const ( kvBackendEnvVar = "KV_BACKEND" debugEnvVar = "LOG_LEVEL" - sysAdminCap = "SYS_ADMIN" - debugVal = "xdebug" + sysAdminCap = "SYS_ADMIN" + debugVal = "xdebug" + computeOnlyLabelVal = "storageos.com/deployment=computeonly" defaultFSType = "ext4" secretNamespaceKey = "adminSecretNamespace" @@ -120,6 +122,12 @@ func (s *Deployment) Deploy() error { return err } + // Compute-only daemonset. + if err := s.createComputeOnlyDaemonSet(); err != nil { + return err + } + + // Storage daemonset. if err := s.createDaemonSet(); err != nil { return err } @@ -318,6 +326,37 @@ func getCSICredsEnvVar(envVarName, secretName, key string) corev1.EnvVar { } } +// addStorageOSLabelsEnvVars checks if the debug mode is set and set the appropriate env var. +func (s Deployment) addStorageOSLabelsEnvVars(env []corev1.EnvVar, labels string) []corev1.EnvVar { + // Return the argument env var if no labels are specified. + if len(labels) == 0 { + return env + } + + // Check if labels env var already exists. + labelsExists := false + for _, envVar := range env { + if envVar.Name == labelsEnvVar { + // Append the label with new label entries. + // The labels are separated by ",". + // e.g.: "storageos.com/deployment=computeonly,country=us,env=prod" + envVar.Value = strings.Join([]string{envVar.Value, labels}, ",") + labelsExists = true + break + } + } + + // Add new labels env var if it doesn't exists. + if !labelsExists { + labelsEnvVar := corev1.EnvVar{ + Name: labelsEnvVar, + Value: labels, + } + return append(env, labelsEnvVar) + } + return env +} + // createOrUpdateObject attempts to create a given object. If the object already // exists and `Deployment.update` is false, no change is made. If update is true, // the existing object is updated. diff --git a/pkg/storageos/deploy_test.go b/pkg/storageos/deploy_test.go index 1dcb82262..844ed6dd0 100644 --- a/pkg/storageos/deploy_test.go +++ b/pkg/storageos/deploy_test.go @@ -5,6 +5,7 @@ import ( "os" "reflect" "strconv" + "strings" "testing" appsv1 "k8s.io/api/apps/v1" @@ -1137,3 +1138,123 @@ func TestDelete(t *testing.T) { t.Fatal("failed to get the created namespace", err) } } + +func TestDeployComputeOnlyNodes(t *testing.T) { + nodeSelectorTerms := []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"baz"}, + }, + }, + }, + } + + testcases := []struct { + name string + nodeSelectorTerms []corev1.NodeSelectorTerm + computeOnlyNodeSelectorTerms []corev1.NodeSelectorTerm + wantComputeOnly bool + }{ + { + name: "2 daemonsets", + nodeSelectorTerms: nodeSelectorTerms, + computeOnlyNodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "storageos.com/deployment", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"computeonly"}, + }, + }, + }, + }, + wantComputeOnly: true, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + stosCluster := &api.StorageOSCluster{ + TypeMeta: metav1.TypeMeta{ + APIVersion: gvk.GroupVersion().String(), + Kind: gvk.Kind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "teststos", + Namespace: "default", + }, + Spec: api.StorageOSClusterSpec{ + CSI: api.StorageOSClusterCSI{ + Enable: true, + }, + NodeSelectorTerms: tc.nodeSelectorTerms, + ComputeOnlyNodeSelectorTerms: tc.computeOnlyNodeSelectorTerms, + }, + } + + c := fake.NewFakeClientWithScheme(testScheme) + if err := c.Create(context.Background(), stosCluster); err != nil { + t.Fatalf("failed to create storageoscluster object: %v", err) + } + + deploy := NewDeployment(c, stosCluster, nil, testScheme, "", false) + err := deploy.Deploy() + if err != nil { + t.Error("deployment failed:", err) + } + + daemonset1 := &appsv1.DaemonSet{} + nsName1 := types.NamespacedName{ + Name: daemonsetName, + Namespace: defaultNS, + } + + if err := c.Get(context.Background(), nsName1, daemonset1); err != nil { + t.Fatal("failed to get the created daemonset", err) + } + + // Check if storage daemonset contains compute-only label. + for _, envvar := range daemonset1.Spec.Template.Spec.Containers[0].Env { + if envvar.Name != labelsEnvVar { + continue + } + if strings.Contains(envvar.Value, computeOnlyLabelVal) { + t.Errorf("expected %q to not be in node labels", computeOnlyLabelVal) + } + } + + daemonset2 := &appsv1.DaemonSet{} + nsName2 := types.NamespacedName{ + Name: computeOnlyDaemonsetName, + Namespace: defaultNS, + } + + if err := c.Get(context.Background(), nsName2, daemonset2); err != nil { + t.Fatal("failed to get the created daemonset", err) + } + + // Check if storage daemonset contains compute-only label. + foundComputeOnly := false + for _, envvar := range daemonset2.Spec.Template.Spec.Containers[0].Env { + if envvar.Name != labelsEnvVar { + continue + } + if !strings.Contains(envvar.Value, computeOnlyLabelVal) { + t.Errorf("expected %q to be in node labels", computeOnlyLabelVal) + } else { + foundComputeOnly = true + } + } + + // Extra check to ensure compute compute only was found. The above + // check would be skipped if there's no LABELS env var. + if !foundComputeOnly { + t.Error("expected to find computeonly label") + } + }) + } +} diff --git a/pkg/storageos/podspec.go b/pkg/storageos/podspec.go index 11840d9e7..28a99134f 100644 --- a/pkg/storageos/podspec.go +++ b/pkg/storageos/podspec.go @@ -252,11 +252,11 @@ func (s *Deployment) addCSI(podSpec *corev1.PodSpec) { // addNodeAffinity adds node affinity to the given pod spec from the cluster // spec NodeSelectorLabel. -func (s *Deployment) addNodeAffinity(podSpec *corev1.PodSpec) { - if len(s.stos.Spec.NodeSelectorTerms) > 0 { +func (s Deployment) addNodeAffinity(podSpec *corev1.PodSpec, nsTerms []corev1.NodeSelectorTerm) { + if len(nsTerms) > 0 { podSpec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: s.stos.Spec.NodeSelectorTerms, + NodeSelectorTerms: nsTerms, }, }} } diff --git a/pkg/storageos/statefulset.go b/pkg/storageos/statefulset.go index 33490382b..d59bb76ac 100644 --- a/pkg/storageos/statefulset.go +++ b/pkg/storageos/statefulset.go @@ -134,7 +134,7 @@ func (s *Deployment) createStatefulSet() error { podSpec := &sset.Spec.Template.Spec - s.addNodeAffinity(podSpec) + s.addNodeAffinity(podSpec, s.stos.Spec.NodeSelectorTerms) if err := s.addTolerations(podSpec); err != nil { return err