[WIP] storageos: Add compute-only deployment support

pkg/apis/storageos/v1/storageoscluster_types.go

@@ -115,6 +115,11 @@ type StorageOSClusterSpec struct {
 	// node affinity requiredDuringSchedulingIgnoredDuringExecution.
 	NodeSelectorTerms []corev1.NodeSelectorTerm `json:"nodeSelectorTerms"`
 
+	// ComputeOnlyNodeSelectorTerms is to set the placement of storageos compute
+	// only pods using node affinity
+	// requiredDuringSchedulingIgnoredDuringExecution.
+	ComputeOnlyNodeSelectorTerms []corev1.NodeSelectorTerm `json:"computeOnlyNodeSelectorTerms"`
+
 	// Tolerations is to set the placement of storageos pods using
 	// pod toleration.
 	Tolerations []corev1.Toleration `json:"tolerations"`

pkg/storageos/daemonset.go

@@ -9,7 +9,49 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+const (
+	// Names of the storageos daemonset resources.
+	daemonsetName            = "storageos-daemonset"
+	computeOnlyDaemonsetName = "storageos-compute-only"
+)
+
+// createDaemonSet creates storageos storage daemonset.
 func (s *Deployment) createDaemonSet() error {
+	dset, err := s.getBasicDaemonSetConfiguration(daemonsetName)
+	if err != nil {
+		return err
+	}
+
+	s.addNodeAffinity(&dset.Spec.Template.Spec, s.stos.Spec.NodeSelectorTerms)
+
+	return s.createOrUpdateObject(dset)
+}
+
+// createComputeOnlyDaemonSet creates storageos compute only daemonset.
+func (s *Deployment) createComputeOnlyDaemonSet() error {
+	// Check if node selector terms for compute only is specified.
+	if len(s.stos.Spec.ComputeOnlyNodeSelectorTerms) < 1 {
+		return nil
+	}
+
+	dset, err := s.getBasicDaemonSetConfiguration(computeOnlyDaemonsetName)
+	if err != nil {
+		return err
+	}
+
+	podSpec := &dset.Spec.Template.Spec
+	nodeContainer := &podSpec.Containers[0]
+
+	// Pass compute-only label.
+	nodeContainer.Env = s.addStorageOSLabelsEnvVars(nodeContainer.Env, computeOnlyLabelVal)
+
+	s.addNodeAffinity(podSpec, s.stos.Spec.ComputeOnlyNodeSelectorTerms)
+
+	return s.createOrUpdateObject(dset)
+}
+
+// getBasicDaemonSet creates a basic daemonset configuration for storageos.
+func (s *Deployment) getBasicDaemonSetConfiguration(name string) (*appsv1.DaemonSet, error) {
 	ls := labelsForDaemonSet(s.stos.Name)
 	privileged := true
 	mountPropagationBidirectional := corev1.MountPropagationBidirectional
@@ -21,7 +63,7 @@ func (s *Deployment) createDaemonSet() error {
 			Kind:       "DaemonSet",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      daemonsetName,
+			Name:      name,
 			Namespace: s.stos.Spec.GetResourceNS(),
 			Labels: map[string]string{
 				"app": "storageos",
@@ -217,10 +259,8 @@ func (s *Deployment) createDaemonSet() error {
 	podSpec := &dset.Spec.Template.Spec
 	nodeContainer := &podSpec.Containers[0]
 
-	s.addNodeAffinity(podSpec)
-
 	if err := s.addTolerations(podSpec); err != nil {
-		return err
+		return nil, err
 	}
 
 	nodeContainer.Env = s.addKVBackendEnvVars(nodeContainer.Env)
@@ -233,7 +273,7 @@ func (s *Deployment) createDaemonSet() error {
 
 	s.addCSI(podSpec)
 
-	return s.createOrUpdateObject(dset)
+	return dset, nil
 }
 
 func (s *Deployment) deleteDaemonSet(name string) error {

pkg/storageos/deploy.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strings"
 
 	"github.com/blang/semver"
 	storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1"
@@ -24,7 +25,6 @@ const (
 	daemonsetKind   = "daemonset"
 	statefulsetKind = "statefulset"
 
-	daemonsetName   = "storageos-daemonset"
 	statefulsetName = "storageos-statefulset"
 
 	tlsSecretType       = "kubernetes.io/tls"
@@ -36,6 +36,7 @@ const (
 	hostnameEnvVar                      = "HOSTNAME"
 	adminUsernameEnvVar                 = "ADMIN_USERNAME"
 	adminPasswordEnvVar                 = "ADMIN_PASSWORD"
+	labelsEnvVar                        = "LABELS"
 	joinEnvVar                          = "JOIN"
 	advertiseIPEnvVar                   = "ADVERTISE_IP"
 	namespaceEnvVar                     = "NAMESPACE"
@@ -61,8 +62,9 @@ const (
 	kvBackendEnvVar                     = "KV_BACKEND"
 	debugEnvVar                         = "LOG_LEVEL"
 
-	sysAdminCap = "SYS_ADMIN"
-	debugVal    = "xdebug"
+	sysAdminCap         = "SYS_ADMIN"
+	debugVal            = "xdebug"
+	computeOnlyLabelVal = "storageos.com/deployment=computeonly"
 
 	defaultFSType                            = "ext4"
 	secretNamespaceKey                       = "adminSecretNamespace"
@@ -120,6 +122,12 @@ func (s *Deployment) Deploy() error {
 		return err
 	}
 
+	// Compute-only daemonset.
+	if err := s.createComputeOnlyDaemonSet(); err != nil {
+		return err
+	}
+
+	// Storage daemonset.
 	if err := s.createDaemonSet(); err != nil {
 		return err
 	}
@@ -318,6 +326,37 @@ func getCSICredsEnvVar(envVarName, secretName, key string) corev1.EnvVar {
 	}
 }
 
+// addStorageOSLabelsEnvVars checks if the debug mode is set and set the appropriate env var.
+func (s Deployment) addStorageOSLabelsEnvVars(env []corev1.EnvVar, labels string) []corev1.EnvVar {
+	// Return the argument env var if no labels are specified.
+	if len(labels) == 0 {
+		return env
+	}
+
+	// Check if labels env var already exists.
+	labelsExists := false
+	for _, envVar := range env {
+		if envVar.Name == labelsEnvVar {
+			// Append the label with new label entries.
+			// The labels are separated by ",".
+			// e.g.: "storageos.com/deployment=computeonly,country=us,env=prod"
+			envVar.Value = strings.Join([]string{envVar.Value, labels}, ",")
+			labelsExists = true
+			break
+		}
+	}
+
+	// Add new labels env var if it doesn't exists.
+	if !labelsExists {
+		labelsEnvVar := corev1.EnvVar{
+			Name:  labelsEnvVar,
+			Value: labels,
+		}
+		return append(env, labelsEnvVar)
+	}
+	return env
+}
+
 // createOrUpdateObject attempts to create a given object. If the object already
 // exists and `Deployment.update` is false, no change is made. If update is true,
 // the existing object is updated.

pkg/storageos/deploy_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"testing"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -1137,3 +1138,123 @@ func TestDelete(t *testing.T) {
 		t.Fatal("failed to get the created namespace", err)
 	}
 }
+
+func TestDeployComputeOnlyNodes(t *testing.T) {
+	nodeSelectorTerms := []corev1.NodeSelectorTerm{
+		{
+			MatchExpressions: []corev1.NodeSelectorRequirement{
+				{
+					Key:      "foo",
+					Operator: corev1.NodeSelectorOpIn,
+					Values:   []string{"baz"},
+				},
+			},
+		},
+	}
+
+	testcases := []struct {
+		name                         string
+		nodeSelectorTerms            []corev1.NodeSelectorTerm
+		computeOnlyNodeSelectorTerms []corev1.NodeSelectorTerm
+		wantComputeOnly              bool
+	}{
+		{
+			name:              "2 daemonsets",
+			nodeSelectorTerms: nodeSelectorTerms,
+			computeOnlyNodeSelectorTerms: []corev1.NodeSelectorTerm{
+				{
+					MatchExpressions: []corev1.NodeSelectorRequirement{
+						{
+							Key:      "storageos.com/deployment",
+							Operator: corev1.NodeSelectorOpIn,
+							Values:   []string{"computeonly"},
+						},
+					},
+				},
+			},
+			wantComputeOnly: true,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			stosCluster := &api.StorageOSCluster{
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: gvk.GroupVersion().String(),
+					Kind:       gvk.Kind,
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "teststos",
+					Namespace: "default",
+				},
+				Spec: api.StorageOSClusterSpec{
+					CSI: api.StorageOSClusterCSI{
+						Enable: true,
+					},
+					NodeSelectorTerms:            tc.nodeSelectorTerms,
+					ComputeOnlyNodeSelectorTerms: tc.computeOnlyNodeSelectorTerms,
+				},
+			}
+
+			c := fake.NewFakeClientWithScheme(testScheme)
+			if err := c.Create(context.Background(), stosCluster); err != nil {
+				t.Fatalf("failed to create storageoscluster object: %v", err)
+			}
+
+			deploy := NewDeployment(c, stosCluster, nil, testScheme, "", false)
+			err := deploy.Deploy()
+			if err != nil {
+				t.Error("deployment failed:", err)
+			}
+
+			daemonset1 := &appsv1.DaemonSet{}
+			nsName1 := types.NamespacedName{
+				Name:      daemonsetName,
+				Namespace: defaultNS,
+			}
+
+			if err := c.Get(context.Background(), nsName1, daemonset1); err != nil {
+				t.Fatal("failed to get the created daemonset", err)
+			}
+
+			// Check if storage daemonset contains compute-only label.
+			for _, envvar := range daemonset1.Spec.Template.Spec.Containers[0].Env {
+				if envvar.Name != labelsEnvVar {
+					continue
+				}
+				if strings.Contains(envvar.Value, computeOnlyLabelVal) {
+					t.Errorf("expected %q to not be in node labels", computeOnlyLabelVal)
+				}
+			}
+
+			daemonset2 := &appsv1.DaemonSet{}
+			nsName2 := types.NamespacedName{
+				Name:      computeOnlyDaemonsetName,
+				Namespace: defaultNS,
+			}
+
+			if err := c.Get(context.Background(), nsName2, daemonset2); err != nil {
+				t.Fatal("failed to get the created daemonset", err)
+			}
+
+			// Check if storage daemonset contains compute-only label.
+			foundComputeOnly := false
+			for _, envvar := range daemonset2.Spec.Template.Spec.Containers[0].Env {
+				if envvar.Name != labelsEnvVar {
+					continue
+				}
+				if !strings.Contains(envvar.Value, computeOnlyLabelVal) {
+					t.Errorf("expected %q to be in node labels", computeOnlyLabelVal)
+				} else {
+					foundComputeOnly = true
+				}
+			}
+
+			// Extra check to ensure compute compute only was found. The above
+			// check would be skipped if there's no LABELS env var.
+			if !foundComputeOnly {
+				t.Error("expected to find computeonly label")
+			}
+		})
+	}
+}

pkg/storageos/podspec.go

@@ -252,11 +252,11 @@ func (s *Deployment) addCSI(podSpec *corev1.PodSpec) {
 
 // addNodeAffinity adds node affinity to the given pod spec from the cluster
 // spec NodeSelectorLabel.
-func (s *Deployment) addNodeAffinity(podSpec *corev1.PodSpec) {
-	if len(s.stos.Spec.NodeSelectorTerms) > 0 {
+func (s Deployment) addNodeAffinity(podSpec *corev1.PodSpec, nsTerms []corev1.NodeSelectorTerm) {
+	if len(nsTerms) > 0 {
 		podSpec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{
 			RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
-				NodeSelectorTerms: s.stos.Spec.NodeSelectorTerms,
+				NodeSelectorTerms: nsTerms,
 			},
 		}}
 	}

pkg/storageos/statefulset.go

@@ -134,7 +134,7 @@ func (s *Deployment) createStatefulSet() error {
 
 	podSpec := &sset.Spec.Template.Spec
 
-	s.addNodeAffinity(podSpec)
+	s.addNodeAffinity(podSpec, s.stos.Spec.NodeSelectorTerms)
 
 	if err := s.addTolerations(podSpec); err != nil {
 		return err