Use roles from the embedded-cluster config

.github/actions/copy-assets/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20 as builder
+FROM golang:1.21 as builder
 WORKDIR /action
 COPY . /action
 

.github/workflows/license.yaml

@@ -5,7 +5,7 @@ on:
   pull_request:
 
 env:
-  TRIVY_VERSION: 0.44.1
+  TRIVY_VERSION: 0.47.0
 
 name: License scan
 
@@ -16,7 +16,9 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v3
+        with:
+          go-version: '^1.20.0'
 
       - name: Install Go deps
         run: go mod download

deploy/okteto/okteto-v2.Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.3
-FROM golang:1.20.5
+FROM golang:1.21.3
 
 EXPOSE 2345
 

deploy/okteto/okteto.Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.3
-FROM golang:1.20-bookworm as builder
+FROM golang:1.21-bookworm as builder
 
 EXPOSE 2345
 

go.mod

@@ -1,6 +1,8 @@
 module github.com/replicatedhq/kots
 
-go 1.20
+go 1.21.0
+
+toolchain go1.21.3
 
 require (
 	cloud.google.com/go/storage v1.34.1
@@ -39,14 +41,15 @@ require (
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a
 	github.com/mitchellh/hashstructure v1.1.0
-	github.com/onsi/ginkgo/v2 v2.11.0
-	github.com/onsi/gomega v1.27.10
+	github.com/onsi/ginkgo/v2 v2.13.0
+	github.com/onsi/gomega v1.29.0
 	github.com/open-policy-agent/opa v0.51.0
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/otiai10/copy v1.9.0
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
+	github.com/replicatedhq/embedded-cluster-operator v0.3.0
 	github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82
 	github.com/replicatedhq/kurlkinds v1.3.6
 	github.com/replicatedhq/troubleshoot v0.76.4-0.20231102041618-a7bb9ea31e61
@@ -74,17 +77,17 @@ require (
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.13.2
-	k8s.io/api v0.28.2
-	k8s.io/apimachinery v0.28.2
+	k8s.io/api v0.28.3
+	k8s.io/apimachinery v0.28.3
 	k8s.io/cli-runtime v0.28.2
-	k8s.io/client-go v0.28.2
+	k8s.io/client-go v0.28.3
 	k8s.io/cluster-bootstrap v0.23.6
 	k8s.io/helm v2.14.3+incompatible
 	k8s.io/kubelet v0.23.6
 	k8s.io/metrics v0.28.2
 	k8s.io/utils v0.0.0-20230505201702-9f6742963106
 	sigs.k8s.io/application v0.8.3
-	sigs.k8s.io/controller-runtime v0.16.2
+	sigs.k8s.io/controller-runtime v0.16.3
 	sigs.k8s.io/kustomize/api v0.14.0
 	sigs.k8s.io/kustomize/kyaml v0.14.3
 	sigs.k8s.io/yaml v1.3.0
@@ -163,7 +166,7 @@ require (
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
@@ -384,9 +387,9 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
-	k8s.io/apiextensions-apiserver v0.28.2 // indirect
-	k8s.io/apiserver v0.28.2 // indirect
-	k8s.io/component-base v0.28.2 // indirect
+	k8s.io/apiextensions-apiserver v0.28.3 // indirect
+	k8s.io/apiserver v0.28.3 // indirect
+	k8s.io/component-base v0.28.3 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-aggregator v0.19.12 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect

hack/dev/skaffoldcache.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20 as deps
+FROM golang:1.21 as deps
 
 RUN go install github.com/go-delve/delve/cmd/[email protected]
 

kurl_proxy/go.mod

@@ -1,6 +1,6 @@
 module github.com/replicatedhq/kots/kurl_proxy
 
-go 1.20
+go 1.21.3
 
 require (
 	github.com/gin-gonic/gin v1.9.1

kurl_proxy/go.sum

@@ -75,6 +75,7 @@ github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=

kurl_proxy/okteto.Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.3
-FROM golang:1.20-buster
+FROM golang:1.21-buster
 
 ENV PROJECTPATH=/go/src/github.com/replicatedhq/kots/kurl_proxy
 WORKDIR $PROJECTPATH
@@ -13,4 +13,4 @@ RUN make build
 
 COPY assets assets
 
-ENTRYPOINT ["./bin/kurl_proxy"]
+ENTRYPOINT ["./bin/kurl_proxy"]

kurl_proxy/skaffold.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20-buster
+FROM golang:1.21-buster
 
 ENV PROJECTPATH=/go/src/github.com/replicatedhq/kots/kurl_proxy
 WORKDIR $PROJECTPATH

pkg/embeddedcluster/helmvm_node.go

@@ -143,10 +143,8 @@ func nodeRolesFromLabels(labels map[string]string) []string {
 
 	numRolesStr, ok := labels[types.EMBEDDED_CLUSTER_ROLE_LABEL]
 	if !ok {
-		// the first node will not initially have a role label, but is a 'controller'
-		if val, ok := labels["node-role.kubernetes.io/control-plane"]; ok && val == "true" {
-			return []string{"controller"}
-		}
+		// every node will have a label like this, so if it's missing, something is wrong
+		fmt.Printf("failed to find role label %q", types.EMBEDDED_CLUSTER_ROLE_LABEL)
 		return nil
 	}
 	numRoles, err := strconv.Atoi(strings.TrimPrefix(numRolesStr, "total-"))

pkg/embeddedcluster/node_join.go

@@ -241,9 +241,14 @@ func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, to
 // GenerateK0sJoinCommand returns the k0s node join command, without the token but with all other required flags
 // (including node labels generated from the roles etc)
 func GenerateK0sJoinCommand(ctx context.Context, client kubernetes.Interface, roles []string) (string, error) {
+	controllerRoleName, err := ControllerRoleName(ctx)
+	if err != nil {
+		return "", fmt.Errorf("failed to get controller role name: %w", err)
+	}
+
 	k0sRole := "worker"
 	for _, role := range roles {
-		if role == "controller" {
+		if role == controllerRoleName {
 			k0sRole = "controller"
 		}
 	}
@@ -253,7 +258,7 @@ func GenerateK0sJoinCommand(ctx context.Context, client kubernetes.Interface, ro
 		cmd = append(cmd, "--enable-worker")
 	}
 
-	labels, err := getRolesNodeLabels(ctx, client, roles)
+	labels, err := getRolesNodeLabels(ctx, roles)
 	if err != nil {
 		return "", fmt.Errorf("failed to get role labels: %w", err)
 	}
@@ -304,35 +309,26 @@ func getControllerNodeIP(ctx context.Context, client kubernetes.Interface) (stri
 	return "", fmt.Errorf("failed to find healthy controller node")
 }
 
-func getRolesNodeLabels(ctx context.Context, client kubernetes.Interface, roles []string) (string, error) {
-	roleLabels := getRoleListLabels(roles)
+func getRolesNodeLabels(ctx context.Context, roles []string) (string, error) {
+	roleListLabels := getRoleListLabels(roles)
 
-	for _, role := range roles {
-		labels, err := getRoleNodeLabels(ctx, client, role)
-		if err != nil {
-			return "", fmt.Errorf("failed to get node labels for role %s: %w", role, err)
-		}
-		roleLabels = append(roleLabels, labels...)
+	labels, err := getRoleNodeLabels(ctx, roles)
+	if err != nil {
+		return "", fmt.Errorf("failed to get node labels for roles %v: %w", roles, err)
 	}
+	roleLabels := append(roleListLabels, labels...)
 
 	return strings.Join(roleLabels, ","), nil
 }
 
-// TODO: look up role in cluster config, apply additional labels based on role
-func getRoleNodeLabels(ctx context.Context, client kubernetes.Interface, role string) ([]string, error) {
-	toReturn := []string{}
-
-	return toReturn, nil
-}
-
 // getRoleListLabels returns the labels needed to identify the roles of this node in the future
 // one label will be the number of roles, and then deterministic label names will be used to store the role names
 func getRoleListLabels(roles []string) []string {
 	toReturn := []string{}
 	toReturn = append(toReturn, fmt.Sprintf("%s=total-%d", types.EMBEDDED_CLUSTER_ROLE_LABEL, len(roles)))
 
 	for idx, role := range roles {
-		toReturn = append(toReturn, fmt.Sprintf("%s-%d=%s", types.EMBEDDED_CLUSTER_ROLE_LABEL, idx, role))
+		toReturn = append(toReturn, fmt.Sprintf("%s-%d=%s", types.EMBEDDED_CLUSTER_ROLE_LABEL, idx, labelify(role)))
 	}
 
 	return toReturn

pkg/embeddedcluster/roles.go

@@ -0,0 +1,140 @@
+package embeddedcluster
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
+)
+
+const DEFAULT_CONTROLLER_ROLE_NAME = "controller"
+
+var labelValueRegex = regexp.MustCompile(`[^a-zA-Z0-9-_.]+`)
+
+// GetRoles will get a list of role names
+func GetRoles(ctx context.Context) ([]string, error) {
+	config, err := ClusterConfig(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cluster config: %w", err)
+	}
+
+	if config == nil {
+		// use the default nil spec
+		config = &embeddedclusterv1beta1.ConfigSpec{}
+	}
+
+	// determine role names
+	roles := []string{}
+	if config.Controller.Name != "" {
+		roles = append(roles, config.Controller.Name)
+	} else {
+		roles = append(roles, DEFAULT_CONTROLLER_ROLE_NAME)
+	}
+
+	for _, role := range config.Custom {
+		if role.Name != "" {
+			roles = append(roles, role.Name)
+		}
+	}
+
+	return roles, nil
+}
+
+// ControllerRoleName determines the name for the 'controller' role
+// this might be part of the config, or it might be the default
+func ControllerRoleName(ctx context.Context) (string, error) {
+	conf, err := ClusterConfig(ctx)
+	if err != nil {
+		return "", fmt.Errorf("failed to get cluster config: %w", err)
+	}
+
+	if conf != nil && conf.Controller.Name != "" {
+		return conf.Controller.Name, nil
+	}
+	return DEFAULT_CONTROLLER_ROLE_NAME, nil
+}
+
+// sort roles by name, but put controller first
+func SortRoles(controllerRole string, inputRoles []string) []string {
+	roles := inputRoles
+
+	// determine if the controller role is present
+	hasController := false
+	controllerIdx := 0
+	for idx, role := range roles {
+		if role == controllerRole {
+			hasController = true
+			controllerIdx = idx
+			break
+		}
+	}
+
+	// if the controller role is present, remove it
+	if hasController {
+		roles = append(roles[:controllerIdx], roles[controllerIdx+1:]...)
+	}
+
+	// sort the roles
+	sort.Strings(roles)
+
+	// if the controller role was present, add it back to the front
+	if hasController {
+		roles = append([]string{controllerRole}, roles...)
+	}
+
+	return roles
+}
+
+// getRoleNodeLabels looks up roles in the cluster config and determines the additional labels to be applied from that
+func getRoleNodeLabels(ctx context.Context, roles []string) ([]string, error) {
+	config, err := ClusterConfig(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cluster config: %w", err)
+	}
+
+	return getRoleLabelsImpl(config, roles), nil
+}
+
+func labelify(s string) string {
+	// remove illegal characters
+	removechars := labelValueRegex.ReplaceAllString(s, "-")
+	// remove leading dashes
+	trimmed := strings.TrimPrefix(removechars, "-")
+	// restrict it to 63 characters
+	if len(trimmed) > 63 {
+		trimmed = trimmed[:63]
+	}
+	// remove trailing dashes
+	trimmed = strings.TrimSuffix(trimmed, "-")
+	return trimmed
+}
+
+func getRoleLabelsImpl(config *embeddedclusterv1beta1.ConfigSpec, roles []string) []string {
+	toReturn := []string{}
+
+	if config == nil {
+		return toReturn
+	}
+
+	for _, role := range roles {
+		if role == config.Controller.Name {
+			for k, v := range config.Controller.Labels {
+				toReturn = append(toReturn, fmt.Sprintf("%s=%s", labelify(k), labelify(v)))
+			}
+		}
+		for _, customRole := range config.Custom {
+			if role == customRole.Name {
+				for k, v := range customRole.Labels {
+					toReturn = append(toReturn, fmt.Sprintf("%s=%s", labelify(k), labelify(v)))
+				}
+			}
+		}
+	}
+
+	sort.Strings(toReturn)
+
+	return toReturn
+}