feat: ability to override admin console and lam host ports (#1201)

* feat: ability to override admin console and lam host ports

* kots overrides

* f

* Revert "kots overrides"

This reverts commit edbded1.

* update kots

Makefile

@@ -269,6 +269,7 @@ list-distros:
 
 .PHONY: create-node%
 create-node%: DISTRO = debian-bookworm
+create-node%: NODE_PORT = 30000
 create-node%:
 	@if ! docker images | grep -q ec-$(DISTRO); then \
 		$(MAKE) -C dev/distros build-$(DISTRO); \
@@ -282,7 +283,7 @@ create-node%:
 		-v /var/lib/k0s \
 		-v $(shell pwd):/replicatedhq/embedded-cluster \
 		-v $(shell dirname $(shell pwd))/kots:/replicatedhq/kots \
-		$(if $(filter node0,node$*),-p 30000:30000) \
+		$(if $(filter node0,node$*),-p $(NODE_PORT):$(NODE_PORT)) \
 		ec-$(DISTRO)
 
 	@$(MAKE) ssh-node$*

cmd/embedded-cluster/flags.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/urfave/cli/v2"
+	k8snet "k8s.io/utils/net"
+)
+
+func getAdminColsolePortFlag() cli.Flag {
+	return &cli.StringFlag{
+		Name:   "admin-console-port",
+		Usage:  "Port on which the Admin Console will be served",
+		Value:  strconv.Itoa(defaults.AdminConsolePort),
+		Hidden: false,
+	}
+}
+
+func getAdminConsolePortFromFlag(c *cli.Context) (int, error) {
+	portStr := c.String("admin-console-port")
+	if portStr == "" {
+		return defaults.AdminConsolePort, nil
+	}
+	// TODO: add first class support for service node port range and validate the port
+	port, err := k8snet.ParsePort(portStr, false)
+	if err != nil {
+		return 0, fmt.Errorf("invalid admin console port: %w", err)
+	}
+	return port, nil
+}
+
+func getLocalArtifactMirrorPortFlag() cli.Flag {
+	return &cli.StringFlag{
+		Name:   "local-artifact-mirror-port",
+		Usage:  "Port on which the Local Artifact Mirror will be served",
+		Value:  strconv.Itoa(defaults.LocalArtifactMirrorPort),
+		Hidden: false,
+	}
+}
+
+func getLocalArtifactMirrorPortFromFlag(c *cli.Context) (int, error) {
+	portStr := c.String("local-artifact-mirror-port")
+	if portStr == "" {
+		return defaults.LocalArtifactMirrorPort, nil
+	}
+	// TODO: add first class support for service node port range and validate the port does not
+	// conflict with this range
+	port, err := k8snet.ParsePort(portStr, false)
+	if err != nil {
+		return 0, fmt.Errorf("invalid local artifact mirror port: %w", err)
+	}
+	if portStr == c.String("admin-console-port") {
+		return 0, fmt.Errorf("local artifact mirror port cannot be the same as admin console port")
+	}
+	return port, nil
+}

cmd/embedded-cluster/install.go

@@ -37,18 +37,55 @@ var ErrNothingElseToAdd = fmt.Errorf("")
 // installAndEnableLocalArtifactMirror installs and enables the local artifact mirror. This
 // service is responsible for serving on localhost, through http, all files that are used
 // during a cluster upgrade.
-func installAndEnableLocalArtifactMirror() error {
+func installAndEnableLocalArtifactMirror(port int) error {
 	if err := goods.MaterializeLocalArtifactMirrorUnitFile(); err != nil {
 		return fmt.Errorf("failed to materialize artifact mirror unit: %w", err)
 	}
+	if err := writeLocalArtifactMirrorEnvironmentFile(port); err != nil {
+		return fmt.Errorf("failed to write local artifact mirror environment file: %w", err)
+	}
 	if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
 		return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
 	}
 	if _, err := helpers.RunCommand("systemctl", "start", "local-artifact-mirror"); err != nil {
 		return fmt.Errorf("unable to start the local artifact mirror: %w", err)
 	}
 	if _, err := helpers.RunCommand("systemctl", "enable", "local-artifact-mirror"); err != nil {
-		return fmt.Errorf("unable to start the local artifact mirror: %w", err)
+		return fmt.Errorf("unable to start the local artifact mirror service: %w", err)
+	}
+	return nil
+}
+
+// updateLocalArtifactMirrorService updates the port on which the local artifact mirror is served.
+func updateLocalArtifactMirrorService(port int) error {
+	if err := writeLocalArtifactMirrorEnvironmentFile(port); err != nil {
+		return fmt.Errorf("failed to write local artifact mirror environment file: %w", err)
+	}
+	if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
+		return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
+	}
+	if _, err := helpers.RunCommand("systemctl", "restart", "local-artifact-mirror"); err != nil {
+		return fmt.Errorf("unable to restart the local artifact mirror service: %w", err)
+	}
+	return nil
+}
+
+const (
+	localArtifactMirrorSystemdConfFile         = "/etc/systemd/system/local-artifact-mirror.service.d/embedded-cluster.conf"
+	localArtifactMirrorEnvironmentFileContents = `[Service]
+Environment="LOCAL_ARTIFACT_MIRROR_PORT=%d"`
+)
+
+func writeLocalArtifactMirrorEnvironmentFile(port int) error {
+	dir := filepath.Dir(localArtifactMirrorSystemdConfFile)
+	err := os.MkdirAll(dir, 0755)
+	if err != nil {
+		return fmt.Errorf("create directory: %w", err)
+	}
+	contents := fmt.Sprintf(localArtifactMirrorEnvironmentFileContents, port)
+	err = os.WriteFile(localArtifactMirrorSystemdConfFile, []byte(contents), 0644)
+	if err != nil {
+		return fmt.Errorf("write file: %w", err)
 	}
 	return nil
 }
@@ -463,7 +500,7 @@ func installAndWaitForK0s(c *cli.Context, applier *addons.Applier, proxy *ecv1be
 		return nil, err
 	}
 	logrus.Debugf("creating systemd unit files")
-	if err := createSystemdUnitFiles(false, proxy); err != nil {
+	if err := createSystemdUnitFiles(false, proxy, applier.GetLocalArtifactMirrorPort()); err != nil {
 		err := fmt.Errorf("unable to create systemd unit files: %w", err)
 		metrics.ReportApplyFinished(c, err)
 		return nil, err
@@ -591,6 +628,8 @@ var installCommand = &cli.Command{
 				Name:  "private-ca",
 				Usage: "Path to a trusted private CA certificate file",
 			},
+			getAdminColsolePortFlag(),
+			getLocalArtifactMirrorPortFlag(),
 		},
 	)),
 	Action: func(c *cli.Context) error {
@@ -709,6 +748,19 @@ func getAddonsApplier(c *cli.Context, adminConsolePwd string, proxy *ecv1beta1.P
 		}
 		opts = append(opts, addons.WithPrivateCAs(privateCAs))
 	}
+
+	adminConsolePort, err := getAdminConsolePortFromFlag(c)
+	if err != nil {
+		return nil, err
+	}
+	opts = append(opts, addons.WithAdminConsolePort(adminConsolePort))
+
+	localArtifactMirrorPort, err := getLocalArtifactMirrorPortFromFlag(c)
+	if err != nil {
+		return nil, err
+	}
+	opts = append(opts, addons.WithLocalArtifactMirrorPort(localArtifactMirrorPort))
+
 	if adminConsolePwd != "" {
 		opts = append(opts, addons.WithAdminConsolePassword(adminConsolePwd))
 	}

cmd/embedded-cluster/join.go

@@ -41,10 +41,11 @@ type JoinCommandResponse struct {
 	K0sUnsupportedOverrides   string    `json:"k0sUnsupportedOverrides"`
 	EndUserK0sConfigOverrides string    `json:"endUserK0sConfigOverrides"`
 	// MetricsBaseURL is the https://replicated.app endpoint url
-	MetricsBaseURL        string                 `json:"metricsBaseURL"`
-	AirgapRegistryAddress string                 `json:"airgapRegistryAddress"`
-	Proxy                 *ecv1beta1.ProxySpec   `json:"proxy"`
-	Network               *ecv1beta1.NetworkSpec `json:"network"`
+	MetricsBaseURL          string                 `json:"metricsBaseURL"`
+	AirgapRegistryAddress   string                 `json:"airgapRegistryAddress"`
+	Proxy                   *ecv1beta1.ProxySpec   `json:"proxy"`
+	Network                 *ecv1beta1.NetworkSpec `json:"network"`
+	LocalArtifactMirrorPort int                    `json:"localArtifactMirrorPort,omitempty"`
 }
 
 // extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that
@@ -265,8 +266,12 @@ var joinCommand = &cli.Command{
 		}
 
 		logrus.Debugf("creating systemd unit files")
+		localArtifactMirrorPort := defaults.LocalArtifactMirrorPort
+		if jcmd.LocalArtifactMirrorPort > 0 {
+			localArtifactMirrorPort = jcmd.LocalArtifactMirrorPort
+		}
 		// both controller and worker nodes will have 'worker' in the join command
-		if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.Proxy); err != nil {
+		if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.Proxy, localArtifactMirrorPort); err != nil {
 			err := fmt.Errorf("unable to create systemd unit files: %w", err)
 			metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
 			return err

cmd/embedded-cluster/restore.go

@@ -39,6 +39,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	k8snet "k8s.io/utils/net"
 	"k8s.io/utils/ptr"
 	k8sconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
 	k8syaml "sigs.k8s.io/yaml"
@@ -261,7 +262,7 @@ func newS3BackupStore() *s3BackupStore {
 	}
 	store.region = prompts.New().Input("Region:", "", true)
 	store.bucket = prompts.New().Input("Bucket:", "", true)
-	store.prefix = prompts.New().Input("Prefix (press Enter to skip):", "", false)
+	store.prefix = strings.TrimPrefix(prompts.New().Input("Prefix (press Enter to skip):", "", false), "/")
 	store.accessKeyID = prompts.New().Input("Access key ID:", "", true)
 	store.secretAccessKey = prompts.New().Password("Secret access key:")
 	logrus.Info("")
@@ -811,10 +812,19 @@ func waitForAdditionalNodes(ctx context.Context, highAvailability bool, networkI
 		return fmt.Errorf("unable to create kube client: %w", err)
 	}
 
+	in, err := kubeutils.GetLatestInstallation(ctx, kcli)
+	if err != nil {
+		return fmt.Errorf("unable to get latest installation: %w", err)
+	}
+	adminConsolePort := defaults.AdminConsolePort
+	if in.Spec.AdminConsole != nil && in.Spec.AdminConsole.Port > 0 {
+		adminConsolePort = in.Spec.AdminConsole.Port
+	}
+
 	successColor := "\033[32m"
 	colorReset := "\033[0m"
 	joinNodesMsg := fmt.Sprintf("\nVisit the Admin Console if you need to add nodes to the cluster: %s%s%s\n",
-		successColor, adminconsole.GetURL(networkInterface), colorReset,
+		successColor, adminconsole.GetURL(networkInterface, adminConsolePort), colorReset,
 	)
 	logrus.Info(joinNodesMsg)
 
@@ -860,7 +870,7 @@ func installAndWaitForRestoredK0sNode(c *cli.Context, applier *addons.Applier) (
 	}
 	proxy := getProxySpecFromFlags(c)
 	logrus.Debugf("creating systemd unit files")
-	if err := createSystemdUnitFiles(false, proxy); err != nil {
+	if err := createSystemdUnitFiles(false, proxy, applier.GetLocalArtifactMirrorPort()); err != nil {
 		return nil, fmt.Errorf("unable to create systemd unit files: %w", err)
 	}
 	logrus.Debugf("installing k0s")
@@ -907,6 +917,12 @@ var restoreCommand = &cli.Command{
 				Value:  false,
 				Hidden: true,
 			},
+			&cli.StringFlag{
+				Name:  "local-artifact-mirror-port",
+				Usage: "Port on which the Local Artifact Mirror will be served. If left empty, the port will be retrieved from the snapshot.",
+				// DefaultText: strconv.Itoa(defaults.LocalArtifactMirrorPort),
+				Hidden: false,
+			},
 		},
 	)),
 	Before: func(c *cli.Context) error {
@@ -1063,7 +1079,11 @@ var restoreCommand = &cli.Command{
 			}
 			logrus.Debugf("restoring embedded cluster installation from backup %q", backupToRestore.Name)
 			if err := restoreFromBackup(c.Context, backupToRestore, disasterRecoveryComponentECInstall); err != nil {
-				return err
+				return fmt.Errorf("unable to restore from backup: %w", err)
+			}
+			logrus.Debugf("updating local artifact mirror port %q", backupToRestore.Name)
+			if err := restoreReconcileLocalArtifactMirrorPort(c, backupToRestore); err != nil {
+				return fmt.Errorf("unable to update local artifact mirror port: %w", err)
 			}
 			fallthrough
 
@@ -1165,3 +1185,71 @@ var restoreCommand = &cli.Command{
 		return nil
 	},
 }
+
+// restoreReconcileLocalArtifactMirrorPort will set the local artifact mirror port in the
+// installation if it was explicitly set using a flag, otherwise it will update the service to use
+// the port from the installation.
+func restoreReconcileLocalArtifactMirrorPort(c *cli.Context, backup *velerov1.Backup) error {
+	if c.IsSet("local-artifact-mirror-port") {
+		logrus.Debugf("updating local artifact mirror port from flag %q", backup.Name)
+		err := restoreReconcileLocalArtifactMirrorPortFromFlag(c)
+		if err != nil {
+			return fmt.Errorf("unable to update local artifact mirror port from flag: %w", err)
+		}
+		return nil
+	}
+
+	logrus.Debugf("updating local artifact mirror port from backup %q", backup.Name)
+	err := restoreReconcileLocalArtifactMirrorPortFromBackup(backup)
+	if err != nil {
+		return fmt.Errorf("unable to update local artifact mirror port from backup: %w", err)
+	}
+	return nil
+}
+
+// restoreReconcileLocalArtifactMirrorPortFromFlag will set the local artifact mirror port in the
+// installation from the flag.
+func restoreReconcileLocalArtifactMirrorPortFromFlag(c *cli.Context) error {
+	kcli, err := kubeutils.KubeClient()
+	if err != nil {
+		return fmt.Errorf("create kube client: %w", err)
+	}
+	in, err := kubeutils.GetLatestInstallation(c.Context, kcli)
+	if err != nil {
+		return fmt.Errorf("get latest installation: %w", err)
+	}
+	if in.Spec.LocalArtifactMirror == nil {
+		in.Spec.LocalArtifactMirror = &ecv1beta1.LocalArtifactMirrorSpec{}
+	}
+	port, err := getLocalArtifactMirrorPortFromFlag(c)
+	if err != nil {
+		return fmt.Errorf("get local artifact mirror port: %w", err)
+	}
+	if in.Spec.LocalArtifactMirror.Port == port {
+		return nil
+	}
+	logrus.Debugf("updating local artifact mirror port from flag to %d on installation %q", port, in.Name)
+	in.Spec.LocalArtifactMirror.Port = port
+	if err := kcli.Update(c.Context, in); err != nil {
+		return fmt.Errorf("update installation: %w", err)
+	}
+	return nil
+}
+
+// restoreReconcileLocalArtifactMirrorPortFromBackup will update the service to use the port from
+// the installation.
+func restoreReconcileLocalArtifactMirrorPortFromBackup(backup *velerov1.Backup) error {
+	portStr := backup.Annotations["kots.io/embedded-cluster-local-artifact-mirror-port"]
+	if portStr == "" {
+		return nil
+	}
+	port, err := k8snet.ParsePort(portStr, false)
+	if err != nil {
+		return fmt.Errorf("unable to parse local artifact mirror port from backup: %w", err)
+	}
+	logrus.Debugf("updating local artifact mirror port from backup to %d", port)
+	if err := updateLocalArtifactMirrorService(port); err != nil {
+		return fmt.Errorf("unable to update local artifact mirror service: %w", err)
+	}
+	return nil
+}

cmd/embedded-cluster/util.go

@@ -12,7 +12,7 @@ import (
 
 // createSystemdUnitFiles links the k0s systemd unit file. this also creates a new
 // systemd unit file for the local artifact mirror service.
-func createSystemdUnitFiles(isWorker bool, proxy *ecv1beta1.ProxySpec) error {
+func createSystemdUnitFiles(isWorker bool, proxy *ecv1beta1.ProxySpec, localArtifactMirrorPort int) error {
 	dst := systemdUnitFileName()
 	if _, err := os.Lstat(dst); err == nil {
 		if err := os.Remove(dst); err != nil {
@@ -36,7 +36,10 @@ func createSystemdUnitFiles(isWorker bool, proxy *ecv1beta1.ProxySpec) error {
 	if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
 		return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
 	}
-	return installAndEnableLocalArtifactMirror()
+	if err := installAndEnableLocalArtifactMirror(localArtifactMirrorPort); err != nil {
+		return fmt.Errorf("unable to install and enable local artifact mirror: %w", err)
+	}
+	return nil
 }
 
 // ensureProxyConfig creates a new http-proxy.conf configuration file. The file is saved in the