feat: enable ip_forward, disable arp_ignore arp_filter (#1484)

* feat: enable ip_forward, disable arp_ignore arp_filter

we are now enabling ip_forward on the node, this is required for the
embedded-cluster to work properly. we are also disabling arp_ignore
and arp_filter to make sure the system is prepared for calico.

* chore: do not fail if unable to config sysctl

* feat: does not fail if unable to write sysctl config

we only fail if the sysctl binary is not present on the system as we
know that preflights depend on it. if we fail to configure sysctl we
just move on as the preflights are expected to fail later on.

* feat: config sysctl on 'run-prelights' command

we need to configure sysctl before running the preflights.

* chore: add unit tests for the new functions

added unit tests around the sysctl configuation functions.

pkg/cmd/install.go

@@ -781,6 +781,12 @@ func installCommand() *cli.Command {
 			}
 
 			metrics.ReportApplyStarted(c)
+
+			logrus.Debugf("configuring sysctl")
+			if err := configutils.ConfigureSysctl(provider); err != nil {
+				return fmt.Errorf("unable to configure sysctl: %w", err)
+			}
+
 			logrus.Debugf("configuring network manager")
 			if err := configureNetworkManager(c, provider); err != nil {
 				return fmt.Errorf("unable to configure network manager: %w", err)

pkg/cmd/join.go

@@ -249,6 +249,11 @@ var joinCommand = &cli.Command{
 			return err
 		}
 
+		logrus.Debugf("configuring sysctl")
+		if err := configutils.ConfigureSysctl(provider); err != nil {
+			return fmt.Errorf("unable to configure sysctl: %w", err)
+		}
+
 		// jcmd.InstallationSpec.MetricsBaseURL is the replicated.app endpoint url
 		replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL
 		proxyRegistryURL := fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress)

pkg/cmd/preflights.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
+	"github.com/replicatedhq/embedded-cluster/pkg/configutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
 	"github.com/replicatedhq/embedded-cluster/pkg/versions"
 	"github.com/sirupsen/logrus"
@@ -79,6 +80,10 @@ func installRunPreflightsCommand() *cli.Command {
 				return err
 			}
 
+			if err := configutils.ConfigureSysctl(provider); err != nil {
+				return err
+			}
+
 			applier, err := getAddonsApplier(c, runtimeConfig, "", proxy)
 			if err != nil {
 				return err
@@ -170,6 +175,10 @@ var joinRunPreflightsCommand = &cli.Command{
 			return err
 		}
 
+		if err := configutils.ConfigureSysctl(provider); err != nil {
+			return err
+		}
+
 		applier, err := getAddonsApplier(c, jcmd.InstallationSpec.RuntimeConfig, "", jcmd.InstallationSpec.Proxy)
 		if err != nil {
 			return err

pkg/cmd/reset.go

@@ -501,6 +501,10 @@ func resetCommand() *cli.Command {
 				return fmt.Errorf("failed to remove embedded cluster data config: %w", err)
 			}
 
+			if err := helpers.RemoveAll("/etc/sysctl.d/99-embedded-cluster.conf"); err != nil {
+				return fmt.Errorf("failed to remove embedded cluster sysctl config: %w", err)
+			}
+
 			if _, err := helpers.RunCommand("reboot"); err != nil {
 				return err
 			}

pkg/cmd/restore.go

@@ -969,6 +969,11 @@ func restoreCommand() *cli.Command {
 				}
 			}
 
+			logrus.Debugf("configuring sysctl")
+			if err := configutils.ConfigureSysctl(provider); err != nil {
+				return fmt.Errorf("unable to configure sysctl: %w", err)
+			}
+
 			proxy, err := getProxySpecFromFlags(c)
 			if err != nil {
 				return fmt.Errorf("unable to get proxy spec from flags: %w", err)

pkg/configutils/runtime.go

@@ -3,14 +3,22 @@ package configutils
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	"github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/replicatedhq/embedded-cluster/pkg/goods"
 	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
+	"github.com/sirupsen/logrus"
 	"sigs.k8s.io/yaml"
 )
 
+// sysctlConfigPath is the path to the sysctl config file that is used to configure
+// the embedded cluster. This could have been a constant but we want to be able to
+// override it for testing purposes.
+var sysctlConfigPath = "/etc/sysctl.d/99-embedded-cluster.conf"
+
 func WriteRuntimeConfig(spec *v1beta1.RuntimeConfigSpec) error {
 	if spec == nil {
 		return nil
@@ -57,3 +65,24 @@ func ReadRuntimeConfig() (*v1beta1.RuntimeConfigSpec, error) {
 
 	return &spec, nil
 }
+
+// ConfigureSysctl writes the sysctl config file for the embedded cluster and
+// reloads the sysctl configuration. This function has a distinct behavior: if
+// the sysctl binary does not exist it returns an error but if it fails to lay
+// down the sysctl config on disk it simply returns nil.
+func ConfigureSysctl(provider *defaults.Provider) error {
+	if _, err := exec.LookPath("sysctl"); err != nil {
+		return fmt.Errorf("unable to find sysctl binary: %w", err)
+	}
+
+	materializer := goods.NewMaterializer(provider)
+	if err := materializer.SysctlConfig(sysctlConfigPath); err != nil {
+		logrus.Debugf("unable to materialize sysctl config: %v", err)
+		return nil
+	}
+
+	if _, err := helpers.RunCommand("sysctl", "--system"); err != nil {
+		logrus.Debugf("unable to configure sysctl: %v", err)
+	}
+	return nil
+}

pkg/configutils/runtime_test.go

@@ -0,0 +1,41 @@
+package configutils
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigureSysctl(t *testing.T) {
+	basedir, err := os.MkdirTemp("", "embedded-cluster-test-base-dir")
+	assert.NoError(t, err)
+	defer os.RemoveAll(basedir)
+
+	orig := sysctlConfigPath
+	defer func() {
+		sysctlConfigPath = orig
+	}()
+
+	provider := defaults.NewProvider(basedir)
+
+	// happy path.
+	dstdir, err := os.MkdirTemp("", "embedded-cluster-test")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dstdir)
+
+	sysctlConfigPath = filepath.Join(dstdir, "sysctl.conf")
+	err = ConfigureSysctl(provider)
+	assert.NoError(t, err)
+
+	// check that the file exists.
+	_, err = os.Stat(sysctlConfigPath)
+	assert.NoError(t, err)
+
+	// now use a non-existing directory.
+	sysctlConfigPath = filepath.Join(dstdir, "non-existing-dir", "sysctl.conf")
+	// we do not expect an error here.
+	assert.NoError(t, err)
+}

pkg/goods/goods.go

@@ -20,6 +20,8 @@ var (
 	systemdfs embed.FS
 	//go:embed internal/bins/*
 	internalBinfs embed.FS
+	//go:embed static/*
+	staticfs embed.FS
 )
 
 // K0sBinarySHA256 returns the SHA256 checksum of the embedded k0s binary.

pkg/goods/materializer.go

@@ -76,6 +76,18 @@ func (m *Materializer) CalicoNetworkManagerConfig() error {
 	return nil
 }
 
+// SysctlConfig writes the embedded sysctl config to the /etc/sysctl.d directory.
+func (m *Materializer) SysctlConfig(dstpath string) error {
+	content, err := staticfs.ReadFile("static/99-embedded-cluster.conf")
+	if err != nil {
+		return fmt.Errorf("unable to open embedded sysctl config file: %w", err)
+	}
+	if err := os.WriteFile(dstpath, content, 0644); err != nil {
+		return fmt.Errorf("unable to write file: %w", err)
+	}
+	return nil
+}
+
 // Materialize writes to disk all embedded assets.
 func (m *Materializer) Materialize() error {
 	if err := m.Binaries(); err != nil {

pkg/goods/materializer_test.go

@@ -0,0 +1,34 @@
+package goods
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMaterializer_SysctlConfig(t *testing.T) {
+	m := NewMaterializer(nil)
+
+	// happy path.
+	dstdir, err := os.MkdirTemp("", "embedded-cluster-test")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dstdir)
+
+	dstpath := filepath.Join(dstdir, "sysctl.conf")
+	err = m.SysctlConfig(dstpath)
+	assert.NoError(t, err)
+
+	expected, err := os.ReadFile(dstpath)
+	assert.NoError(t, err)
+
+	content, err := staticfs.ReadFile("static/99-embedded-cluster.conf")
+	assert.NoError(t, err)
+	assert.Equal(t, string(expected), string(content))
+
+	// write to a non-existent directory.
+	dstpath = filepath.Join(dstdir, "dir-does-not-exist", "sysctl.conf")
+	err = m.SysctlConfig(dstpath)
+	assert.Contains(t, err.Error(), "no such file or directory")
+}

pkg/goods/static/99-embedded-cluster.conf

@@ -0,0 +1,11 @@
+# this entry enables ip forwarding. this feature is necessary as embedded
+# cluster creates virtual network interfaces and need the traffic among them to
+# be forwarded.
+net.ipv4.ip_forward = 1
+
+# arp filter and ignore need to be disabled otherwise we can't have arp
+# resolving across the calico network interfaces.
+net.ipv4.conf.default.arp_filter = 0
+net.ipv4.conf.default.arp_ignore = 0
+net.ipv4.conf.all.arp_filter = 0
+net.ipv4.conf.all.arp_ignore = 0