Skip to content

Commit

Permalink
Run host preflights as part of the install2 command (#1578)
Browse files Browse the repository at this point in the history 
* Run host preflights as part of the install2 command

---------

Co-authored-by: Andrew Lavery <[email protected]>
  • Loading branch information
@sgalsaleh @laverya
sgalsaleh and laverya authored Dec 5, 2024
1 parent dcf0031 commit 04e7357
Show file tree
Hide file tree
Showing 20 changed files with 412 additions and 216 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,11 +278,11 @@ vet: static

.PHONY: e2e-tests
e2e-tests: embedded-release
go test -timeout 60m -ldflags="$(LD_FLAGS)" -parallel 1 -failfast -v ./e2e
go test -tags exclude_graphdriver_btrfs -timeout 60m -ldflags="$(LD_FLAGS)" -parallel 1 -failfast -v ./e2e

.PHONY: e2e-test
e2e-test:
go test -timeout 60m -ldflags="$(LD_FLAGS)" -v ./e2e -run ^$(TEST_NAME)$$
go test -tags exclude_graphdriver_btrfs -timeout 60m -ldflags="$(LD_FLAGS)" -v ./e2e -run ^$(TEST_NAME)$$

.PHONY: dryrun-tests
dryrun-tests: export DRYRUN_MATCH = Test
Expand Down
42 changes: 29 additions & 13 deletions cmd/installer/cli/cidr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,29 +35,45 @@ func validateCIDRFlags(cmd *cobra.Command) error {
return nil
}

// getPODAndServiceCIDR determines, based on the command line flags,
// what are the pod and service CIDRs to be used for the cluster. If both
// --pod-cidr and --service-cidr have been set, they are used. Otherwise,
type CIDRConfig struct {
PodCIDR string
ServiceCIDR string
GlobalCIDR *string
}

// getCIDRConfig determines, based on the command line flags,
// what are the pod and service CIDRs to be used for the cluster. If either
// of --pod-cidr or --service-cidr have been set, they are used. Otherwise,
// the cidr flag is split into pod and service CIDRs.
func getPODAndServiceCIDR(cmd *cobra.Command) (string, string, error) {
func getCIDRConfig(cmd *cobra.Command) (*CIDRConfig, error) {
if cmd.Flags().Changed("pod-cidr") || cmd.Flags().Changed("service-cidr") {
podCIDRFlag, err := cmd.Flags().GetString("pod-cidr")
podCIDR, err := cmd.Flags().GetString("pod-cidr")
if err != nil {
return "", "", fmt.Errorf("unable to get pod-cidr flag: %w", err)
return nil, fmt.Errorf("unable to get pod-cidr flag: %w", err)
}
serviceCIDRFlag, err := cmd.Flags().GetString("service-cidr")
serviceCIDR, err := cmd.Flags().GetString("service-cidr")
if err != nil {
return "", "", fmt.Errorf("unable to get service-cidr flag: %w", err)
return nil, fmt.Errorf("unable to get service-cidr flag: %w", err)
}
return podCIDRFlag, serviceCIDRFlag, nil
return &CIDRConfig{
PodCIDR: podCIDR,
ServiceCIDR: serviceCIDR,
}, nil
}

cidrFlag, err := cmd.Flags().GetString("cidr")
globalCIDR, err := cmd.Flags().GetString("cidr")
if err != nil {
return "", "", fmt.Errorf("unable to get cidr flag: %w", err)
return nil, fmt.Errorf("unable to get cidr flag: %w", err)
}

return netutils.SplitNetworkCIDR(cidrFlag)
podCIDR, serviceCIDR, err := netutils.SplitNetworkCIDR(globalCIDR)
if err != nil {
return nil, fmt.Errorf("unable to split cidr flag: %w", err)
}
return &CIDRConfig{
PodCIDR: podCIDR,
ServiceCIDR: serviceCIDR,
GlobalCIDR: &globalCIDR,
}, nil
}

// cleanCIDR returns a `.0/x` subnet instead of a `.2/x` etc subnet
Expand Down
10 changes: 5 additions & 5 deletions cmd/installer/cli/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,12 +204,12 @@ func InstallCmd(ctx context.Context, name string) *cobra.Command {
proxyRegistryURL = fmt.Sprintf("https://%s", runtimeconfig.ProxyRegistryAddress)
}

fromCIDR, toCIDR, err := getPODAndServiceCIDR(cmd)
cidrCfg, err := getCIDRConfig(cmd)
if err != nil {
return fmt.Errorf("unable to determine pod and service CIDRs: %w", err)
}

if err := RunHostPreflights(cmd, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, proxy, fromCIDR, toCIDR, assumeYes); err != nil {
if err := RunHostPreflights(cmd, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, proxy, cidrCfg, assumeYes); err != nil {
metrics.ReportApplyFinished(cmd.Context(), licenseFile, nil, err)
if err == ErrPreflightsHaveFail {
return ErrNothingElseToAdd
Expand Down Expand Up @@ -643,12 +643,12 @@ func ensureK0sConfig(cmd *cobra.Command, applier *addons.Applier) (*k0sconfig.Cl
cfg.Spec.API.Address = address
cfg.Spec.Storage.Etcd.PeerAddress = address

podCIDR, serviceCIDR, err := getPODAndServiceCIDR(cmd)
cidrCfg, err := getCIDRConfig(cmd)
if err != nil {
return nil, fmt.Errorf("unable to determine pod and service CIDRs: %w", err)
}
cfg.Spec.Network.PodCIDR = podCIDR
cfg.Spec.Network.ServiceCIDR = serviceCIDR
cfg.Spec.Network.PodCIDR = cidrCfg.PodCIDR
cfg.Spec.Network.ServiceCIDR = cidrCfg.ServiceCIDR
if err := config.UpdateHelmConfigs(applier, cfg); err != nil {
return nil, fmt.Errorf("unable to update helm configs: %w", err)
}
Expand Down
47 changes: 19 additions & 28 deletions cmd/installer/cli/install2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,7 @@ type Install2CmdFlags struct {

license *kotsv1beta1.License
proxy *ecv1beta1.ProxySpec

// cidr flags are deprecated, but these values are still
// used. if the --cidr flag is passed, the values will be
// calculated
podCIDR string
serviceCIDR string
cidrCfg *CIDRConfig
}

// Install2Cmd returns a cobra command for installing the embedded cluster.
Expand Down Expand Up @@ -84,12 +79,11 @@ func Install2Cmd(ctx context.Context, name string) *cobra.Command {
}

// parse the various cidr flags to make sure we have exactly what we want
pod, svc, err := getPODAndServiceCIDR(cmd)
cidrCfg, err := getCIDRConfig(cmd)
if err != nil {
return fmt.Errorf("unable to determine pod and service CIDRs: %w", err)
}
flags.podCIDR = pod
flags.serviceCIDR = svc
flags.cidrCfg = cidrCfg

// if a network interface flag was not provided, attempt to discover it
if flags.networkInterface == "" {
Expand Down Expand Up @@ -179,8 +173,19 @@ func runInstall2(cmd *cobra.Command, args []string, name string, flags Install2C
}

logrus.Debugf("running host preflights")
if err := runInstallPreflights(cmd.Context(), flags.license, flags.proxy, flags.podCIDR, flags.serviceCIDR); err != nil {
return err
if err := preflights.PrepareAndRun(cmd.Context(), preflights.PrepareAndRunOptions{
License: flags.license,
Proxy: flags.proxy,
PodCIDR: flags.cidrCfg.PodCIDR,
ServiceCIDR: flags.cidrCfg.ServiceCIDR,
GlobalCIDR: flags.cidrCfg.GlobalCIDR,
PrivateCAs: flags.privateCAs,
IsAirgap: flags.airgapBundle != "",
SkipHostPreflights: flags.skipHostPreflights,
IgnoreHostPreflights: flags.ignoreHostPreflights,
AssumeYes: flags.assumeYes,
}); err != nil {
return fmt.Errorf("unable to prepare and run preflights: %w", err)
}

logrus.Debugf("configuring sysctl")
Expand All @@ -193,7 +198,7 @@ func runInstall2(cmd *cobra.Command, args []string, name string, flags Install2C
return fmt.Errorf("unable to configure network manager: %w", err)
}

clusterConfig, err := installAndStartCluster(cmd.Context(), flags.networkInterface, flags.airgapBundle, flags.license, flags.proxy, flags.podCIDR, flags.serviceCIDR, flags.overrides)
clusterConfig, err := installAndStartCluster(cmd.Context(), flags.networkInterface, flags.airgapBundle, flags.license, flags.proxy, flags.cidrCfg, flags.overrides)
if err != nil {
metrics.ReportApplyFinished(cmd.Context(), "", flags.license, err)
return err
Expand Down Expand Up @@ -324,27 +329,13 @@ func runInstallVerifyAndPrompt(ctx context.Context, name string, flags *Install2
return nil
}

func runInstallPreflights(ctx context.Context, license *kotsv1beta1.License, proxy *ecv1beta1.ProxySpec, podCIDR string, serviceCIDR string) error {
preflightOptions := preflights.PrepareAndRunOptions{
License: license,
Proxy: proxy,
ConnectivityFromCIDR: podCIDR,
ConnectivityToCIDR: serviceCIDR,
}
if err := preflights.PrepareAndRun(ctx, preflightOptions); err != nil {
return fmt.Errorf("unable to prepare and run preflights: %w", err)
}

return nil
}

func installAndStartCluster(ctx context.Context, networkInterface string, airgapBundle string, license *kotsv1beta1.License, proxy *ecv1beta1.ProxySpec, podCIDR string, serviceCIDR string, overrides string) (*k0sconfig.ClusterConfig, error) {
func installAndStartCluster(ctx context.Context, networkInterface string, airgapBundle string, license *kotsv1beta1.License, proxy *ecv1beta1.ProxySpec, cidrCfg *CIDRConfig, overrides string) (*k0sconfig.ClusterConfig, error) {
loading := spinner.Start()
defer loading.Close()
loading.Infof("Installing %s node", runtimeconfig.BinaryName())
logrus.Debugf("creating k0s configuration file")

cfg, err := k0s.WriteK0sConfig(ctx, networkInterface, airgapBundle, podCIDR, serviceCIDR, overrides)
cfg, err := k0s.WriteK0sConfig(ctx, networkInterface, airgapBundle, cidrCfg.PodCIDR, cidrCfg.ServiceCIDR, overrides)
if err != nil {
err := fmt.Errorf("unable to create config file: %w", err)
metrics.ReportApplyFinished(ctx, "", license, err)
Expand Down
38 changes: 8 additions & 30 deletions cmd/installer/cli/install_runpreflights.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/replicatedhq/embedded-cluster/pkg/helpers"
"github.com/replicatedhq/embedded-cluster/pkg/metrics"
"github.com/replicatedhq/embedded-cluster/pkg/preflights"
preflightstypes "github.com/replicatedhq/embedded-cluster/pkg/preflights/types"
"github.com/replicatedhq/embedded-cluster/pkg/prompts"
"github.com/replicatedhq/embedded-cluster/pkg/release"
"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
Expand Down Expand Up @@ -120,12 +121,12 @@ func InstallRunPreflightsCmd(ctx context.Context, name string) *cobra.Command {
proxyRegistryURL = fmt.Sprintf("https://%s", runtimeconfig.ProxyRegistryAddress)
}

fromCIDR, toCIDR, err := getPODAndServiceCIDR(cmd)
cidrCfg, err := getCIDRConfig(cmd)
if err != nil {
return fmt.Errorf("unable to determine pod and service CIDRs: %w", err)
}

if err := RunHostPreflights(cmd, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, proxy, fromCIDR, toCIDR, assumeYes); err != nil {
if err := RunHostPreflights(cmd, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, proxy, cidrCfg, assumeYes); err != nil {
if err == ErrPreflightsHaveFail {
return ErrNothingElseToAdd
}
Expand Down Expand Up @@ -329,15 +330,15 @@ func getAddonsApplier(cmd *cobra.Command, opts addonsApplierOpts, adminConsolePw
// RunHostPreflights runs the host preflights we found embedded in the binary
// on all configured hosts. We attempt to read HostPreflights from all the
// embedded Helm Charts and from the Kots Application Release files.
func RunHostPreflights(cmd *cobra.Command, applier *addons.Applier, replicatedAPIURL, proxyRegistryURL string, isAirgap bool, proxy *ecv1beta1.ProxySpec, fromCIDR, toCIDR string, assumeYes bool) error {
func RunHostPreflights(cmd *cobra.Command, applier *addons.Applier, replicatedAPIURL, proxyRegistryURL string, isAirgap bool, proxy *ecv1beta1.ProxySpec, cidrCfg *CIDRConfig, assumeYes bool) error {
hpf, err := applier.HostPreflights()
if err != nil {
return fmt.Errorf("unable to read host preflights: %w", err)
}

privateCAs := getPrivateCAPath(cmd)

data, err := preflights.TemplateData{
data, err := preflightstypes.TemplateData{
ReplicatedAPIURL: replicatedAPIURL,
ProxyRegistryURL: proxyRegistryURL,
IsAirgap: isAirgap,
Expand All @@ -348,9 +349,9 @@ func RunHostPreflights(cmd *cobra.Command, applier *addons.Applier, replicatedAP
OpenEBSDataDir: runtimeconfig.EmbeddedClusterOpenEBSLocalSubDir(),
PrivateCA: privateCAs,
SystemArchitecture: runtime.GOARCH,
FromCIDR: fromCIDR,
ToCIDR: toCIDR,
}.WithCIDRData(getCIDRs(cmd))
FromCIDR: cidrCfg.PodCIDR,
ToCIDR: cidrCfg.ServiceCIDR,
}.WithCIDRData(cidrCfg.PodCIDR, cidrCfg.ServiceCIDR, cidrCfg.GlobalCIDR)

if err != nil {
return fmt.Errorf("unable to get host preflights data: %w", err)
Expand Down Expand Up @@ -525,26 +526,3 @@ func getPrivateCAPath(cmd *cobra.Command) string {
}
return privateCA
}

// getCIDRs returns the CIDRs in use based on the provided cli flags.
func getCIDRs(cmd *cobra.Command) (string, string, string) {
podCIDRFlag, err := cmd.Flags().GetString("pod-cidr")
if err != nil {
return "", "", ""
}

serviceCIDRFlag, err := cmd.Flags().GetString("service-cidr")
if err != nil {
return "", "", ""
}

cidrFlag, err := cmd.Flags().GetString("cidr")
if err != nil {
return "", "", ""
}

if podCIDRFlag != "" || serviceCIDRFlag != "" {
return podCIDRFlag, serviceCIDRFlag, ""
}
return "", "", cidrFlag
}
13 changes: 9 additions & 4 deletions cmd/installer/cli/join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,24 +160,29 @@ func JoinCmd(ctx context.Context, name string) *cobra.Command {
return fmt.Errorf("unable to configure sysctl: %w", err)
}

fromCIDR, toCIDR, err := netutils.SplitNetworkCIDR(ecv1beta1.DefaultNetworkCIDR)
podCIDR, serviceCIDR, err := netutils.SplitNetworkCIDR(ecv1beta1.DefaultNetworkCIDR)
if err != nil {
return fmt.Errorf("unable to split default network CIDR: %w", err)
}

if jcmd.InstallationSpec.Network != nil {
if jcmd.InstallationSpec.Network.PodCIDR != "" {
fromCIDR = jcmd.InstallationSpec.Network.PodCIDR
podCIDR = jcmd.InstallationSpec.Network.PodCIDR
}
if jcmd.InstallationSpec.Network.ServiceCIDR != "" {
toCIDR = jcmd.InstallationSpec.Network.ServiceCIDR
serviceCIDR = jcmd.InstallationSpec.Network.ServiceCIDR
}
}

cidrCfg := &CIDRConfig{
PodCIDR: podCIDR,
ServiceCIDR: serviceCIDR,
}

// jcmd.InstallationSpec.MetricsBaseURL is the replicated.app endpoint url
replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL
proxyRegistryURL := fmt.Sprintf("https://%s", runtimeconfig.ProxyRegistryAddress)
if err := RunHostPreflights(cmd, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy, fromCIDR, toCIDR, assumeYes); err != nil {
if err := RunHostPreflights(cmd, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy, cidrCfg, assumeYes); err != nil {
metrics.ReportJoinFailed(cmd.Context(), jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
if err == ErrPreflightsHaveFail {
return ErrNothingElseToAdd
Expand Down
13 changes: 9 additions & 4 deletions cmd/installer/cli/join_runpreflights.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,24 +100,29 @@ func JoinRunPreflightsCmd(ctx context.Context, name string) *cobra.Command {
return err
}

fromCIDR, toCIDR, err := netutils.SplitNetworkCIDR(ecv1beta1.DefaultNetworkCIDR)
podCIDR, serviceCIDR, err := netutils.SplitNetworkCIDR(ecv1beta1.DefaultNetworkCIDR)
if err != nil {
return fmt.Errorf("unable to split default network CIDR: %w", err)
}

if jcmd.InstallationSpec.Network != nil {
if jcmd.InstallationSpec.Network.PodCIDR != "" {
fromCIDR = jcmd.InstallationSpec.Network.PodCIDR
podCIDR = jcmd.InstallationSpec.Network.PodCIDR
}
if jcmd.InstallationSpec.Network.ServiceCIDR != "" {
toCIDR = jcmd.InstallationSpec.Network.ServiceCIDR
serviceCIDR = jcmd.InstallationSpec.Network.ServiceCIDR
}
}

cidrCfg := &CIDRConfig{
PodCIDR: podCIDR,
ServiceCIDR: serviceCIDR,
}

logrus.Debugf("running host preflights")
replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL
proxyRegistryURL := fmt.Sprintf("https://%s", runtimeconfig.ProxyRegistryAddress)
if err := RunHostPreflights(cmd, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy, fromCIDR, toCIDR, assumeYes); err != nil {
if err := RunHostPreflights(cmd, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy, cidrCfg, assumeYes); err != nil {
if err == ErrPreflightsHaveFail {
return ErrNothingElseToAdd
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/installer/cli/proxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,11 +96,11 @@ func combineNoProxySuppliedValuesAndDefaults(cmd *cobra.Command, proxy *ecv1beta
noProxy := strings.Split(proxy.ProvidedNoProxy, ",")
if len(noProxy) > 0 || proxy.HTTPProxy != "" || proxy.HTTPSProxy != "" {
noProxy = append(runtimeconfig.DefaultNoProxy, noProxy...)
podnet, svcnet, err := getPODAndServiceCIDR(cmd)
cidrCfg, err := getCIDRConfig(cmd)
if err != nil {
return fmt.Errorf("unable to determine pod and service CIDRs: %w", err)
}
noProxy = append(noProxy, podnet, svcnet)
noProxy = append(noProxy, cidrCfg.PodCIDR, cidrCfg.ServiceCIDR)
proxy.NoProxy = strings.Join(noProxy, ",")
}
return nil
Expand Down
6 changes: 3 additions & 3 deletions cmd/installer/cli/restore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -751,13 +751,13 @@ func ensureK0sConfigForRestore(cmd *cobra.Command, applier *addons.Applier) (*k0
cfg.Spec.API.Address = address
cfg.Spec.Storage.Etcd.PeerAddress = address

podCIDR, serviceCIDR, err := getPODAndServiceCIDR(cmd)
cidrCfg, err := getCIDRConfig(cmd)
if err != nil {
return nil, fmt.Errorf("unable to determine pod and service CIDRs: %w", err)
}

cfg.Spec.Network.PodCIDR = podCIDR
cfg.Spec.Network.ServiceCIDR = serviceCIDR
cfg.Spec.Network.PodCIDR = cidrCfg.PodCIDR
cfg.Spec.Network.ServiceCIDR = cidrCfg.ServiceCIDR

if err := config.UpdateHelmConfigsForRestore(applier, cfg); err != nil {
return nil, fmt.Errorf("unable to update helm configs: %w", err)
Expand Down
Loading

0 comments on commit 04e7357

Please sign in to comment.