Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(metrics): kots will send upgrade events instead of the operator #1527

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,9 @@ rebuild-release: check-env-EC_VERSION check-env-APP_VERSION
./scripts/build-and-release.sh

.PHONY: upgrade-release
upgrade-release: export EC_VERSION = $(VERSION)-$(CURRENT_USER)-upgrade
upgrade-release: export APP_VERSION = appver-dev-$(call random-string)-upgrade
upgrade-release: RANDOM_STRING = $(call random-string)
upgrade-release: export EC_VERSION = $(VERSION)-$(CURRENT_USER)-upgrade-$(RANDOM_STRING)
upgrade-release: export APP_VERSION = appver-dev-$(call random-string)-upgrade-$(RANDOM_STRING)
upgrade-release: check-env-EC_VERSION check-env-APP_VERSION
UPLOAD_BINARIES=1 \
RELEASE_YAML_DIR=e2e/kots-release-upgrade \
Expand Down
5 changes: 4 additions & 1 deletion operator/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ include ../common.mk
OS ?= linux
ARCH ?= $(shell go env GOARCH)

ADMIN_CONSOLE_IMAGE_OVERRIDE =

# VERSION defines the project version for the bundle.
# Update this value when you upgrade the version of your project.
# To re-generate a bundle for another specific version without changing the standard setup, you can:
Expand All @@ -29,7 +31,8 @@ export PATH := $(shell pwd)/bin:$(PATH)

LD_FLAGS = \
-X github.com/replicatedhq/embedded-cluster/pkg/versions.K0sVersion=$(K0S_VERSION) \
-X github.com/replicatedhq/embedded-cluster/pkg/versions.Version=$(VERSION)
-X github.com/replicatedhq/embedded-cluster/pkg/versions.Version=$(VERSION) \
-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.AdminConsoleImageOverride=$(ADMIN_CONSOLE_IMAGE_OVERRIDE)

# Setting SHELL to bash allows bash commands to be executed by recipes.
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
Expand Down
47 changes: 0 additions & 47 deletions operator/controllers/installation_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -248,47 +248,6 @@ func (r *InstallationReconciler) ReportNodesChanges(ctx context.Context, in *v1b
}
}

// ReportInstallationChanges reports back to the metrics server if the installation status has changed.
func (r *InstallationReconciler) ReportInstallationChanges(ctx context.Context, before, after *v1beta1.Installation) {
if len(before.Status.State) == 0 || before.Status.State == after.Status.State {
return
}
var err error
var beforeVer, afterVer string
if before.Spec.Config != nil {
beforeVer = before.Spec.Config.Version
}
if after.Spec.Config != nil {
afterVer = after.Spec.Config.Version
}
switch after.Status.State {
case v1beta1.InstallationStateInstalling:
if beforeVer != "" {
err = metrics.NotifyUpgradeStarted(ctx, after.Spec.MetricsBaseURL, metrics.UpgradeStartedEvent{
ClusterID: after.Spec.ClusterID,
TargetVersion: afterVer,
InitialVersion: beforeVer,
})
}
case v1beta1.InstallationStateInstalled:
err = metrics.NotifyUpgradeSucceeded(ctx, after.Spec.MetricsBaseURL, metrics.UpgradeSucceededEvent{
ClusterID: after.Spec.ClusterID,
TargetVersion: afterVer,
InitialVersion: beforeVer,
})
case v1beta1.InstallationStateFailed:
err = metrics.NotifyUpgradeFailed(ctx, after.Spec.MetricsBaseURL, metrics.UpgradeFailedEvent{
ClusterID: after.Spec.ClusterID,
Reason: after.Status.Reason,
TargetVersion: afterVer,
InitialVersion: beforeVer,
})
}
if err != nil {
ctrl.LoggerFrom(ctx).Error(err, "failed to notify cluster installation status")
}
}

func (r *InstallationReconciler) ReconcileOpenebs(ctx context.Context, in *v1beta1.Installation) error {
log := ctrl.LoggerFrom(ctx)

Expand Down Expand Up @@ -641,11 +600,6 @@ func (r *InstallationReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, fmt.Errorf("failed to update installation status: %w", err)
}

// we create a copy of the installation so we can compare if it
// changed its status after the reconcile (this is mostly for
// calling back to us with events).
before := in.DeepCopy()

// verify if a new node has been added, removed or changed.
events, err := r.ReconcileNodeStatuses(ctx, in)
if err != nil {
Expand Down Expand Up @@ -697,7 +651,6 @@ func (r *InstallationReconciler) Reconcile(ctx context.Context, req ctrl.Request
// if we are not in an airgap environment this is the time to call back to
// replicated and inform the status of this installation.
if !in.Spec.AirGap {
r.ReportInstallationChanges(ctx, before, in)
r.ReportNodesChanges(ctx, in, events)
}

Expand Down
11 changes: 0 additions & 11 deletions operator/pkg/cli/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

clusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
"github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil"
"github.com/replicatedhq/embedded-cluster/operator/pkg/metrics"
"github.com/replicatedhq/embedded-cluster/operator/pkg/upgrade"
"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -59,16 +58,6 @@ func UpgradeCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("failed to upgrade: %w", err)
}
if !in.Spec.AirGap {
err = metrics.NotifyUpgradeStarted(cmd.Context(), in.Spec.MetricsBaseURL, metrics.UpgradeStartedEvent{
ClusterID: in.Spec.ClusterID,
TargetVersion: in.Spec.Config.Version,
InitialVersion: previousInstallation.Spec.Config.Version,
})
if err != nil {
fmt.Printf("failed to report that the upgrade was started: %v\n", err)
}
}

fmt.Println("Upgrade job created successfully")

Expand Down
15 changes: 0 additions & 15 deletions operator/pkg/cli/upgrade_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,9 @@ package cli

import (
"fmt"
"strings"
"time"

"github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil"
"github.com/replicatedhq/embedded-cluster/operator/pkg/metrics"
"github.com/replicatedhq/embedded-cluster/operator/pkg/upgrade"
"github.com/replicatedhq/embedded-cluster/pkg/versions"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -44,25 +42,12 @@ func UpgradeJobCmd() *cobra.Command {
fmt.Printf("Upgrading to installation %s (version %s)\n", in.Name, in.Spec.Config.Version)

i := 0
allErrors := []string{}
sleepDuration := time.Second * 5
for {
err = upgrade.Upgrade(cmd.Context(), cli, in)
if err != nil {
fmt.Printf("Upgrade failed, retrying: %s\n", err.Error())
allErrors = append(allErrors, err.Error())
if i >= 10 {
if !in.Spec.AirGap {
err = metrics.NotifyUpgradeFailed(cmd.Context(), in.Spec.MetricsBaseURL, metrics.UpgradeFailedEvent{
ClusterID: in.Spec.ClusterID,
TargetVersion: in.Spec.Config.Version,
InitialVersion: previousInstallationVersion,
Reason: strings.Join(allErrors, ", "),
})
if err != nil {
fmt.Printf("failed to report that the upgrade was started: %v\n", err)
}
}
return fmt.Errorf("failed to upgrade after %s", (sleepDuration * time.Duration(i)).String())
}

Expand Down
37 changes: 0 additions & 37 deletions operator/pkg/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,28 +39,6 @@ type NodeEvent struct {
Version string `json:"version"`
}

// UpgradeStartedEvent is send back home when the upgrade starts.
type UpgradeStartedEvent struct {
ClusterID string `json:"clusterID"`
TargetVersion string `json:"targetVersion"`
InitialVersion string `json:"initialVersion"`
}

// UpgradeFailedEvent is send back home when the upgrade fails.
type UpgradeFailedEvent struct {
ClusterID string `json:"clusterID"`
TargetVersion string `json:"targetVersion"`
InitialVersion string `json:"initialVersion"`
Reason string `json:"reason"`
}

// UpgradeSucceededEvent event is send back home when the upgrade succeeds.
type UpgradeSucceededEvent struct {
ClusterID string `json:"clusterID"`
TargetVersion string `json:"targetVersion"`
InitialVersion string `json:"initialVersion"`
}

// Hash returns the hash of the node.
func (n NodeEvent) Hash() (string, error) {
hasher := sha256.New()
Expand Down Expand Up @@ -117,18 +95,3 @@ func NotifyNodeAdded(ctx context.Context, baseURL string, ev NodeEvent) error {
func NotifyNodeRemoved(ctx context.Context, baseURL string, ev NodeRemovedEvent) error {
return sendEvent(ctx, "NodeRemoved", baseURL, ev)
}

// NotifyUpgradeStarted notifies the metrics server that an upgrade has started.
func NotifyUpgradeStarted(ctx context.Context, baseURL string, ev UpgradeStartedEvent) error {
return sendEvent(ctx, "UpgradeStarted", baseURL, ev)
}

// NotifyUpgradeFailed notifies the metrics server that an upgrade has failed.
func NotifyUpgradeFailed(ctx context.Context, baseURL string, ev UpgradeFailedEvent) error {
return sendEvent(ctx, "UpgradeFailed", baseURL, ev)
}

// NotifyUpgradeSucceeded notifies the metrics server that an upgrade has succeeded.
func NotifyUpgradeSucceeded(ctx context.Context, baseURL string, ev UpgradeSucceededEvent) error {
return sendEvent(ctx, "UpgradeSucceeded", baseURL, ev)
}
21 changes: 10 additions & 11 deletions pkg/addons/adminconsole/adminconsole.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,11 @@ import (
_ "embed"
"encoding/base64"
"fmt"
"github.com/replicatedhq/embedded-cluster/pkg/versions"
"time"

k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
"github.com/replicatedhq/embedded-cluster/kinds/types"
"github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/bcrypt"
"gopkg.in/yaml.v3"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/replicatedhq/embedded-cluster/pkg/addons/registry"
"github.com/replicatedhq/embedded-cluster/pkg/defaults"
"github.com/replicatedhq/embedded-cluster/pkg/helm"
Expand All @@ -33,6 +22,16 @@ import (
"github.com/replicatedhq/embedded-cluster/pkg/netutils"
"github.com/replicatedhq/embedded-cluster/pkg/release"
"github.com/replicatedhq/embedded-cluster/pkg/spinner"
"github.com/replicatedhq/embedded-cluster/pkg/versions"
"github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/bcrypt"
"gopkg.in/yaml.v3"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
Expand Down
Loading