operator uninstall with operands (#45)

Signed-off-by: Daniel Sover <[email protected]>

go.mod

@@ -5,7 +5,7 @@ go 1.16
 require (
 	github.com/containerd/containerd v1.4.3
 	github.com/onsi/ginkgo v1.14.1
-	github.com/onsi/gomega v1.10.2
+	github.com/onsi/gomega v1.11.0
 	github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6
 	github.com/operator-framework/api v0.7.1
 	github.com/operator-framework/operator-lifecycle-manager v0.0.0-20200521062108-408ca95d458f

go.sum

@@ -583,6 +583,8 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
 github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug=
+github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
@@ -898,6 +900,8 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1145,6 +1149,8 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/cmd/operator_list_operands.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"sort"
 	"strings"
 	"text/tabwriter"
 	"time"
@@ -64,18 +63,6 @@ the operator's ClusterServiceVersion.`,
 				return
 			}
 
-			sort.Slice(operands.Items, func(i, j int) bool {
-				if operands.Items[i].GetAPIVersion() != operands.Items[j].GetAPIVersion() {
-					return operands.Items[i].GetAPIVersion() < operands.Items[j].GetAPIVersion()
-				}
-				if operands.Items[i].GetKind() != operands.Items[j].GetKind() {
-					return operands.Items[i].GetKind() < operands.Items[j].GetKind()
-				}
-				if operands.Items[i].GetNamespace() != operands.Items[j].GetNamespace() {
-					return operands.Items[i].GetNamespace() < operands.Items[j].GetNamespace()
-				}
-				return operands.Items[i].GetName() < operands.Items[j].GetName()
-			})
 			if err := writeOutput(os.Stdout, operands); err != nil {
 				log.Fatal(err)
 			}

internal/cmd/operator_uninstall.go

@@ -15,8 +15,29 @@ func newOperatorUninstallCmd(cfg *action.Configuration) *cobra.Command {
 
 	cmd := &cobra.Command{
 		Use:   "uninstall <operator>",
-		Short: "Uninstall an operator",
-		Args:  cobra.ExactArgs(1),
+		Short: "Uninstall an operator and operands",
+		Long: `Uninstall removes the subscription, operator and optionally operands managed by the operator as well as 
+the relevant operatorgroup.
+
+Warning: this command permanently deletes objects from the cluster. Running uninstall concurrently with other operations
+could result in undefined behavior. 
+
+The uninstall command first checks to find the subscription associated with the operator. It then 
+lists all operands found throughout the cluster for the operator
+specified if one is found. Since the scope of an operator is restricted by
+its operator group, this search will include namespace-scoped operands from the
+operator group's target namespaces and all cluster-scoped operands. 
+
+The operand-deletion strategy is then considered if any operands are found on-cluster. One of cancel|ignore|delete. 
+By default, the strategy is "cancel", which means that if any operands are found when deleting the operator abort the 
+uninstall without deleting anything. 
+The "ignore" strategy keeps the operands on cluster and deletes the subscription and the operator.
+The "delete" strategy deletes the subscription, operands, and after they have finished finalizing, the operator itself.
+
+Setting --delete-operator-groups to true will delete the operatorgroup in the provided namespace if no other active 
+subscriptions are currently in that namespace, after removing the operator. The subscription and operatorgroup will be 
+removed even if the operator is not found.`,
+		Args: cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			u.Package = args[0]
 			if err := u.Run(cmd.Context()); err != nil {
@@ -30,8 +51,7 @@ func newOperatorUninstallCmd(cfg *action.Configuration) *cobra.Command {
 }
 
 func bindOperatorUninstallFlags(fs *pflag.FlagSet, u *internalaction.OperatorUninstall) {
-	fs.BoolVarP(&u.DeleteAll, "delete-all", "X", false, "enable all delete flags")
-	fs.BoolVar(&u.DeleteCRDs, "delete-crds", false, "delete all owned CRDs and all CRs")
 	fs.BoolVar(&u.DeleteOperatorGroups, "delete-operator-groups", false, "delete operator groups if no other operators remain")
 	fs.StringSliceVar(&u.DeleteOperatorGroupNames, "delete-operator-group-names", nil, "specific operator group names to delete (only effective with --delete-operator-groups)")
+	fs.VarP(&u.OperandStrategy, "operand-strategy", "s", "determines how to handle operands when deleting the operator, one of cancel|ignore|delete")
 }

internal/pkg/action/action_suite_test.go

@@ -0,0 +1,13 @@
+package action_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestCommand(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Internal action Suite")
+}

internal/pkg/action/constants.go

@@ -2,5 +2,4 @@ package action
 
 const (
 	csvKind = "ClusterServiceVersion"
-	crdKind = "CustomResourceDefinition"
 )

internal/pkg/action/operator_uninstall.go

@@ -9,23 +9,21 @@ import (
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/operator-framework/kubectl-operator/internal/pkg/operand"
 	"github.com/operator-framework/kubectl-operator/pkg/action"
 )
 
 type OperatorUninstall struct {
 	config *action.Configuration
 
 	Package                  string
-	DeleteAll                bool
-	DeleteCRDs               bool
+	OperandStrategy          operand.DeletionStrategy
 	DeleteOperatorGroups     bool
 	DeleteOperatorGroupNames []string
-
-	Logf func(string, ...interface{})
+	Logf                     func(string, ...interface{})
 }
 
 func NewOperatorUninstall(cfg *action.Configuration) *OperatorUninstall {
@@ -44,11 +42,6 @@ func (e ErrPackageNotFound) Error() string {
 }
 
 func (u *OperatorUninstall) Run(ctx context.Context) error {
-	if u.DeleteAll {
-		u.DeleteCRDs = true
-		u.DeleteOperatorGroups = true
-	}
-
 	subs := v1alpha1.SubscriptionList{}
 	if err := u.config.Client.List(ctx, &subs, client.InNamespace(u.config.Namespace)); err != nil {
 		return fmt.Errorf("list subscriptions: %v", err)
@@ -69,62 +62,56 @@ func (u *OperatorUninstall) Run(ctx context.Context) error {
 	csv, csvName, err := u.getSubscriptionCSV(ctx, sub)
 	if err != nil && !apierrors.IsNotFound(err) {
 		if csvName == "" {
-			return fmt.Errorf("get subscription CSV: %v", err)
+			return fmt.Errorf("get subscription csv: %v", err)
 		}
-		return fmt.Errorf("get subscription CSV %q: %v", csvName, err)
+		return fmt.Errorf("get subscription csv %q: %v", csvName, err)
 	}
 
-	// Deletion order:
-	//
-	// 1. Subscription to prevent further installs or upgrades of the operator while cleaning up.
-	// 2. CustomResourceDefinitions so the operator has a chance to handle CRs that have finalizers.
-	// 3. ClusterServiceVersion. OLM puts an ownerref on every namespaced resource to the CSV,
-	//    and an owner label on every cluster scoped resource so they get gc'd on deletion.
+	// find operands related to the operator on cluster
+	lister := action.NewOperatorListOperands(u.config)
+	operands, err := lister.Run(ctx, u.Package)
+	if err != nil {
+		return fmt.Errorf("list operands for operator %q: %v", u.Package, err)
+	}
+	// validate the provided deletion strategy before proceeding to deletion
+	if err := u.validStrategy(operands); err != nil {
+		return fmt.Errorf("could not proceed with deletion of %q: %s", u.Package, err)
+	}
+
+	/*
+		Deletion order:
+		1. Subscription to prevent further installs or upgrades of the operator while cleaning up.
+
+		If the CSV exists:
+		  2. Operands so the operator has a chance to handle CRs that have finalizers.
+		  Note: the correct strategy must be chosen in order to process an opertor delete with operand on-cluster.
+		  3. ClusterServiceVersion. OLM puts an ownerref on every namespaced resource to the CSV,
+		   and an owner label on every cluster scoped resource so they get gc'd on deletion.
+
+		4. OperatorGroup in the namespace if no other subscriptions are in that namespace and OperatorGroup deletion is specified
+	*/
 
 	// Subscriptions can be deleted asynchronously.
 	if err := u.deleteObjects(ctx, sub); err != nil {
 		return err
 	}
 
-	if csv != nil {
-		// Ensure CustomResourceDefinitions are deleted next, so that the operator
-		// has a chance to handle CRs that have finalizers.
-		if u.DeleteCRDs {
-			crds := getCRDs(csv)
-			if err := u.deleteObjects(ctx, crds...); err != nil {
-				return err
-			}
-		}
-
-		// OLM puts an ownerref on every namespaced resource to the CSV,
-		// and an owner label on every cluster scoped resource. When CSV is deleted
-		// kube and olm gc will remove all the referenced resources.
-		if err := u.deleteObjects(ctx, csv); err != nil {
+	// If we could not find a csv associated with the subscription, that likely
+	// means there is no CSV associated with it yet. Delete non-CSV related items only like the operatorgroup.
+	if csv == nil {
+		u.Logf("csv for package %q not found", u.Package)
+	} else {
+		if err := u.deleteCSVRelatedResources(ctx, csv, operands); err != nil {
 			return err
 		}
 	}
 
 	if u.DeleteOperatorGroups {
-		subs := v1alpha1.SubscriptionList{}
-		if err := u.config.Client.List(ctx, &subs, client.InNamespace(u.config.Namespace)); err != nil {
-			return fmt.Errorf("list subscriptions: %v", err)
-		}
-		// If there are no subscriptions left, delete the operator group(s).
-		if len(subs.Items) == 0 {
-			ogs := v1.OperatorGroupList{}
-			if err := u.config.Client.List(ctx, &ogs, client.InNamespace(u.config.Namespace)); err != nil {
-				return fmt.Errorf("list operatorgroups: %v", err)
-			}
-			for _, og := range ogs.Items {
-				og := og
-				if len(u.DeleteOperatorGroupNames) == 0 || contains(u.DeleteOperatorGroupNames, og.GetName()) {
-					if err := u.deleteObjects(ctx, &og); err != nil {
-						return err
-					}
-				}
-			}
+		if err := u.deleteOperatorGroup(ctx); err != nil {
+			return fmt.Errorf("delete operatorgroup: %v", err)
 		}
 	}
+
 	return nil
 }
 
@@ -165,28 +152,73 @@ func (u *OperatorUninstall) getSubscriptionCSV(ctx context.Context, subscription
 	return csv, name, nil
 }
 
-func csvNameFromSubscription(subscription *v1alpha1.Subscription) string {
-	if subscription.Status.InstalledCSV != "" {
-		return subscription.Status.InstalledCSV
+func (u *OperatorUninstall) deleteOperatorGroup(ctx context.Context) error {
+	subs := v1alpha1.SubscriptionList{}
+	if err := u.config.Client.List(ctx, &subs, client.InNamespace(u.config.Namespace)); err != nil {
+		return fmt.Errorf("list subscriptions: %v", err)
 	}
-	return subscription.Status.CurrentCSV
+
+	// If there are no subscriptions left, delete the operator group(s).
+	if len(subs.Items) == 0 {
+		ogs := v1.OperatorGroupList{}
+		if err := u.config.Client.List(ctx, &ogs, client.InNamespace(u.config.Namespace)); err != nil {
+			return fmt.Errorf("list operatorgroups: %v", err)
+		}
+		for _, og := range ogs.Items {
+			og := og
+			if len(u.DeleteOperatorGroupNames) == 0 || contains(u.DeleteOperatorGroupNames, og.GetName()) {
+				if err := u.deleteObjects(ctx, &og); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
 }
 
-// getCRDs returns the list of CRDs required by a CSV.
-func getCRDs(csv *v1alpha1.ClusterServiceVersion) (crds []client.Object) {
-	for _, resource := range csv.Status.RequirementStatus {
-		if resource.Kind == crdKind {
-			obj := &unstructured.Unstructured{}
-			obj.SetGroupVersionKind(schema.GroupVersionKind{
-				Group:   resource.Group,
-				Version: resource.Version,
-				Kind:    resource.Kind,
-			})
-			obj.SetName(resource.Name)
-			crds = append(crds, obj)
+// validStrategy validates the deletion strategy against the operands on-cluster
+// TODO define and use an OperandStrategyError that the cmd can use errors.As() on to provide external callers a more generic error
+func (u *OperatorUninstall) validStrategy(operands *unstructured.UnstructuredList) error {
+	if len(operands.Items) > 0 && u.OperandStrategy.Kind == operand.Cancel {
+		return fmt.Errorf("%d operands exist and operand strategy %q is in use: "+
+			"delete operands manually or re-run uninstall with a different operand deletion strategy."+
+			"\n\nSee kubectl operator uninstall --help for more information on operand deletion strategies.", len(operands.Items), operand.Cancel)
+	}
+	return nil
+}
+
+func (u *OperatorUninstall) deleteCSVRelatedResources(ctx context.Context, csv *v1alpha1.ClusterServiceVersion, operands *unstructured.UnstructuredList) error {
+	switch u.OperandStrategy.Kind {
+	case operand.Ignore:
+		for _, op := range operands.Items {
+			u.Logf("%s %q orphaned", strings.ToLower(op.GetKind()), prettyPrint(op))
+		}
+	case operand.Delete:
+		for _, op := range operands.Items {
+			op := op
+			if err := u.deleteObjects(ctx, &op); err != nil {
+				return err
+			}
 		}
+	default:
+		return fmt.Errorf("unknown operand deletion strategy %q", u.OperandStrategy)
+	}
+
+	// OLM puts an ownerref on every namespaced resource to the CSV,
+	// and an owner label on every cluster scoped resource. When CSV is deleted
+	// kube and olm gc will remove all the referenced resources.
+	if err := u.deleteObjects(ctx, csv); err != nil {
+		return err
 	}
-	return
+
+	return nil
+}
+
+func csvNameFromSubscription(subscription *v1alpha1.Subscription) string {
+	if subscription.Status.InstalledCSV != "" {
+		return subscription.Status.InstalledCSV
+	}
+	return subscription.Status.CurrentCSV
 }
 
 func contains(haystack []string, needle string) bool {
@@ -197,3 +229,11 @@ func contains(haystack []string, needle string) bool {
 	}
 	return false
 }
+
+func prettyPrint(op unstructured.Unstructured) string {
+	namespaced := op.GetNamespace() != ""
+	if namespaced {
+		return fmt.Sprint(op.GetName() + "/" + op.GetNamespace())
+	}
+	return op.GetName()
+}