use CCO config object

- [x] to determine whether CCO is in manual/disabled mode
- [x] to bypass permissions checking when annotating the cloud secret (AWS/Azure/GCP)
- [x] update role to grant access to read the config
- [x] copy generated CRD from openshift/api into manifests so that it gets applied
- [x] add clusteroperator status reporting for bad modes (and conflicting legacy settings and new modes).

Fix conditions merging to not drop old conditions when new StatusHandler has no conditions to set.
Fix fallout from that fix where StatusHandlers registered during tests were stacking up and interfering with each other.

manifests/00-config-crd.yaml

@@ -0,0 +1,155 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: cloudcredentials.operator.openshift.io
+spec:
+  scope: Cluster
+  preserveUnknownFields: false
+  group: operator.openshift.io
+  names:
+    kind: CloudCredential
+    listKind: CloudCredentialList
+    plural: cloudcredentials
+    singular: cloudcredential
+  subresources:
+    status: {}
+  versions:
+  - name: v1
+    served: true
+    storage: true
+  validation:
+    openAPIV3Schema:
+      description: CloudCredential provides a means to configure an operator to manage
+        CredentialsRequests.
+      type: object
+      required:
+      - spec
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: CloudCredentialSpec is the specification of the desired behavior
+            of the cloud-credential-operator.
+          type: object
+          properties:
+            credentialsMode:
+              description: CredentialsMode allows informing CCO that it should not
+                attempt to dynamically determine the root cloud credentials capabilities,
+                and it should just run in the specified mode. It also allows putting
+                the operator into "manual" mode if desired. Leaving the field in default
+                mode runs CCO so that the cluster's cloud credentials will be dynamically
+                probed for capabilities (on supported clouds/platforms).
+              type: string
+              enum:
+              - ""
+              - Manual
+              - Mint
+              - Passthrough
+            logLevel:
+              description: logLevel is an intent based logging for an overall component.  It
+                does not give fine grained control, but it is a simple way to manage
+                coarse grained logging choices that operators have to interpret for
+                their operands.
+              type: string
+            managementState:
+              description: managementState indicates whether and how the operator
+                should manage the component
+              type: string
+              pattern: ^(Managed|Unmanaged|Force|Removed)$
+            observedConfig:
+              description: observedConfig holds a sparse config that controller has
+                observed from the cluster state.  It exists in spec because it is
+                an input to the level for the operator
+              type: object
+              nullable: true
+              x-kubernetes-preserve-unknown-fields: true
+            operatorLogLevel:
+              description: operatorLogLevel is an intent based logging for the operator
+                itself.  It does not give fine grained control, but it is a simple
+                way to manage coarse grained logging choices that operators have to
+                interpret for themselves.
+              type: string
+            unsupportedConfigOverrides:
+              description: 'unsupportedConfigOverrides holds a sparse config that
+                will override any previously set options.  It only needs to be the
+                fields to override it will end up overlaying in the following order:
+                1. hardcoded defaults 2. observedConfig 3. unsupportedConfigOverrides'
+              type: object
+              nullable: true
+              x-kubernetes-preserve-unknown-fields: true
+        status:
+          description: CloudCredentialStatus defines the observed status of the cloud-credential-operator.
+          type: object
+          properties:
+            conditions:
+              description: conditions is a list of conditions and their status
+              type: array
+              items:
+                description: OperatorCondition is just the standard condition fields.
+                type: object
+                properties:
+                  lastTransitionTime:
+                    type: string
+                    format: date-time
+                  message:
+                    type: string
+                  reason:
+                    type: string
+                  status:
+                    type: string
+                  type:
+                    type: string
+            generations:
+              description: generations are used to determine when an item needs to
+                be reconciled or has changed in a way that needs a reaction.
+              type: array
+              items:
+                description: GenerationStatus keeps track of the generation for a
+                  given resource so that decisions about forced updates can be made.
+                type: object
+                properties:
+                  group:
+                    description: group is the group of the thing you're tracking
+                    type: string
+                  hash:
+                    description: hash is an optional field set for resources without
+                      generation that are content sensitive like secrets and configmaps
+                    type: string
+                  lastGeneration:
+                    description: lastGeneration is the last generation of the workload
+                      controller involved
+                    type: integer
+                    format: int64
+                  name:
+                    description: name is the name of the thing you're tracking
+                    type: string
+                  namespace:
+                    description: namespace is where the thing you're tracking is
+                    type: string
+                  resource:
+                    description: resource is the resource type of the thing you're
+                      tracking
+                    type: string
+            observedGeneration:
+              description: observedGeneration is the last generation change you've
+                dealt with
+              type: integer
+              format: int64
+            readyReplicas:
+              description: readyReplicas indicates how many replicas are ready and
+                at the desired state
+              type: integer
+              format: int32
+            version:
+              description: version is the level this availability applies to
+              type: string

manifests/01-cluster-role.yaml

@@ -112,3 +112,11 @@ rules:
   - subjectaccessreviews
   verbs:
   - create
+- apiGroups:
+  - operator.openshift.io
+  resources:
+  - cloudcredentials
+  verbs:
+  - get
+  - list
+  - watch

pkg/aws/actuator/actuator.go

@@ -283,20 +283,21 @@ func (a *AWSActuator) sync(ctx context.Context, cr *minterv1.CredentialsRequest)
 		return err
 	}
 
-	if cloudCredsSecret.Annotations[constants.AnnotationKey] == constants.InsufficientAnnotation {
+	switch cloudCredsSecret.Annotations[constants.AnnotationKey] {
+	case constants.InsufficientAnnotation:
 		msg := "cloud credentials insufficient to satisfy credentials request"
 		logger.Error(msg)
 		return &actuatoriface.ActuatorError{
 			ErrReason: minterv1.InsufficientCloudCredentials,
 			Message:   msg,
 		}
-	} else if cloudCredsSecret.Annotations[constants.AnnotationKey] == constants.PassthroughAnnotation {
+	case constants.PassthroughAnnotation:
 		logger.Debugf("provisioning with passthrough")
 		err := a.syncPassthrough(ctx, cr, cloudCredsSecret, logger)
 		if err != nil {
 			return err
 		}
-	} else if cloudCredsSecret.Annotations[constants.AnnotationKey] == constants.MintAnnotation {
+	case constants.MintAnnotation:
 		logger.Debugf("provisioning with cred minting")
 		err := a.syncMint(ctx, cr, logger)
 		if err != nil {
@@ -307,8 +308,13 @@ func (a *AWSActuator) sync(ctx context.Context, cr *minterv1.CredentialsRequest)
 				Message:   fmt.Sprintf("%v: %v", msg, err),
 			}
 		}
-	} else {
-		logger.Infof("unknown or missing %s annotation on admin credentials Secret, skipping reconcile", constants.AnnotationKey)
+	default:
+		msg := fmt.Sprintf("unexpected value or missing %s annotation on admin credentials Secret", constants.AnnotationKey)
+		logger.Info(msg)
+		return &actuatoriface.ActuatorError{
+			ErrReason: minterv1.CredentialsProvisionFailure,
+			Message:   msg,
+		}
 	}
 
 	return nil

pkg/azure/actuator.go

@@ -230,22 +230,21 @@ func (a *Actuator) sync(ctx context.Context, cr *minterv1.CredentialsRequest) er
 		return err
 	}
 
-	if cloudCredsSecret.Annotations[constants.AnnotationKey] == constants.InsufficientAnnotation {
+	switch cloudCredsSecret.Annotations[constants.AnnotationKey] {
+	case constants.InsufficientAnnotation:
 		msg := "cloud credentials insufficient to satisfy credentials request"
 		logger.Error(msg)
 		return &actuatoriface.ActuatorError{
 			ErrReason: minterv1.InsufficientCloudCredentials,
 			Message:   msg,
 		}
-	}
-
-	if cloudCredsSecret.Annotations[constants.AnnotationKey] == constants.PassthroughAnnotation {
+	case constants.PassthroughAnnotation:
 		logger.Debugf("provisioning with passthrough")
 		err := a.syncPassthrough(ctx, cr, cloudCredsSecret, logger)
 		if err != nil {
 			return err
 		}
-	} else if cloudCredsSecret.Annotations[constants.AnnotationKey] == constants.MintAnnotation {
+	case constants.MintAnnotation:
 		logger.Debugf("provisioning with cred minting")
 		err := a.syncMint(ctx, cr, cloudCredsSecret, infraName, infraResourceGroups, logger)
 		if err != nil {
@@ -256,6 +255,13 @@ func (a *Actuator) sync(ctx context.Context, cr *minterv1.CredentialsRequest) er
 				Message:   fmt.Sprintf("%v: %v", msg, err),
 			}
 		}
+	default:
+		msg := fmt.Sprintf("unexpected value or missing %s annotation on admin credentials Secret", constants.AnnotationKey)
+		logger.Info(msg)
+		return &actuatoriface.ActuatorError{
+			ErrReason: minterv1.CredentialsProvisionFailure,
+			Message:   msg,
+		}
 	}
 
 	return nil

pkg/cmd/operator/cmd.go

@@ -26,11 +26,9 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
-	"github.com/openshift/cloud-credential-operator/pkg/apis"
 	minterv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1"
 	controller "github.com/openshift/cloud-credential-operator/pkg/operator"
-
-	openshiftapiv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/cloud-credential-operator/pkg/util"
 
 	"k8s.io/apimachinery/pkg/util/wait"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@@ -84,15 +82,7 @@ func NewOperator() *cobra.Command {
 			log.Info("registering components")
 
 			// Setup Scheme for all resources
-			log.Info("setting up scheme")
-			if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
-				log.WithError(err).Fatal("unable to add APIs to scheme")
-			}
-
-			// Setup Openshift API scheme:
-			if err := openshiftapiv1.Install(mgr.GetScheme()); err != nil {
-				log.Fatal(err)
-			}
+			util.SetupScheme(mgr.GetScheme())
 
 			// Setup all Controllers
 			log.Info("setting up controller")

pkg/gcp/actuator/actuator.go

@@ -199,22 +199,21 @@ func (a *Actuator) sync(ctx context.Context, cr *minterv1.CredentialsRequest) er
 		return err
 	}
 
-	if cloudCredsSecret.Annotations[constants.AnnotationKey] == constants.InsufficientAnnotation {
+	switch cloudCredsSecret.Annotations[constants.AnnotationKey] {
+	case constants.InsufficientAnnotation:
 		msg := "cloud credentials insufficient to satisfy credentials request"
 		logger.Error(msg)
 		return &actuatoriface.ActuatorError{
 			ErrReason: minterv1.InsufficientCloudCredentials,
 			Message:   msg,
 		}
-	}
-
-	if cloudCredsSecret.Annotations[constants.AnnotationKey] == constants.PassthroughAnnotation {
+	case constants.PassthroughAnnotation:
 		logger.Debug("provisioning with passthrough")
 		err := a.syncPassthrough(ctx, cr, cloudCredsSecret, logger)
 		if err != nil {
 			return err
 		}
-	} else if cloudCredsSecret.Annotations[constants.AnnotationKey] == constants.MintAnnotation {
+	case constants.MintAnnotation:
 		logger.Debug("provisioning with cred minting")
 		err := a.syncMint(ctx, cr, infraName, logger)
 		if err != nil {
@@ -225,6 +224,13 @@ func (a *Actuator) sync(ctx context.Context, cr *minterv1.CredentialsRequest) er
 				Message:   fmt.Sprintf("%s: %v", msg, err),
 			}
 		}
+	default:
+		msg := fmt.Sprintf("unexpected value or missing %s annotation on admin credentials Secret", constants.AnnotationKey)
+		logger.Info(msg)
+		return &actuatoriface.ActuatorError{
+			ErrReason: minterv1.CredentialsProvisionFailure,
+			Message:   msg,
+		}
 	}
 
 	return nil

pkg/operator/constants/constants.go

@@ -32,6 +32,15 @@ const (
 	// running under (typically just haven't added support for the cloud/platform)
 	ModeUnknown CredentialsMode = "unknown"
 
+	// StatusModeMismatch is used to set a clusteroperator condition when
+	// the legacy configmap setting of disabled: "true" conflicts with the
+	// specified operator config mode.
+	StatusModeMismatch = "ModeMismatch"
+
+	// StatusModeInvalid is used to set a clusteroperator condition when
+	// the operator config CR specifies an invalide mode
+	StatusModeInvalid = "ModeInvalid"
+
 	// secret annoation vars
 
 	// AnnotationKey is the annotation the cloud credentials secret will be annotated with to indicate
@@ -67,6 +76,9 @@ const (
 	// CloudCredOperatorConfigMap is an optional ConfigMap that can be used to alter behavior of the operator.
 	CloudCredOperatorConfigMap = "cloud-credential-operator-config"
 
+	// CloudCredOperatorConfig is the name of the credentialsrequest.operator.openshift.io CR holding CCO's config
+	CloudCredOperatorConfig = "cluster"
+
 	// CloudCredSecretNamespace is where the cloud credentials can be found
 	CloudCredSecretNamespace = "kube-system"
 

pkg/operator/credentialsrequest/credentialsrequest_controller.go

@@ -34,6 +34,7 @@ import (
 	"github.com/openshift/cloud-credential-operator/pkg/util/clusteroperator"
 
 	configv1 "github.com/openshift/api/config/v1"
+	operatorv1 "github.com/openshift/api/operator/v1"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -372,12 +373,15 @@ func (r *ReconcileCredentialsRequest) Reconcile(request reconcile.Request) (reco
 		metrics.MetricControllerReconcileTime.WithLabelValues(controllerName).Observe(dur.Seconds())
 	}()
 
-	operatorIsDisabled, err := utils.IsOperatorDisabled(r.Client, logger)
+	mode, conflict, err := utils.GetOperatorConfiguration(r.Client, logger)
 	if err != nil {
 		logger.WithError(err).Error("error checking if operator is disabled")
 		return reconcile.Result{}, err
-	} else if operatorIsDisabled {
-		logger.Infof("operator disabled in %s ConfigMap", constants.CloudCredOperatorConfigMap)
+	} else if conflict {
+		logger.Error("configuration conflict betwen legacy configmap and operator config")
+		return reconcile.Result{}, fmt.Errorf("configuration conflict")
+	} else if mode == operatorv1.CloudCredentialsModeManual {
+		logger.Infof("operator set to disabled / manual mode")
 		return reconcile.Result{}, err
 	}