added x-metrics pod monitor

Makefile

@@ -14,6 +14,8 @@ PLATFORMS ?= linux_amd64
 UP_VERSION = v0.21.0
 UP_CHANNEL = stable
 UPTEST_VERSION = v0.6.1
+UXP_INSTALL_OPTS = "--unstable"
+UXP_VERSION = 1.14.0-up.1.rc.2"
 
 -include build/makelib/k8s_tools.mk
 # ====================================================================================
@@ -34,6 +36,7 @@ XPKGS = $(PROJECT_NAME)
 
 CROSSPLANE_NAMESPACE = upbound-system
 CROSSPLANE_ARGS = "--enable-usages"
+KIND_CLUSTER_NAME = "crossplane-local-dev"
 -include build/makelib/local.xpkg.mk
 -include build/makelib/controlplane.mk
 
@@ -69,11 +72,13 @@ build.init: $(UP)
 #   You can check the basic implementation here: https://github.com/upbound/uptest/blob/main/internal/templates/01-delete.yaml.tmpl.
 uptest: $(UPTEST) $(KUBECTL) $(KUTTL)
 	@$(INFO) running automated tests
-	@KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) $(UPTEST) e2e examples/folder-grafana.yaml,examples/dashboard-grafana-crossplane-health.yaml,examples/dashboard-grafana-crossplane-mr.yaml,examples/dashboard-grafana-crossplane-resources-ttr.yaml,examples/dashboard-grafana-crossplane-sli-metrics.yaml,examples/oss.yaml --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL)
+	@KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) $(UPTEST) e2e examples/dashboards/folder-grafana.yaml,examples/dashboards/dashboard-grafana-crossplane-health.yaml,examples/dashboards/dashboard-grafana-crossplane-mr.yaml,examples/dashboards/dashboard-grafana-crossplane-resources-ttr.yaml,examples/dashboards/dashboard-grafana-crossplane-sli-metrics.yaml,examples/dashboards/dashboard-grafana-crossplane-top-level.yaml,examples/dashboards/dashboard-grafana-crossplane-xmetrics.yaml,examples/oss.yaml --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL)
 	@$(OK) running automated tests
 
 # This target requires the following environment variables to be set:
 # - UPTEST_CLOUD_CREDENTIALS, cloud credentials for the provider being tested, e.g. export UPTEST_CLOUD_CREDENTIALS=$(cat ~/.aws/credentials)
 e2e: build controlplane.up local.xpkg.deploy.configuration.$(PROJECT_NAME) uptest
 
+cluster: build controlplane.up local.xpkg.deploy.configuration.$(PROJECT_NAME)
+
 .PHONY: uptest e2e

README.md

@@ -5,52 +5,67 @@ with open source software integrations such as Prometheus and Grafana.
 Observability is a measure of how well platform performance can be inferred
 from knowledge of its metrics, logs and traces outputs.
 
+## Note: Happily Operational Management Cluster
+This configuration provides useful insights into the
+health of Crossplane and its providers. To do so,
+it installs 3rd party open source software as part of
+its package. This software includes Prometheus and Grafana.
+Both are tunable to accomodate the arbitrary scale that
+may be needed based on the amount of resources for which
+metrics are collected and visualized. Take a look
+at the [Prometheus configuration options](https://prometheus.io/docs/prometheus/latest/configuration/configuration/)
+prior to applying this Crossplane observability
+configuration on mission critical clusters. Tune as appropriate
+for your use case to help keep your control planes happy
+and operational.
+
+## Note: Using Metrics With Confidence
+We are blessed with Crossplane and provider metrics endpoints
+that help us gain insight about their performance.
+Similar to early Kubernetes metrics APIs,
+Crossplane does not yet distinguish between alpha, beta and v1
+metrics endpoints. Metrics API evolution is expected over time.
+
 ## Purpose
 The goal for configuration-observability-oss is to complement
 other configurations such as configuration-caas. See the
 [Upbound Marketplace](https://marketplace.upbound.io/) for
 additional configurations.
 
 ## Usage
-Run `make e2e` directly to exercise end to end tests
-for the observability integrations. After running the
-tests, the kind cluster will remain but the tests will
-clean up the operator namespace and delete the pods in it
-at the conclusion of the tests by default.
+Run `make cluster` to spin up a cluster with a
+Crossplane control plane.
 
-Apply the resource claim as follows to re-create
-the namespace, Prometheus, Grafana and dependencies for further
-exploration.
+Apply the resource claim as follows to create
+the namespace, Prometheus, Grafana and dependencies for
+exploration. Note that the xmetrics configuration examples
+rely on the CRDs to be installed through the oss composition
+first.
 ```
 kubectl apply -f examples/oss.yaml
 ```
+Wait until xmetrics CRDs have been installed, then apply
+the xmetrics configuration to see metrics flowing.
+```
+kubectl apply -f examples/xmetrics.yaml
+```
 
-To load dashboards that are part of this configuration repository,
-please apply the following dashboard resource claims.
+The following command will apply all the dashboard
+resource claims located in the `dashboards` folder.
+Each claim installs a respective Grafana dashboard.
 ```
-kubectl apply -f examples/dashboards/folder-grafana.yaml
-kubectl apply -f examples/dashboards/dashboard-grafana-crossplane-health.yaml
-kubectl apply -f examples/dashboards/dashboard-grafana-crossplane-mr.yaml
-kubectl apply -f examples/dashboards/dashboard-grafana-crossplane-resources-ttr.yaml
-kubectl apply -f examples/dashboards/dashboard-grafana-crossplane-sli-metrics.yaml
+kubectl apply -f examples/dashboards
 ```
 
-Use the following to forward localhost:9090 to the Prometheus pod.
+To access the Prometheus dashboard, port-forward
+to the the Prometheus services on port 9090.
 ```
-PROMETHEUS_POD_NAME=$(k -n operators get pods|\
-    awk '{print $1}'|\
-    tail +2|\
-    grep prometheus-0)
-kubectl -n operators port-forward ${PROMETHEUS_POD_NAME} 9090
+kubectl -n operators port-forward svc/oss-kube-prometheus-stack-prometheus 9090:9090
 ```
 
-Use the following to forward localhost:3000 to the Grafana pod.
+To access the Grafana dashboard, port-forward to the Grafana service on port 80.
 ```
-GRAFANA_POD_NAME=$(k -n operators get pods|\
-    awk '{print $1}'|\
-    tail +2|\
-    grep grafana)
-kubectl -n operators port-forward ${GRAFANA_POD_NAME} 3000
+kubectl -n operators port-forward svc/oss-grafana 3000:80
 ```
 
 Log in to Grafana at http://localhost:3000 with the credentials
@@ -59,6 +74,15 @@ obtained from running the following.
 scripts/grafana-get-creds.sh
 ```
 
+#### Uptest
+Run `make e2e` to exercise end to end tests
+for the observability integrations. `make e2e`
+has no prerequisites, that is it does not require
+a previously created cluster. After running the
+tests, the kind cluster will remain but the tests will
+clean up the operator namespace and delete the pods in it
+at the conclusion of the tests by default.
+
 See example dashboards below.
 
 #### Crossplane MR Dashboard

apis/oss/composition.yaml

@@ -22,7 +22,7 @@ spec:
         apiVersion: kubernetes.crossplane.io/v1alpha1
         kind: Object
         metadata:
-          labels: 
+          labels:
             type: podMonitorCrossplaneRbacManager
         spec:
           forProvider:
@@ -31,10 +31,14 @@ spec:
               kind: PodMonitor
               metadata:
                 name: crossplane-rbac-manager
-                namespace: upbound-system
+                namespace: operators
                 labels:
                   app: crossplane-rbac-manager
               spec:
+                namespaceSelector:
+                  matchNames:
+                    - crossplane-system
+                    - upbound-system
                 selector:
                   matchLabels:
                     app: crossplane-rbac-manager
@@ -49,7 +53,7 @@ spec:
         apiVersion: kubernetes.crossplane.io/v1alpha1
         kind: Object
         metadata:
-          labels: 
+          labels:
             type: podMonitorCrossplane
         spec:
           forProvider:
@@ -58,10 +62,14 @@ spec:
               kind: PodMonitor
               metadata:
                 name: crossplane
-                namespace: upbound-system
+                namespace: operators
                 labels:
                   app: crossplane
               spec:
+                namespaceSelector:
+                  matchNames:
+                    - crossplane-system
+                    - upbound-system
                 selector:
                   matchLabels:
                     app: crossplane
@@ -76,7 +84,7 @@ spec:
         apiVersion: kubernetes.crossplane.io/v1alpha1
         kind: Object
         metadata:
-          labels: 
+          labels:
             type: podMonitorCrossplaneProviders
         spec:
           forProvider:
@@ -85,10 +93,14 @@ spec:
               kind: PodMonitor
               metadata:
                 name: crossplane-providers
-                namespace: upbound-system
+                namespace: operators
                 labels:
                   app: crossplane-providers
               spec:
+                namespaceSelector:
+                  matchNames:
+                    - crossplane-system
+                    - upbound-system
                 selector:
                   matchLabels:
                     app: crossplane-providers
@@ -98,6 +110,39 @@ spec:
         - type: PatchSet
           patchSetName: common
 
+    - name: podMonitorXmetrics
+      base:
+        apiVersion: kubernetes.crossplane.io/v1alpha1
+        kind: Object
+        metadata:
+          labels: 
+            type: podMonitorXmetrics
+        spec:
+          forProvider:
+            manifest:
+              apiVersion: monitoring.coreos.com/v1
+              kind: PodMonitor
+              metadata:
+                name: x-metrics
+                namespace: operators
+                labels:
+                  app: x-metrics
+              spec:
+                namespaceSelector:
+                  matchNames:
+                    - crossplane-system
+                    - upbound-system
+                    - x-metrics
+                selector:
+                  matchLabels:
+                    app: x-metrics
+                podMetricsEndpoints:
+                  - port: metrics
+                    path: "/x-metrics"
+      patches:
+        - type: PatchSet
+          patchSetName: common
+
     - name: releasePrometheus
       base:
         apiVersion: helm.crossplane.io/v1beta1
@@ -122,6 +167,12 @@ spec:
           patchSetName: common
         - fromFieldPath: spec.parameters.id
           toFieldPath: metadata.name
+        - fromFieldPath: metadata.labels
+          toFieldPath: metadata.labels
+        - fromFieldPath: metadata.annotations
+          toFieldPath: metadata.annotations
+        - fromFieldPath: spec.parameters.operators.prometheus.version
+          toFieldPath: spec.forProvider.chart.version
         - fromFieldPath: spec.parameters.id
           toFieldPath: metadata.annotations[crossplane.io/external-name]
           transforms:
@@ -131,12 +182,24 @@ spec:
                   - type: regexp
                     regexp: '.*'
                     result: oss
-        - fromFieldPath: metadata.labels
-          toFieldPath: metadata.labels
-        - fromFieldPath: metadata.annotations
-          toFieldPath: metadata.annotations
-        - fromFieldPath: spec.parameters.operators.prometheus.version
-          toFieldPath: spec.forProvider.chart.version
+
+    - name: usagePrometheusByPodMonitorXmetrics
+      base:
+        apiVersion: apiextensions.crossplane.io/v1alpha1
+        kind: Usage
+        spec:
+          of:
+            apiVersion: helm.crossplane.io/v1beta1
+            kind: Release
+            resourceSelector:
+              matchControllerRef: true
+          by:
+            apiVersion: kubernetes.crossplane.io/v1alpha1
+            kind: Object
+            resourceSelector:
+              matchControllerRef: true
+              matchLabels:
+                type: podMonitorXmetrics
 
     - name: usagePrometheusByPodMonitorCrossplaneRbacManager
       base:
@@ -191,3 +254,19 @@ spec:
               matchControllerRef: true
               matchLabels:
                 type: podMonitorCrossplaneProviders
+
+    - name: releaseXMetrics
+      base:
+        apiVersion: helm.crossplane.io/v1beta1
+        kind: Release
+        spec:
+          rollbackLimit: 3
+          forProvider:
+            namespace: x-metrics
+            chart:
+              name: x-metrics
+              repository: https://crossplane-contrib.github.io/x-metrics
+              version: "0.1.3"
+      patches:
+        - type: PatchSet
+          patchSetName: common