Setup grafana, prometheus & thanos for monitoring

DiamondLightSource · Sep 8, 2023 · 597184c · 597184c
1 parent 4a9eb19
commit 597184c
Show file tree

Hide file tree

Showing 4 changed files with 321 additions and 17 deletions.
diff --git a/charts/xchemlab/Chart.lock b/charts/xchemlab/Chart.lock
@@ -1,18 +1,27 @@
 dependencies:
-- name: chimp-chomp
-  repository: ""
-  version: 0.0.1
-- name: pin-packing
-  repository: ""
-  version: 0.0.1
-- name: keda
-  repository: https://kedacore.github.io/charts
-  version: 2.10.2
-- name: opa-kube-mgmt
-  repository: https://open-policy-agent.github.io/kube-mgmt/charts
-  version: 8.3.0
-- name: rabbitmq
-  repository: oci://docker.io/bitnamicharts
-  version: 12.0.7
-digest: sha256:a2fb48c058cd1b0d4b43b0d4d9c27174a7841724e9174fff63de9e5d1d865b8f
-generated: "2023-08-09T14:47:57.727687985+01:00"
+  - name: chimp-chomp
+    repository: ""
+    version: 0.0.1
+  - name: pin-packing
+    repository: ""
+    version: 0.0.1
+  - name: grafana
+    repository: https://grafana.github.io/helm-charts
+    version: 6.58.8
+  - name: keda
+    repository: https://kedacore.github.io/charts
+    version: 2.10.2
+  - name: opa-kube-mgmt
+    repository: https://open-policy-agent.github.io/kube-mgmt/charts
+    version: 8.3.0
+  - name: prometheus
+    repository: https://prometheus-community.github.io/helm-charts
+    version: 23.3.0
+  - name: thanos
+    repository: oci://docker.io/bitnamicharts
+    version: 12.11.0
+  - name: rabbitmq
+    repository: oci://docker.io/bitnamicharts
+    version: 12.0.7
+digest: sha256:4ba713c858795f9002912462f8ea5de75dc5c81e75f1e0c1dc2f53ce1172fd78
+generated: "2023-08-11T12:16:27.672917968+01:00"
diff --git a/charts/xchemlab/Chart.yaml b/charts/xchemlab/Chart.yaml
@@ -21,6 +21,10 @@ dependencies:
   - name: pin-packing
     version: 0.0.1
     condition: pin-packing.enabled
+  - name: grafana
+    repository: https://grafana.github.io/helm-charts
+    version: 6.58.8
+    condition: grafana.enabled
   - name: keda
     repository: https://kedacore.github.io/charts
     version: 2.10.2
@@ -29,6 +33,14 @@ dependencies:
     repository: https://open-policy-agent.github.io/kube-mgmt/charts
     version: 8.3.0
     condition: opa-kube-mgmt.enabled
+  - name: prometheus
+    repository: https://prometheus-community.github.io/helm-charts
+    version: 23.3.0
+    condition: prometheus.enabled
+  - name: thanos
+    repository: oci://docker.io/bitnamicharts
+    version: 12.11.0
+    condition: thanos.enabled
   - name: rabbitmq
     repository: oci://docker.io/bitnamicharts
     version: 12.0.7

diff --git a/charts/xchemlab/templates/prometheus-s3-secret.yaml b/charts/xchemlab/templates/prometheus-s3-secret.yaml
@@ -0,0 +1,16 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: prometheus-s3-secret
+  namespace: xchemlab
+spec:
+  encryptedData:
+    objstore.yml: AgCM+HBIHLC4LeJ386kRDa26mT3Gi1/Z22FqI/hTcrLIW/a42SMg7K+q0G480EnUavbVjKPDHvLoMf3H4dm887ycznolbk0dmAjKhr1KKFFBcsWLD9xlDuX2J0MY0aDFLiYdMyAsXrK7SOPyZuedopcBgxIc38OZWQPXQChsujEBN9fsu6spx2feyZ/LK9Eq8haSjWxJHITvwugGDVpSy1PBtn9mZJXvkf+eUpqEvEvbBezWABx8xPv7xmxnThFw3gdrJGsaWFCZhUI3Pgwj7f/PYJbJTzVXZWp+Sc9+mGH8KnoVLruGSE9LIvfZkirJa3+oLVMeyK3Ia2iFi9itAezX+C1gh7k9FyMdMtjmWTOP9nOLXJD2nP9ICb6gqooKRdo8fJssw5+C75bSHe91530YTqPZ6DyQj2+rijR75ozOK1BfmT41LnF8nXH1iEWwv/Z1lTYnm/7+ALtaVawzj21WBzdO9SOsSlo4vAYtrE6vm2daw5Nzxw1qW+hWAk6OlI9pZSfnMNZ/Bn7OiTjD4CbQoKhIU3jE7c4jFce0krbitSt8vBKtaUJmtcE/fIjGGEvu3i+RaMpq5kKFESB1GmqA9GgIhvyHRiH08OXzghQRlPmBkot4KgTQ42LssxWh5HyP60cs8PnIqEY8gBCW/4F5Q3HpIa9xYF1P0OxC8TGa3p6mSZW5Mvh2e8yoTff5ZhspEBDxzlTRIJrzKeYNmcQKAxq9e1B/HdfpdZEGC6YYRIbxDFw4o+I7eVD6cJAzmkCRJPeugl2ou4zauQP9kCxeRiBWb7bs/Agc8ywt1162UdoPtVeUX5/uNtKlX12SuCt57CPP0Z/5ffII4XnVc5L9fQA+72T8XiCWLB4CAMMdiQPNKte18hQ/5VgFushLgD2W8n2b/lk1xq8F56nde8DFbbubJV0kxJKUxCPXT2+t+aJdwjMI3GCaQaszi+mnZaGSxIJmsik4Df3DsB+u3pObKRH0u6r+JwyoCGyAcnxWK1LAOoh3GQ0XQzvbXTmUCe9Kbru3Lk5zmb1H3bNb9vGisJwdZH4E145KR3iJcLKv0fcTZVK9OFgBz1XWwfeUQ+zOMdjOYLEbKKMmfwOCp8hCQpnYZ9WwvYeR8zrq25H3jzLLGcbL8Lz1gB2PCRAYka8EUP0yHyDrQm08ZZ+qkJ3Ea3Tz+WmWzXVrJDUMeefOE4QL
+  template:
+    data: null
+    metadata:
+      creationTimestamp: null
+      name: prometheus-s3-secret
+      namespace: xchemlab
+
diff --git a/charts/xchemlab/values.yaml b/charts/xchemlab/values.yaml
@@ -30,20 +30,287 @@ pin-packing:
   opa:
     serviceSuffix: opa-kube-mgmt
 
+grafana:
+  enabled: true
+
+  rbac:
+    create: false
+
+  serviceAccount:
+    create: false
+    name: default-full-access-mounted
+
+  ingress:
+    enabled: true
+    hosts:
+      - xchemlab-grafana.diamond.ac.uk
+    path: /
+    tls:
+      - hosts:
+          - xchemlab.grafana.diamond.ac.uk
+
+  datasources:
+    datasources.yaml:
+      apiVersion: 1
+      datasources:
+        - name: Prometheus
+          type: prometheus
+          url: https://xchemlab-thanos.diamond.ac.uk
+          isDefault: true
+
 keda:
   enabled: false
 
 opa-kube-mgmt:
   enabled: true
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/path: "/metrics"
+    prometheus.io/port: "8182"
   mgmt:
     enabled: false
+  prometheus:
+    enabled: true
   serviceAccount:
     create: false
   rbac:
     create: false
 
+prometheus:
+  enabled: true
+
+  kube-state-metrics:
+    enabled: false
+
+  prometheus-node-exporter:
+    enabled: false
+
+  prometheus-pushgateway:
+    enabled: false
+
+  rbac:
+    create: false
+
+  alertmanager:
+    serviceAccount:
+      create: false
+      name: default-full-access-mounted
+    ingress:
+      enabled: true
+      hosts:
+        - host: xchemlab-alertmanager.diamond.ac.uk
+          paths:
+            - path: /
+              pathType: ImplementationSpecific
+      tls:
+        - hosts:
+            - xchemlab-alertmanager.diamond.ac.uk
+    extraArgs:
+      web.external-url: https://xchemlab-alertmanager.diamond.ac.uk
+    configmapReload:
+      enabled: true
+
+  server:
+    remoteWrite:
+      - url: http://{{ .Release.name }}-thanos-receive.xchemlab.svc.cluster.local:19291/api/v1/receive
+    affinity:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                    - prometheus
+            topologyKey: kubernetes.io/hostname
+    persistentVolume:
+      enabled: false
+    retention: 7d
+    statefulSet:
+      enabled: true
+    alertmanagers:
+      - kubernetes_sd_configs:
+          - role: pod
+            namespaces:
+              own_namespace: true
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_namespace]
+            regex: <namespace>
+            action: keep
+          - source_labels:
+              [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+            regex: <release-name>
+            action: keep
+          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+            regex: alertmanager
+            action: keep
+          - source_labels: [__meta_kubernetes_pod_container_port_number]
+            regex: "9093"
+            action: keep
+
+  serviceAccounts:
+    prometheus-node-exporter:
+      create: false
+      name: default-full-access-mounted
+    pushgateway:
+      create: false
+      name: default-full-access-mounted
+    server:
+      create: false
+      name: default-full-access-mounted
+
+  serverFiles:
+    prometheus.yml:
+      scrape_configs:
+        - job_name: prometheus
+          static_configs:
+            - targets:
+                - localhost:9090
+        - job_name: "kubernetes-service-endpoints"
+          kubernetes_sd_configs:
+            - role: endpoints
+              namespaces:
+                own_namespace: true
+          relabel_configs:
+            - source_labels:
+                [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+              action: keep
+              regex: true
+            - source_labels:
+                [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+              action: replace
+              target_label: __scheme__
+              regex: (https?)
+            - source_labels:
+                [__meta_kubernetes_service_annotation_prometheus_io_path]
+              action: replace
+              target_label: __metrics_path__
+              regex: (.+)
+            - source_labels:
+                [
+                  __address__,
+                  __meta_kubernetes_service_annotation_prometheus_io_port,
+                ]
+              action: replace
+              target_label: __address__
+              regex: (.+?)(?::\d+)?;(\d+)
+              replacement: $1:$2
+            - action: labelmap
+              regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
+              replacement: __param_$1
+            - action: labelmap
+              regex: __meta_kubernetes_service_label_(.+)
+            - source_labels: [__meta_kubernetes_namespace]
+              action: replace
+              target_label: namespace
+            - source_labels: [__meta_kubernetes_service_name]
+              action: replace
+              target_label: service
+            - source_labels: [__meta_kubernetes_pod_node_name]
+              action: replace
+              target_label: node
+        - job_name: "kubernetes-pods"
+          kubernetes_sd_configs:
+            - role: pod
+              namespaces:
+                own_namespace: true
+          relabel_configs:
+            - source_labels:
+                [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+              action: keep
+              regex: true
+            - source_labels:
+                [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+              action: keep
+              regex: true
+            - source_labels:
+                [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+              action: replace
+              regex: (https?)
+              target_label: __scheme__
+            - source_labels:
+                [__meta_kubernetes_pod_annotation_prometheus_io_path]
+              action: replace
+              target_label: __metrics_path__
+              regex: (.+)
+            - source_labels:
+                [
+                  __address__,
+                  __meta_kubernetes_pod_annotation_prometheus_io_port,
+                ]
+              action: replace
+              regex: ([^:]+)(?::\d+)?;(\d+)
+              replacement: $1:$2
+              target_label: __address__
+            - action: labelmap
+              regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
+              replacement: __param_$1
+            - action: labelmap
+              regex: __meta_kubernetes_pod_label_(.+)
+            - source_labels: [__meta_kubernetes_namespace]
+              action: replace
+              target_label: namespace
+            - source_labels: [__meta_kubernetes_pod_name]
+              action: replace
+              target_label: pod
+            - source_labels: [__meta_kubernetes_pod_phase]
+              regex: Pending|Succeeded|Failed|Completed
+              action: drop
+
+thanos:
+  enabled: true
+
+  existingServiceAccount: default-full-access-mounted
+
+  existingObjstoreSecret: prometheus-s3-secret
+
+  query:
+    enabled: true
+    ingress:
+      enabled: false
+
+  queryFrontend:
+    ingress:
+      enabled: true
+      hostname: xchemlab-thanos.diamond.ac.uk
+      path: /
+      tls: true
+
+  storegateway:
+    enabled: true
+    persistence:
+      enabled: false
+
+  receive:
+    enabled: true
+    tsdbRetention: 1d
+    persistence:
+      enabled: false
+    service:
+      type: LoadBalancer
+    ingress:
+      enabled: false
+
+  compactor:
+    enabled: true
+    retentionResolutionRaw: 90d
+    retentionResolution5m: 90d
+    retentionResolution1h: 90d
+    consistencyDelay: 30m
+    persistence:
+      enabled: false
+
 rabbitmq:
   enabled: true
+  metrics:
+    enabled: true
+    podAnnotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/path: /metrics
+      prometheus.io/port: "9419"
   persistence:
     enabled: false
   serviceBindings: