diff --git a/kubernetes/talos-flux/apps/observability/grafana/app/config/contactpoints.yaml b/kubernetes/talos-flux/apps/observability/grafana/app/config/contactpoints.yaml new file mode 100644 index 000000000..768cc3ae2 --- /dev/null +++ b/kubernetes/talos-flux/apps/observability/grafana/app/config/contactpoints.yaml @@ -0,0 +1,12 @@ +--- +# https://grafana.com/docs/grafana/latest/alerting/set-up/provision-alerting-resources/file-provisioning/#import-contact-points +apiVersion: 1 +contactPoints: + - orgId: 1 + name: alertmanager-notifications + receivers: + - uid: cp1 + type: prometheus-alertmanager + disableResolveMessage: false + settings: + url: http://prometheus-alertmanager.observability.svc.cluster.local:9093 diff --git a/kubernetes/talos-flux/apps/observability/grafana/app/config/datasources.yaml b/kubernetes/talos-flux/apps/observability/grafana/app/config/datasources.yaml new file mode 100644 index 000000000..8db6a3b53 --- /dev/null +++ b/kubernetes/talos-flux/apps/observability/grafana/app/config/datasources.yaml @@ -0,0 +1,28 @@ +--- +# https://grafana.com/docs/grafana/latest/datasources/ +apiVersion: 1 +# list of datasources that should be deleted from the database +deleteDatasources: + - name: Loki + orgId: 1 + - name: Prometheus + orgId: 1 + - name: GitHub + orgId: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus-prometheus:9090/ + isDefault: true + - name: Loki + type: loki + access: proxy + url: http://loki-gateway:80/ + - name: GitHub + type: grafana-github-datasource + jsonData: + owner: tyriis + repository: home-ops + secureJsonData: + accessToken: $GITHUB_PAT diff --git a/kubernetes/talos-flux/apps/observability/grafana/app/config/grafana.ini b/kubernetes/talos-flux/apps/observability/grafana/app/config/grafana.ini new file mode 100644 index 000000000..a783672eb --- /dev/null +++ b/kubernetes/talos-flux/apps/observability/grafana/app/config/grafana.ini @@ -0,0 +1,29 @@ +;https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/ +[analytics] +check_for_updates = false +[auth.google] +enabled = true +allow_sign_up = true +allowed_domains = techtales.io +auth_url = https://accounts.google.com/o/oauth2/auth +scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email +token_url = https://accounts.google.com/o/oauth2/token +[date_formats] +use_browser_locale = true +[explore] +enabled = true +[log] +mode = console +level = info +[panels] +disable_sanitize_html = true +[paths] +data = /var/lib/grafana/ +logs = /var/log/grafana +plugins = /var/lib/grafana/plugins +provisioning = /etc/grafana/provisioning +[server] +domain = grafana.techtales.io +root_url = https://grafana.techtales.io +[users] +auto_assign_org_role = Admin diff --git a/kubernetes/talos-flux/apps/observability/grafana/app/config/policies.yaml b/kubernetes/talos-flux/apps/observability/grafana/app/config/policies.yaml new file mode 100644 index 000000000..9361949cb --- /dev/null +++ b/kubernetes/talos-flux/apps/observability/grafana/app/config/policies.yaml @@ -0,0 +1,12 @@ +--- +# https://grafana.com/docs/grafana/latest/alerting/set-up/provision-alerting-resources/file-provisioning/#import-notification-policies +apiVersion: 1 +policies: + - orgId: 1 + receiver: alertmanager-notifications + group_by: + - grafana_folder + - alertname + group_wait: 30s + group_interval: 5m + repeat_interval: 12h diff --git a/kubernetes/talos-flux/apps/observability/grafana/app/grafana-admin.sops.yaml b/kubernetes/talos-flux/apps/observability/grafana/app/grafana-admin.sops.yaml new file mode 100644 index 000000000..d9a65e44d --- /dev/null +++ b/kubernetes/talos-flux/apps/observability/grafana/app/grafana-admin.sops.yaml @@ -0,0 +1,28 @@ +# yamllint disable +apiVersion: v1 +kind: Secret +metadata: + name: grafana-admin +stringData: + USERNAME: ENC[AES256_GCM,data:n57P/S4=,iv:7qajI1QASF15zpMqcVjal8XvBEG+BxezxegKPrJePdg=,tag:tbf5BF3JKgFkeFI+4FFJ5g==,type:str] + PASSWORD: ENC[AES256_GCM,data:J4Zg6RQ8K2ECZRg2/1jbjHS2u+T7j7U1f2OZOAUuVbFHt7yjpNFEN8tiUts=,iv:s2ifsOTwHK/h4CMBfpDx75ibAuzeo6+tNJshR7xfgCs=,tag:dzsXSSgspFSTdLFhOk2TFg==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age16zqeqx5y6ay3flwz0d06rn83yjv9ckys3j8tpkysf9v6295fhc6sf4r0uj + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIZzFkbDhYbElUcTJQZ1ls + T25TeWN3M3NXRzAvMXJjQXVsa0VGSk1tYjNrClJrb0JQbjk2NnhaUjNtbVN1U1JU + dVpXZW9OaUh0YU1GQmhiOHIyNnBYNDAKLS0tIHdZdk4rUVdIYzUyMURzc2FsV3hL + TTcwSDlkQ3VPM1NTWFdoTzZ5MVBEeDAKXeIe9FM/ZenGa8kVJjMIC9hcAwktLR/U + T5O1xTcVAhgBUDYbKdrexWuFIAsqhXVMAh0xhQEs3m9gdygDPAL6Mw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-04-03T20:17:49Z" + mac: ENC[AES256_GCM,data:+XnUw8M+hyNs8DHsR2juJKhfLV+mNGbeBlJ07j8RmQVYAuDrpVlneRWAtx6yxrC44z4C5+Jao61MkXqz+NClf2UX+dEnRIoYr15O9LPk9pBTcpEU0crb7VcJVSGxKQf8SulurCTdErGj6umdJeXyn43xebkE0QRV2F/46VavGT8=,iv:v9lnYP96v12HvaCucAXcsIerIyFVqwhUiYZVELP8hlQ=,tag:zTkjsfQrJC1v8m1DgalnMQ==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.8.1 diff --git a/kubernetes/talos-flux/apps/observability/grafana/app/grafana-env.sops.yaml b/kubernetes/talos-flux/apps/observability/grafana/app/grafana-env.sops.yaml index fbe05e11f..29a0f10db 100644 --- a/kubernetes/talos-flux/apps/observability/grafana/app/grafana-env.sops.yaml +++ b/kubernetes/talos-flux/apps/observability/grafana/app/grafana-env.sops.yaml @@ -1,14 +1,12 @@ # yamllint disable apiVersion: v1 kind: Secret -type: Opaque metadata: name: grafana-env stringData: - GF_AUTH_GOOGLE_ENABLED: ENC[AES256_GCM,data:Vf684g==,iv:lb3T33RlQFvsJUwhQsJc4fySF+ricMiO1LgCZ0hs3ro=,tag:Pjr0vd0Eallec2eorwU51g==,type:str] GF_AUTH_GOOGLE_CLIENT_ID: ENC[AES256_GCM,data:EaJ5MS9jmXKZ1gbBwJU8xL09E95Lzog/ncqnq8+TYC/e27JpcE4LdQC2ZwjIWPcTDuUM5m7NpjMzUzIvISXoOJdDSxe/waRr,iv:Oub/mmnsIBGMB4GMMvL2eUK7Uz4XgHdIIdPiAeFuiXY=,tag:gKByk2mmlmJc5+yJmkXcAQ==,type:str] GF_AUTH_GOOGLE_CLIENT_SECRET: ENC[AES256_GCM,data:YLlBwyOWV0zksdQoc2Rp0GrNxXCBtsxbvhPSSBHy/By4djo=,iv:sHFcGqQj6Ak0GvA+I7guGLPY40bHO3re7XPV2xsToPA=,tag:1Ns/P/xXdUpNS6iYjv4Rxw==,type:str] - TEST_URL: ENC[AES256_GCM,data:3a+t4QGIjlOyTZ9zuGP5k+t84yFAAh5NHqVmZ8j0AIRfvvNLMLceqHpNkLAFfXEU+eZluMBIQ4cRlmAcoI8VhSvnFw==,iv:fL2xK2uaIMQmhWospg9ea0y4bZT8NroW3A+utaluGi4=,tag:/eZd9XOBIREKM8BVTbnfCg==,type:str] + GITHUB_PAT: ENC[AES256_GCM,data:XkHDWOs2UQP3yT5NwiilDrDTOK7NDNAKWBqF4wuY31yA2TW70Vqfjg==,iv:SgHDC9NTU5bVpJNIX9DxwKo6WgNLs6T1Kjsi7T0HpcQ=,tag:E967XwHdFfwU0ERYYMIetQ==,type:str] sops: kms: [] gcp_kms: [] @@ -24,8 +22,8 @@ sops: eUQwcWJxQWZIUkRsb291SHpGSDhqT1EKgTQ1qSb4D0VNoXTiTkz9sHrHFPNHcPCW IQ8/QYEA6iWVt+v8s+ATb2OaLZhha5FgwCOGVyIv6GJLP1kBlz8RwQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-04-03T09:52:00Z" - mac: ENC[AES256_GCM,data:DUi4H37y8DF36dwKIQomOdytiNH2Yic3o0D9I30ChOo283G3p7Lr3sZalEAp6tJ75jAanL/cp1oUPZCGUwkqtrmHJK9Z1QjAHUmDOHZ27WKYpT/FNOxeXummx0w5IyXZeF1YMpzO9ddQ5LaBSyXt1S6fpqJLEeMeeZLy/NDst1o=,iv:K93XNG1Au/M81KhIw+xU4SHuqJ/JSGSteyT65t6FNxA=,tag:1l9fxpMJMjEKjnCRsEiJNg==,type:str] + lastmodified: "2024-04-03T20:30:42Z" + mac: ENC[AES256_GCM,data:7isbGbxz0t0NxjhGER9Mq/TWUK20My9Z4b0yYcD1SWJ+toAGH36NmxC2i36bChU16d0ZPanlW6wmUL7O+1Ihvp/5tQyFp7ZW+6cPIAhWekdFYOrFk68pfB+IlD/YYOt6HO6bmcfXispqyqDZkr9k8UgirsEt7sA36NMs/nByWXY=,iv:y7jqoj+G/HzAKXmY6yI82AEfwo/ueGw1p8gbS9/GkTA=,tag:QagM09nKrdiO/mCBFDOSzw==,type:str] pgp: [] encrypted_regex: ^(data|stringData)$ version: 3.8.1 diff --git a/kubernetes/talos-flux/apps/observability/grafana/app/helm-release.yaml b/kubernetes/talos-flux/apps/observability/grafana/app/helm-release.yaml index b29d57d20..c6048ea5d 100644 --- a/kubernetes/talos-flux/apps/observability/grafana/app/helm-release.yaml +++ b/kubernetes/talos-flux/apps/observability/grafana/app/helm-release.yaml @@ -27,115 +27,24 @@ spec: values: replicas: 1 - alerting: - contactpoints.yaml: - secret: - apiVersion: 1 - contactPoints: - - orgId: 1 - name: alertmanager-notifications - receivers: - - uid: test - type: prometheus-alertmanager - settings: - url: $TEST_URL - send_resolved: true - env: TZ: ${SETTING_TZ} - GF_EXPLORE_ENABLED: "true" - GF_PANELS_DISABLE_SANITIZE_HTML: "true" - GF_DATE_FORMATS_USE_BROWSER_LOCALE: "true" - VAR_BLOCKY_URL: http://blocky.networking.svc.cluster.local:4000 envFromSecrets: - name: grafana-env - adminPassword: "${SECRET_GRAFANA_PASSWORD}" - grafana.ini: - server: - root_url: "https://grafana.${SECRET_DOMAIN}" - users: - auto_assign_org_role: "Admin" - auth.google: - enabled: true - scopes: https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email - auth_url: https://accounts.google.com/o/oauth2/auth - token_url: https://accounts.google.com/o/oauth2/token - allowed_domains: "${SECRET_DOMAIN}" - allow_sign_up: true - - dashboardProviders: - dashboardproviders.yaml: - apiVersion: 1 - providers: - - name: default - orgId: 1 - folder: "" - type: file - disableDeletion: false - # allowUiUpdates: false - editable: true - options: - path: /var/lib/grafana/dashboards/default - - name: flux - orgId: 1 - folder: Flux - type: file - disableDeletion: false - editable: true - # allowUiUpdates: true - options: - path: /var/lib/grafana/dashboards/flux - datasources: - datasources.yaml: - apiVersion: 1 - # list of datasources that should be deleted from the database - deleteDatasources: - - name: Loki - orgId: 1 - - name: Prometheus - orgId: 1 - - name: GitHub - orgId: 1 - datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus-prometheus:9090/ - isDefault: true - - name: Loki - type: loki - access: proxy - url: http://loki-gateway - - name: GitHub - type: grafana-github-datasource - jsonData: - owner: "tyriis" - repository: "home-ops" - secureJsonData: - accessToken: "${SECRET_GH_PAT}" + admin: + existingSecret: grafana-admin + userKey: USERNAME + passwordKey: PASSWORD dashboards: - # default: - # flux: - # flux-cluster: - # url: https://raw.githubusercontent.com/fluxcd/flux2/main/manifests/monitoring/monitoring-config/dashboards/cluster.json - # datasource: Prometheus - # flux-control-plane: - # url: https://raw.githubusercontent.com/fluxcd/flux2/main/manifests/monitoring/monitoring-config/dashboards/control-plane.json - # datasource: Prometheus default: # Ref: https://grafana.com/grafana/dashboards/11074 "Node Exporter for Prometheus Dashboard": gnetId: 11074 revision: 9 datasource: Prometheus - # Ref: https://grafana.com/grafana/dashboards/13768 - blocky: - gnetId: 13768 - revision: 3 - datasource: Prometheus sidecar: dashboards: @@ -154,8 +63,6 @@ spec: - grafana-github-datasource serviceMonitor: enabled: true - rbac: - pspEnabled: false ingress: enabled: true @@ -167,12 +74,12 @@ spec: traefik.ingress.kubernetes.io/router.entrypoints: websecure external-dns.alpha.kubernetes.io/target: "${SECRET_CLOUDFLARE_TUNNEL_ID}.cfargotunnel.com" hosts: - - "grafana.${SECRET_DOMAIN}" + - &host grafana.techtales.io path: / tls: - secretName: grafana-cert hosts: - - "grafana.${SECRET_DOMAIN}" + - *host serviceAccount: create: true @@ -180,3 +87,26 @@ spec: persistence: enabled: false + + createConfigmap: true + extraConfigmapMounts: + - name: grafana-contactpoints + mountPath: /etc/grafana/alerting/ + subPath: contactpoints.yaml + configMap: grafana-contactpoints + readOnly: true + - name: grafana-datasources + mountPath: /etc/grafana/datasources/ + subPath: datasources.yaml + configMap: grafana-datasources + readOnly: true + - name: grafana-ini + mountPath: /etc/grafana/ + subPath: grafana.ini + configMap: grafana-ini + readOnly: true + - name: grafana-policies + mountPath: /etc/grafana/alerting/ + subPath: policies.yaml + configMap: grafana-policies + readOnly: true diff --git a/kubernetes/talos-flux/apps/observability/grafana/app/kustomization.yaml b/kubernetes/talos-flux/apps/observability/grafana/app/kustomization.yaml index 762b04032..a03967a8f 100644 --- a/kubernetes/talos-flux/apps/observability/grafana/app/kustomization.yaml +++ b/kubernetes/talos-flux/apps/observability/grafana/app/kustomization.yaml @@ -4,5 +4,26 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: observability resources: + - grafana-admin.sops.yaml - grafana-env.sops.yaml - helm-release.yaml +configMapGenerator: + - name: grafana-contactpoints + files: + - contactpoints.yaml=config/contactpoints.yaml + - name: grafana-datasources + files: + - datasources.yaml=config/datasources.yaml + - name: grafana-ini + files: + - grafana.ini=config/grafana.ini + - name: grafana-policies + files: + - policies.yaml=config/policies.yaml +commonLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +generatorOptions: + disableNameSuffixHash: true + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled