feat: etcd backup (#475)

kubernetes/etcd-backup/app/config.yaml

@@ -0,0 +1,25 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/refs/heads/master/master/configmap.json
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: etcd-backup
+  name: rustic-config
+data:
+  rustic.toml: | #toml
+    [repository]
+    repository = "opendal:s3"
+    # password = RUSTIC_PASSWORD
+
+    [repository.options]
+    endpoint = "https://s3.us-east-005.backblazeb2.com"
+    # access_key_id = OPENDAL_ACCESS_KEY_ID
+    # secret_access_key = OPENDAL_SECRET_ACCESS_KEY
+    bucket = "timtor-homelab-etcd-backup"
+    root = "/"
+    region = "us-east-005"
+
+    [forget]
+    keep-hourly = -1
+    keep-daily = 5
+    keep-weekly = 4

kubernetes/etcd-backup/app/netpol.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  namespace: etcd-backup
+  name: etcd-backup-app-policy
+specs:
+  - endpointSelector:
+      matchLabels:
+        app.kubernetes.io/name: etcd-backup
+    egress:
+      # allow dns connection
+      - toEndpoints:
+          - matchLabels:
+              k8s:io.kubernetes.pod.namespace: kube-system
+              k8s-app: kube-dns
+        toPorts:
+          - ports:
+              - protocol: ANY
+                port: "53"
+            rules:
+              dns:
+                - matchName: talos.default.svc.cluster.local.
+                - matchName: &s3 s3.us-east-005.backblazeb2.com
+      # allow connection to master node
+      - toEntities:
+          - remote-node
+      # allow connection to b2
+      - toFQDNs:
+          - matchName: *s3
+        toPorts:
+          - ports:
+              - protocol: TCP
+                port: "443"

kubernetes/etcd-backup/app/release.yaml

@@ -0,0 +1,142 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  namespace: etcd-backup
+  name: etcd-backup
+spec:
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+      chart: app-template
+      version: 3.1.0
+  interval: 1h
+  maxHistory: 1
+  timeout: 1m0s
+  values:
+    controllers:
+      main:
+        type: cronjob
+        cronjob:
+          schedule: "0 * * * *"
+          concurrencyPolicy: Forbid
+          backoffLimit: 3
+          successfulJobsHistory: 3
+          failedJobsHistory: 1
+        annotations:
+          secret.reloader.stakater.com/reload: &s rustic-secret
+        pod:
+          hostname: etcd-backup
+          restartPolicy: OnFailure
+          automountServiceAccountToken: false
+          securityContext:
+            fsGroup: 65534
+          dnsConfig:
+            options:
+              - name: ndots
+                value: "1"
+        initContainers:
+          talos:
+            image:
+              repository: ghcr.io/siderolabs/talosctl
+              tag: v1.7.1
+            args:
+              - "-n"
+              - "192.168.253.1"
+              - "etcd"
+              - "snapshot"
+              - "/data/etcd.snapshot"
+            securityContext: &sc
+              runAsNonRoot: true
+              runAsUser: 65534
+              runAsGroup: 65534
+              allowPrivilegeEscalation: false
+              readOnlyRootFilesystem: true
+              capabilities:
+                drop: ["ALL"]
+              seccompProfile:
+                type: RuntimeDefault
+        containers:
+          rustic-backup:
+            image:
+              repository: ghcr.io/rustic-rs/rustic
+              tag: v0.9.4
+            env: &env
+              RUSTIC_PASSWORD:
+                valueFrom:
+                  secretKeyRef:
+                    name: *s
+                    key: RUSTIC_PASSWORD
+              OPENDAL_ACCESS_KEY_ID:
+                valueFrom:
+                  secretKeyRef:
+                    name: *s
+                    key: OPENDAL_ACCESS_KEY_ID
+              OPENDAL_SECRET_ACCESS_KEY:
+                valueFrom:
+                  secretKeyRef:
+                    name: *s
+                    key: OPENDAL_SECRET_ACCESS_KEY
+            args:
+              - backup
+              - --init
+              - /data
+            resources: {}
+            securityContext: *sc
+          rustic-forget:
+            image:
+              repository: ghcr.io/rustic-rs/rustic
+              tag: v0.9.4
+            env: *env
+            args:
+              - forget
+            resources: {}
+            securityContext: *sc
+
+    serviceAccount:
+      create: true
+      annotations:
+        eks.amazonaws.com/role-arn: arn:aws:iam::262264826613:role/amethyst-etcd-backup
+        eks.amazonaws.com/audience: sts.amazonaws.com
+
+    persistence:
+      talos-secret:
+        type: secret
+        name: etcd-backup-talos-sa
+        advancedMounts:
+          main:
+            talos:
+              - path: /var/run/secrets/talos.dev
+                readOnly: true
+      data:
+        type: emptyDir
+        sizeLimit: 500Mi
+        advancedMounts:
+          main:
+            talos:
+              - path: /data
+            rustic-backup:
+              - path: /data
+      rustic-config:
+        type: configMap
+        name: rustic-config
+        advancedMounts:
+          main:
+            rustic-backup:
+              - path: /rustic.toml
+                subPath: rustic.toml
+                readOnly: true
+            rustic-forget:
+              - path: /rustic.toml
+                subPath: rustic.toml
+                readOnly: true
+      rustic-secret:
+        type: custom
+        volumeSpec:
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: *s

kubernetes/etcd-backup/app/repo.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  namespace: etcd-backup
+  name: bjw-s
+spec:
+  url: https://bjw-s.github.io/helm-charts
+  interval: 24h

kubernetes/etcd-backup/app/secret.yaml

@@ -0,0 +1,30 @@
+---
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  namespace: etcd-backup
+  name: &name rustic-secret
+spec:
+  provider: aws
+  parameters:
+    region: us-west-2
+    objects: |
+      - objectType: ssmparameter
+        objectName: /kubernetes/etcd-backup
+        jmesPath:
+          - path: RUSTIC_PASSWORD
+            objectAlias: RUSTIC_PASSWORD
+          - path: OPENDAL_ACCESS_KEY_ID
+            objectAlias: OPENDAL_ACCESS_KEY_ID
+          - path: OPENDAL_SECRET_ACCESS_KEY
+            objectAlias: OPENDAL_SECRET_ACCESS_KEY
+  secretObjects:
+    - secretName: *name
+      type: Opaque
+      data:
+        - key: RUSTIC_PASSWORD
+          objectName: RUSTIC_PASSWORD
+        - key: OPENDAL_ACCESS_KEY_ID
+          objectName: OPENDAL_ACCESS_KEY_ID
+        - key: OPENDAL_SECRET_ACCESS_KEY
+          objectName: OPENDAL_SECRET_ACCESS_KEY

kubernetes/etcd-backup/app/talos-sa.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: talos.dev/v1alpha1
+kind: ServiceAccount
+metadata:
+  namespace: etcd-backup
+  name: etcd-backup-talos-sa
+spec:
+  roles:
+    - os:etcd:backup

kubernetes/etcd-backup/base/netpol.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  namespace: etcd-backup
+  name: etcd-backup-default-policy
+specs:
+  - endpointSelector:
+      matchLabels: {}
+    ingress:
+      - {}
+    egress:
+      - {}

kubernetes/etcd-backup/base/ns.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: etcd-backup
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted

kubernetes/etcd-backup/kustomization.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - base/ns.yaml
+  - base/netpol.yaml
+  - app/repo.yaml
+  - app/talos-sa.yaml
+  - app/config.yaml
+  - app/secret.yaml
+  - app/netpol.yaml
+  - app/release.yaml

talos/controlplane.yaml

@@ -37,6 +37,12 @@ machine:
   # -- Talos features
   features:
     rbac: true
+    kubernetesTalosAPIAccess:
+      enabled: true
+      allowedRoles:
+        - os:etcd:backup
+      allowedKubernetesNamespaces:
+        - etcd-backup
     stableHostname: true
     apidCheckExtKeyUsage: true
     kubePrism:

terraform/aws-kubernetes-irsa.tf

@@ -127,6 +127,49 @@ resource "aws_iam_role_policy_attachment" "cloudflared" {
   policy_arn = aws_iam_policy.cloudflared.arn
 }
 
+resource "aws_iam_role" "etcd-backup" {
+  name = "${local.project}-etcd-backup"
+  assume_role_policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Principal" : {
+          "Federated" : "${aws_iam_openid_connect_provider.kubernetes-oidc.arn}"
+        },
+        "Action" : "sts:AssumeRoleWithWebIdentity",
+        "Condition" : {
+          "StringEquals" : {
+            "${aws_iam_openid_connect_provider.kubernetes-oidc.url}:sub" : "system:serviceaccount:etcd-backup:etcd-backup",
+            "${aws_iam_openid_connect_provider.kubernetes-oidc.url}:aud" : "sts.amazonaws.com"
+          }
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_policy" "etcd-backup" {
+  name = "${local.project}-etcd-backup"
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Action" : "ssm:GetParameters",
+        "Effect" : "Allow",
+        "Resource" : [
+          "arn:aws:ssm:${data.aws_region.main.name}:${data.aws_caller_identity.main.account_id}:parameter/kubernetes/etcd-backup"
+        ]
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "etcd-backup" {
+  role       = aws_iam_role.etcd-backup.name
+  policy_arn = aws_iam_policy.etcd-backup.arn
+}
+
 resource "aws_iam_role" "snmp-exporter-mikrotik" {
   name = "${local.project}-snmp-exporter-mikrotik"
   assume_role_policy = jsonencode({