feat(kube-nas): setup cilium as helm chart #2008

tyriis · Oct 19, 2023 · 5a9cc42 · 5a9cc42
1 parent 92d0c6f
commit 5a9cc42
Show file tree

Hide file tree

Showing 9 changed files with 274 additions and 42 deletions.
diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-l2-announcement-policy.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-l2-announcement-policy.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: policy
+spec:
+  loadBalancerIPs: true
+  interfaces:
+    - eno1
+    - wlp58s0
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/os: linux
diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-load-balancer-ip-pool.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-load-balancer-ip-pool.yaml
@@ -0,0 +1,8 @@
+---
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: pool
+spec:
+  cidrs:
+    - cidr: 192.168.1.90/30
diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/app/helm-release.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/app/helm-release.yaml
@@ -0,0 +1,122 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta1.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: cilium
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: cilium
+      version: 1.14.3
+      sourceRef:
+        kind: HelmRepository
+        name: cilium-charts
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    autoDirectNodeRoutes: true
+    bandwidthManager:
+      enabled: true
+      bbr: true
+    bpf:
+      masquerade: true
+    bgp:
+      enabled: false
+    cluster:
+      name: kube-nas
+      id: 1
+    containerRuntime:
+      integration: containerd
+      socketPath: /var/run/k3s/containerd/containerd.sock
+    endpointRoutes:
+      enabled: true
+    hubble:
+      enabled: true
+      metrics:
+        enabled:
+          - dns:query
+          - drop
+          - tcp
+          - flow
+          - port-distribution
+          - icmp
+          - http
+        serviceMonitor:
+          enabled: false
+        dashboards:
+          enabled: false
+          annotations:
+            grafana_folder: Cilium
+      relay:
+        enabled: true
+        rollOutPods: true
+        prometheus:
+          serviceMonitor:
+            enabled: false
+      ui:
+        enabled: true
+        rollOutPods: true
+        ingress:
+          enabled: true
+          className: nginx
+          annotations:
+            cert-manager.io/cluster-issuer: self-signed
+            kubernetes.io/tls-acme: "true"
+            nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+            nginx.ingress.kubernetes.io/ssl-redirect: "true"
+          hosts:
+            - &host hubble.tyriis.dev
+          tls:
+            - hosts:
+                - *host
+    ipam:
+      mode: kubernetes
+    ipv4NativeRoutingCIDR: 10.42.0.0/16
+    # k8sServiceHost: 192.168.1.90
+    # k8sServicePort: 6443
+    kubeProxyReplacement: strict
+    # kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+    l2announcements:
+      enabled: true
+      leaseDuration: 120s
+      leaseRenewDeadline: 60s
+      leaseRetryPeriod: 1s
+    loadBalancer:
+      algorithm: maglev
+      mode: dsr
+    localRedirectPolicy: true
+    operator:
+      replicas: 1
+      rollOutPods: true
+      prometheus:
+        enabled: true
+        serviceMonitor:
+          enabled: false
+      dashboards:
+        enabled: false
+        annotations:
+          grafana_folder: Cilium
+    prometheus:
+      enabled: true
+      serviceMonitor:
+        enabled: false
+        trustCRDsExist: false
+    dashboards:
+      enabled: true
+      annotations:
+        grafana_folder: Cilium
+    rollOutCiliumPods: true
+    securityContext:
+      privileged: true
+    tunnel: disabled
diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/app/kustomization.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/app/kustomization.yaml
@@ -0,0 +1,10 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kube-system
+resources:
+  - ./helm-release.yaml
+  # as we need to bootstrap cilium it is fine to use crd before install
+  - ./cilium-l2-announcement-policy.yaml
+  - ./cilium-load-balancer-ip-pool.yaml
diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/flux-sync.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/flux-sync.yaml
@@ -0,0 +1,17 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1beta2.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: apps-cilium
+  namespace: flux-system
+spec:
+  path: ./kubernetes/kube-nas/apps/kube-system/cilium/app
+  prune: false # never should be deleted
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
diff --git a/kubernetes/kube-nas/apps/kube-system/kustomization.yaml b/kubernetes/kube-nas/apps/kube-system/kustomization.yaml
@@ -4,6 +4,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
+  - ./cilium.yaml
   - ./coredns/flux-sync.yaml
   - ./kubelet-csr-approver/flux-sync.yaml
   # - ./local-path-provisioner/flux-sync.yaml

diff --git a/kubernetes/kube-nas/bootstrap/cilium/l2.yaml b/kubernetes/kube-nas/bootstrap/cilium/l2.yaml
@@ -6,16 +6,17 @@ metadata:
 spec:
   loadBalancerIPs: true
   interfaces:
-    - ^eno.*|^wlp.*
+    - eno1
+    - wlp58s0
   nodeSelector:
     matchLabels:
       kubernetes.io/os: linux
+
 ---
-# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/cilium.io/ciliumloadbalancerippool_v2alpha1.json
 apiVersion: cilium.io/v2alpha1
 kind: CiliumLoadBalancerIPPool
 metadata:
   name: pool
 spec:
   cidrs:
-    - cidr: 192.168.1.243/31
+    - cidr: 192.168.1.90/30
diff --git a/kubernetes/kube-nas/bootstrap/cilium/values.yaml b/kubernetes/kube-nas/bootstrap/cilium/values.yaml
@@ -1,46 +1,96 @@
 ---
-# autoDirectNodeRoutes: true
-# bpf:
-#   masquerade: true
-# bgp:
-#   enabled: false
-# cluster:
-#   name: kube-nas
-#   id: 1
-# containerRuntime:
-#   integration: containerd
-#   socketPath: /var/run/k3s/containerd/containerd.sock
-# endpointRoutes:
-#   enabled: true
-# hubble:
-#   enabled: false
-# ipam:
-#   mode: kubernetes
-# ipv4NativeRoutingCIDR: 10.32.0.0/16
-# k8sServiceHost: 192.168.1.242
-# k8sServicePort: 6443
-# kubeProxyReplacement: true
-# kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
-# l2announcements:
-#   enabled: true
-#   leaseDuration: 120s
-#   leaseRenewDeadline: 60s
-#   leaseRetryPeriod: 1s
-# loadBalancer:
-#   algorithm: maglev
-#   mode: dsr
-# localRedirectPolicy: true
-# operator:
-#   rollOutPods: true
-# rollOutCiliumPods: true
-# securityContext:
-#   privileged: true
-# tunnel: disabled
-
+autoDirectNodeRoutes: true
+bandwidthManager:
+  enabled: true
+  bbr: true
+bpf:
+  masquerade: true
+bgp:
+  enabled: false
+cluster:
+  name: kube-nas
+  id: 1
 containerRuntime:
   integration: containerd
   socketPath: /var/run/k3s/containerd/containerd.sock
-
+endpointRoutes:
+  enabled: true
+hubble:
+  enabled: true
+  metrics:
+    enabled:
+      - dns:query
+      - drop
+      - tcp
+      - flow
+      - port-distribution
+      - icmp
+      - http
+    serviceMonitor:
+      enabled: false
+    dashboards:
+      enabled: false
+      annotations:
+        grafana_folder: Cilium
+  relay:
+    enabled: true
+    rollOutPods: true
+    prometheus:
+      serviceMonitor:
+        enabled: false
+  ui:
+    enabled: true
+    rollOutPods: true
+    ingress:
+      enabled: true
+      className: nginx
+      annotations:
+        cert-manager.io/cluster-issuer: self-signed
+        kubernetes.io/tls-acme: "true"
+        nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+        nginx.ingress.kubernetes.io/ssl-redirect: "true"
+      hosts:
+        - &host hubble.tyriis.dev
+      tls:
+        - hosts:
+            - *host
+ipam:
+  mode: kubernetes
+ipv4NativeRoutingCIDR: 10.42.0.0/16
+# k8sServiceHost: 192.168.1.90
+# k8sServicePort: 6443
 kubeProxyReplacement: strict
+# kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+l2announcements:
+  enabled: true
+  leaseDuration: 120s
+  leaseRenewDeadline: 60s
+  leaseRetryPeriod: 1s
+loadBalancer:
+  algorithm: maglev
+  mode: dsr
+localRedirectPolicy: true
 operator:
+  rollOutPods: true
   replicas: 1
+  prometheus:
+    enabled: true
+    serviceMonitor:
+      enabled: false
+  dashboards:
+    enabled: false
+    annotations:
+      grafana_folder: Cilium
+prometheus:
+  enabled: true
+  serviceMonitor:
+    enabled: false
+    trustCRDsExist: false
+dashboards:
+  enabled: true
+  annotations:
+    grafana_folder: Cilium
+rollOutCiliumPods: true
+securityContext:
+  privileged: true
+tunnel: disabled
diff --git a/kubernetes/kube-nas/flux/repositories/helm/cilium-charts.yaml b/kubernetes/kube-nas/flux/repositories/helm/cilium-charts.yaml
@@ -0,0 +1,10 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrepository-source-v1beta2.json
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: cilium-charts
+  namespace: flux-system
+spec:
+  interval: 2h
+  url: https://helm.cilium.io