From 5a9cc428e983d6829e72eb0e8ba5cb5a0120005d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nils=20M=C3=BCller?= Date: Thu, 19 Oct 2023 21:44:48 +0200 Subject: [PATCH] feat(kube-nas): setup cilium as helm chart #2008 --- .../app/cilium-l2-announcement-policy.yaml | 13 ++ .../app/cilium-load-balancer-ip-pool.yaml | 8 ++ .../kube-system/cilium/app/helm-release.yaml | 122 +++++++++++++++++ .../kube-system/cilium/app/kustomization.yaml | 10 ++ .../apps/kube-system/cilium/flux-sync.yaml | 17 +++ .../apps/kube-system/kustomization.yaml | 1 + kubernetes/kube-nas/bootstrap/cilium/l2.yaml | 7 +- .../kube-nas/bootstrap/cilium/values.yaml | 128 ++++++++++++------ .../flux/repositories/helm/cilium-charts.yaml | 10 ++ 9 files changed, 274 insertions(+), 42 deletions(-) create mode 100644 kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-l2-announcement-policy.yaml create mode 100644 kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-load-balancer-ip-pool.yaml create mode 100644 kubernetes/kube-nas/apps/kube-system/cilium/app/helm-release.yaml create mode 100644 kubernetes/kube-nas/apps/kube-system/cilium/app/kustomization.yaml create mode 100644 kubernetes/kube-nas/apps/kube-system/cilium/flux-sync.yaml create mode 100644 kubernetes/kube-nas/flux/repositories/helm/cilium-charts.yaml diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-l2-announcement-policy.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-l2-announcement-policy.yaml new file mode 100644 index 000000000..4fa9c1a60 --- /dev/null +++ b/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-l2-announcement-policy.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: cilium.io/v2alpha1 +kind: CiliumL2AnnouncementPolicy +metadata: + name: policy +spec: + loadBalancerIPs: true + interfaces: + - eno1 + - wlp58s0 + nodeSelector: + matchLabels: + kubernetes.io/os: linux diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-load-balancer-ip-pool.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-load-balancer-ip-pool.yaml new file mode 100644 index 000000000..0f3c2806a --- /dev/null +++ b/kubernetes/kube-nas/apps/kube-system/cilium/app/cilium-load-balancer-ip-pool.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: pool +spec: + cidrs: + - cidr: 192.168.1.90/30 diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/app/helm-release.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/app/helm-release.yaml new file mode 100644 index 000000000..f9676788f --- /dev/null +++ b/kubernetes/kube-nas/apps/kube-system/cilium/app/helm-release.yaml @@ -0,0 +1,122 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta1.json +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: cilium +spec: + interval: 30m + chart: + spec: + chart: cilium + version: 1.14.3 + sourceRef: + kind: HelmRepository + name: cilium-charts + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + autoDirectNodeRoutes: true + bandwidthManager: + enabled: true + bbr: true + bpf: + masquerade: true + bgp: + enabled: false + cluster: + name: kube-nas + id: 1 + containerRuntime: + integration: containerd + socketPath: /var/run/k3s/containerd/containerd.sock + endpointRoutes: + enabled: true + hubble: + enabled: true + metrics: + enabled: + - dns:query + - drop + - tcp + - flow + - port-distribution + - icmp + - http + serviceMonitor: + enabled: false + dashboards: + enabled: false + annotations: + grafana_folder: Cilium + relay: + enabled: true + rollOutPods: true + prometheus: + serviceMonitor: + enabled: false + ui: + enabled: true + rollOutPods: true + ingress: + enabled: true + className: nginx + annotations: + cert-manager.io/cluster-issuer: self-signed + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + hosts: + - &host hubble.tyriis.dev + tls: + - hosts: + - *host + ipam: + mode: kubernetes + ipv4NativeRoutingCIDR: 10.42.0.0/16 + # k8sServiceHost: 192.168.1.90 + # k8sServicePort: 6443 + kubeProxyReplacement: strict + # kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + l2announcements: + enabled: true + leaseDuration: 120s + leaseRenewDeadline: 60s + leaseRetryPeriod: 1s + loadBalancer: + algorithm: maglev + mode: dsr + localRedirectPolicy: true + operator: + replicas: 1 + rollOutPods: true + prometheus: + enabled: true + serviceMonitor: + enabled: false + dashboards: + enabled: false + annotations: + grafana_folder: Cilium + prometheus: + enabled: true + serviceMonitor: + enabled: false + trustCRDsExist: false + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + rollOutCiliumPods: true + securityContext: + privileged: true + tunnel: disabled diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/app/kustomization.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/app/kustomization.yaml new file mode 100644 index 000000000..8a66b80e4 --- /dev/null +++ b/kubernetes/kube-nas/apps/kube-system/cilium/app/kustomization.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kube-system +resources: + - ./helm-release.yaml + # as we need to bootstrap cilium it is fine to use crd before install + - ./cilium-l2-announcement-policy.yaml + - ./cilium-load-balancer-ip-pool.yaml diff --git a/kubernetes/kube-nas/apps/kube-system/cilium/flux-sync.yaml b/kubernetes/kube-nas/apps/kube-system/cilium/flux-sync.yaml new file mode 100644 index 000000000..23f311579 --- /dev/null +++ b/kubernetes/kube-nas/apps/kube-system/cilium/flux-sync.yaml @@ -0,0 +1,17 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1beta2.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: apps-cilium + namespace: flux-system +spec: + path: ./kubernetes/kube-nas/apps/kube-system/cilium/app + prune: false # never should be deleted + sourceRef: + kind: GitRepository + name: home-ops + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/kube-nas/apps/kube-system/kustomization.yaml b/kubernetes/kube-nas/apps/kube-system/kustomization.yaml index ecc436f8e..da85cc5ca 100644 --- a/kubernetes/kube-nas/apps/kube-system/kustomization.yaml +++ b/kubernetes/kube-nas/apps/kube-system/kustomization.yaml @@ -4,6 +4,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml + - ./cilium.yaml - ./coredns/flux-sync.yaml - ./kubelet-csr-approver/flux-sync.yaml # - ./local-path-provisioner/flux-sync.yaml diff --git a/kubernetes/kube-nas/bootstrap/cilium/l2.yaml b/kubernetes/kube-nas/bootstrap/cilium/l2.yaml index a575d24c8..5c0e79c98 100644 --- a/kubernetes/kube-nas/bootstrap/cilium/l2.yaml +++ b/kubernetes/kube-nas/bootstrap/cilium/l2.yaml @@ -6,16 +6,17 @@ metadata: spec: loadBalancerIPs: true interfaces: - - ^eno.*|^wlp.* + - eno1 + - wlp58s0 nodeSelector: matchLabels: kubernetes.io/os: linux + --- -# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/cilium.io/ciliumloadbalancerippool_v2alpha1.json apiVersion: cilium.io/v2alpha1 kind: CiliumLoadBalancerIPPool metadata: name: pool spec: cidrs: - - cidr: 192.168.1.243/31 + - cidr: 192.168.1.90/30 diff --git a/kubernetes/kube-nas/bootstrap/cilium/values.yaml b/kubernetes/kube-nas/bootstrap/cilium/values.yaml index 6d3731433..d851d9de4 100644 --- a/kubernetes/kube-nas/bootstrap/cilium/values.yaml +++ b/kubernetes/kube-nas/bootstrap/cilium/values.yaml @@ -1,46 +1,96 @@ --- -# autoDirectNodeRoutes: true -# bpf: -# masquerade: true -# bgp: -# enabled: false -# cluster: -# name: kube-nas -# id: 1 -# containerRuntime: -# integration: containerd -# socketPath: /var/run/k3s/containerd/containerd.sock -# endpointRoutes: -# enabled: true -# hubble: -# enabled: false -# ipam: -# mode: kubernetes -# ipv4NativeRoutingCIDR: 10.32.0.0/16 -# k8sServiceHost: 192.168.1.242 -# k8sServicePort: 6443 -# kubeProxyReplacement: true -# kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 -# l2announcements: -# enabled: true -# leaseDuration: 120s -# leaseRenewDeadline: 60s -# leaseRetryPeriod: 1s -# loadBalancer: -# algorithm: maglev -# mode: dsr -# localRedirectPolicy: true -# operator: -# rollOutPods: true -# rollOutCiliumPods: true -# securityContext: -# privileged: true -# tunnel: disabled - +autoDirectNodeRoutes: true +bandwidthManager: + enabled: true + bbr: true +bpf: + masquerade: true +bgp: + enabled: false +cluster: + name: kube-nas + id: 1 containerRuntime: integration: containerd socketPath: /var/run/k3s/containerd/containerd.sock - +endpointRoutes: + enabled: true +hubble: + enabled: true + metrics: + enabled: + - dns:query + - drop + - tcp + - flow + - port-distribution + - icmp + - http + serviceMonitor: + enabled: false + dashboards: + enabled: false + annotations: + grafana_folder: Cilium + relay: + enabled: true + rollOutPods: true + prometheus: + serviceMonitor: + enabled: false + ui: + enabled: true + rollOutPods: true + ingress: + enabled: true + className: nginx + annotations: + cert-manager.io/cluster-issuer: self-signed + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + hosts: + - &host hubble.tyriis.dev + tls: + - hosts: + - *host +ipam: + mode: kubernetes +ipv4NativeRoutingCIDR: 10.42.0.0/16 +# k8sServiceHost: 192.168.1.90 +# k8sServicePort: 6443 kubeProxyReplacement: strict +# kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 +l2announcements: + enabled: true + leaseDuration: 120s + leaseRenewDeadline: 60s + leaseRetryPeriod: 1s +loadBalancer: + algorithm: maglev + mode: dsr +localRedirectPolicy: true operator: + rollOutPods: true replicas: 1 + prometheus: + enabled: true + serviceMonitor: + enabled: false + dashboards: + enabled: false + annotations: + grafana_folder: Cilium +prometheus: + enabled: true + serviceMonitor: + enabled: false + trustCRDsExist: false +dashboards: + enabled: true + annotations: + grafana_folder: Cilium +rollOutCiliumPods: true +securityContext: + privileged: true +tunnel: disabled diff --git a/kubernetes/kube-nas/flux/repositories/helm/cilium-charts.yaml b/kubernetes/kube-nas/flux/repositories/helm/cilium-charts.yaml new file mode 100644 index 000000000..8112ff6ef --- /dev/null +++ b/kubernetes/kube-nas/flux/repositories/helm/cilium-charts.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrepository-source-v1beta2.json +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: cilium-charts + namespace: flux-system +spec: + interval: 2h + url: https://helm.cilium.io