enable bgp control plane / change ip-addr

GiganticMinecraft · Dec 2, 2023 · 0621756 · 0621756
1 parent d5105e9
commit 0621756
Show file tree

Hide file tree

Showing 9 changed files with 127 additions and 10 deletions.
diff --git a/seichi-onp-k8s/cluster-boot-up/README.md b/seichi-onp-k8s/cluster-boot-up/README.md
@@ -50,14 +50,16 @@ KubernetesノードのVMは cloudinit イメージで作成されています。
  - Storage Network (192.168.16.0/22)
  - Kubernetes
    - Internal
-     - Pod Network (10.128.0.0/16)
-     - Service Network (10.96.0.0/16)
+    - Pod Network (10.96.128.0/18)
+    - Service Network (10.96.64.0/18)
    - External
      - Node IP
        - Service Network (192.168.0.0-192.168.0.127)
+         - 192.168.0.0/22 の一部を使用
        - Storage Network (192.168.18.0-192.168.18.127)
+         - 192.168.16.0/22 の一部を使用
      - API Endpoint (192.168.18.100)
-     - LoadBalancer VIP (192.168.0.128-192.168.0.255)
+    - LoadBalancer VIP (10.96.0.0/22)
 
 ## Kubernetesクラスタの構成
 

diff --git a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh
@@ -280,8 +280,8 @@ nodeRegistration:
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: ClusterConfiguration
 networking:
-  serviceSubnet: "10.96.0.0/16"
-  podSubnet: "10.128.0.0/16"
+  serviceSubnet: "10.96.64.0/18"
+  podSubnet: "10.96.128.0/18"
 kubernetesVersion: "v1.27.5"
 controlPlaneEndpoint: "${KUBE_API_SERVER_VIP}:8443"
 apiServer:
@@ -328,7 +328,8 @@ helm install cilium cilium/cilium \
     --namespace kube-system \
     --set kubeProxyReplacement=strict \
     --set k8sServiceHost=${KUBE_API_SERVER_VIP} \
-    --set k8sServicePort=8443
+    --set k8sServicePort=8443 \
+    --set bgpControlPlane.enabled=true
 
 # Generate control plane certificate
 KUBEADM_UPLOADED_CERTS=$(kubeadm init phase upload-certs --upload-certs | tail -n 1)

diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/README.md b/seichi-onp-k8s/manifests/seichi-kubernetes/README.md
@@ -42,8 +42,8 @@ TCP パケットをそのまま送り届ける必要があります。
 
 | サービス                     | `Service` の VIP                                            | 
 | ---------------------------- | ----------------------------------------------------------- | 
-|  BungeeCord (本番環境用)     | [`192.168.0.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | 
-|  BungeeCord (デバッグ環境用) | [`192.168.0.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | 
+|  BungeeCord (本番環境用)     | [`10.96.0.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | 
+|  BungeeCord (デバッグ環境用) | [`10.96.0.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | 
 |  投票受付サーバー            | (まだ k8s 上に乗っていないので、 `Service` の VIP ではない) |
 
 ### オンプレネットワーク内からのトラフィックを受ける `Service`

diff --git a/...-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml b/...-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml
@@ -15,6 +15,8 @@ spec:
         kubeProxyReplacement: strict
         k8sServiceHost: 192.168.18.100 # modify it if necessary
         k8sServicePort: 8443
+        bgpControlPlane:
+          enabled: true
         pprof:
           enabled: true
         loadBalancer:
@@ -95,3 +97,104 @@ spec:
     automated:
       prune: true
       selfHeal: true
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: "lb-pool"
+spec:
+  cidrs:
+  # 10.96.0.0-10.96.3.255 をloadBalancerのIPに割当可能
+  - cidr: "10.96.0.0/22"
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-cp-1
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-cp-1
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-cp-2
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-cp-2
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    serviceSelector:
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-cp-3
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-cp-3
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-wk-1
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-wk-1
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-wk-2
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-wk-2
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-wk-3
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-wk-3
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
diff --git a/...s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml b/...s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml
@@ -19,6 +19,9 @@ spec:
           notify-keyspace-events "Eg$x"
         master:
           service:
+            # loadBalancerIP has been deprecated in k8s v1.24
+            # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある
+            # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips
             type: LoadBalancer
             loadBalancerIP: 192.168.0.134
           resources:

diff --git a/...ichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/...ichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml
@@ -3,6 +3,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: bungeecord
+  annotations:
+    "io.cilium/lb-ipam-ips": "10.96.0.131"
 spec:
   type: LoadBalancer
-  loadBalancerIP: 192.168.0.131
diff --git a/...sts/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/...sts/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml
@@ -3,6 +3,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: bungeecord
+  annotations:
+    "io.cilium/lb-ipam-ips": "10.96.0.130"
 spec:
   type: LoadBalancer
-  loadBalancerIP: 192.168.0.130
diff --git a/...np-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml b/...np-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml
@@ -19,6 +19,9 @@ spec:
           notify-keyspace-events "Eg$x"
         master:
           service:
+            # loadBalancerIP has been deprecated in k8s v1.24
+            # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある
+            # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips
             type: LoadBalancer
             loadBalancerIP: 192.168.0.133
           resources:

diff --git a/...hi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml b/...hi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml
@@ -19,6 +19,9 @@ spec:
           notify-keyspace-events "Eg$x"
         master:
           service:
+            # loadBalancerIP has been deprecated in k8s v1.24
+            # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある
+            # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips
             type: LoadBalancer
             loadBalancerIP: 192.168.0.132
           resources: