From 0621756ac5a9e54c6a2324607888575700b706d1 Mon Sep 17 00:00:00 2001 From: unchama <11990197+unchama@users.noreply.github.com> Date: Sat, 2 Dec 2023 12:19:47 +0900 Subject: [PATCH] enable bgp control plane / change ip-addr --- seichi-onp-k8s/cluster-boot-up/README.md | 8 +- .../scripts/nodes/k8s-node-setup.sh | 7 +- .../manifests/seichi-kubernetes/README.md | 4 +- .../app-of-other-apps/cilium.yaml | 103 ++++++++++++++++++ .../bungeecord/redisbungee-redis.yaml | 3 + .../service-bungeecord-loadbalancer.yaml | 3 +- .../service-bungeecord-loadbalancer.yaml | 3 +- .../redis/bungeesemaphore-redis.yaml | 3 + .../redis/redisbungee-redis.yaml | 3 + 9 files changed, 127 insertions(+), 10 deletions(-) diff --git a/seichi-onp-k8s/cluster-boot-up/README.md b/seichi-onp-k8s/cluster-boot-up/README.md index cf3615992..fbf5bc599 100644 --- a/seichi-onp-k8s/cluster-boot-up/README.md +++ b/seichi-onp-k8s/cluster-boot-up/README.md @@ -50,14 +50,16 @@ KubernetesノードのVMは cloudinit イメージで作成されています。 - Storage Network (192.168.16.0/22) - Kubernetes - Internal - - Pod Network (10.128.0.0/16) - - Service Network (10.96.0.0/16) + - Pod Network (10.96.128.0/18) + - Service Network (10.96.64.0/18) - External - Node IP - Service Network (192.168.0.0-192.168.0.127) + - 192.168.0.0/22 の一部を使用 - Storage Network (192.168.18.0-192.168.18.127) + - 192.168.16.0/22 の一部を使用 - API Endpoint (192.168.18.100) - - LoadBalancer VIP (192.168.0.128-192.168.0.255) + - LoadBalancer VIP (10.96.0.0/22) ## Kubernetesクラスタの構成 diff --git a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh index 41e8eb974..cd036622e 100644 --- a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh +++ b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh @@ -280,8 +280,8 @@ nodeRegistration: apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration networking: - serviceSubnet: "10.96.0.0/16" - podSubnet: "10.128.0.0/16" + serviceSubnet: "10.96.64.0/18" + podSubnet: "10.96.128.0/18" kubernetesVersion: "v1.27.5" controlPlaneEndpoint: "${KUBE_API_SERVER_VIP}:8443" apiServer: @@ -328,7 +328,8 @@ helm install cilium cilium/cilium \ --namespace kube-system \ --set kubeProxyReplacement=strict \ --set k8sServiceHost=${KUBE_API_SERVER_VIP} \ - --set k8sServicePort=8443 + --set k8sServicePort=8443 \ + --set bgpControlPlane.enabled=true # Generate control plane certificate KUBEADM_UPLOADED_CERTS=$(kubeadm init phase upload-certs --upload-certs | tail -n 1) diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/README.md b/seichi-onp-k8s/manifests/seichi-kubernetes/README.md index 5395ddbbd..14536395e 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/README.md +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/README.md @@ -42,8 +42,8 @@ TCP パケットをそのまま送り届ける必要があります。 | サービス | `Service` の VIP | | ---------------------------- | ----------------------------------------------------------- | -| BungeeCord (本番環境用) | [`192.168.0.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | -| BungeeCord (デバッグ環境用) | [`192.168.0.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | +| BungeeCord (本番環境用) | [`10.96.0.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | +| BungeeCord (デバッグ環境用) | [`10.96.0.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | | 投票受付サーバー | (まだ k8s 上に乗っていないので、 `Service` の VIP ではない) | ### オンプレネットワーク内からのトラフィックを受ける `Service` diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml index 1867f5a1b..eacfa5fac 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml @@ -15,6 +15,8 @@ spec: kubeProxyReplacement: strict k8sServiceHost: 192.168.18.100 # modify it if necessary k8sServicePort: 8443 + bgpControlPlane: + enabled: true pprof: enabled: true loadBalancer: @@ -95,3 +97,104 @@ spec: automated: prune: true selfHeal: true +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumLoadBalancerIPPool +metadata: + name: "lb-pool" +spec: + cidrs: + # 10.96.0.0-10.96.3.255 をloadBalancerのIPに割当可能 + - cidr: "10.96.0.0/22" +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-cp-1 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-cp-1 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-cp-2 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-cp-2 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + serviceSelector: + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-cp-3 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-cp-3 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-wk-1 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-wk-1 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-wk-2 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-wk-2 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-wk-3 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-wk-3 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml index 68e0f5018..5c757e8c3 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml @@ -19,6 +19,9 @@ spec: notify-keyspace-events "Eg$x" master: service: + # loadBalancerIP has been deprecated in k8s v1.24 + # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある + # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips type: LoadBalancer loadBalancerIP: 192.168.0.134 resources: diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml index bf66790cd..72a68e463 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: Service metadata: name: bungeecord + annotations: + "io.cilium/lb-ipam-ips": "10.96.0.131" spec: type: LoadBalancer - loadBalancerIP: 192.168.0.131 diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml index 560d90aa1..ad849760e 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: Service metadata: name: bungeecord + annotations: + "io.cilium/lb-ipam-ips": "10.96.0.130" spec: type: LoadBalancer - loadBalancerIP: 192.168.0.130 diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml index 166331fab..f850cbf60 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml @@ -19,6 +19,9 @@ spec: notify-keyspace-events "Eg$x" master: service: + # loadBalancerIP has been deprecated in k8s v1.24 + # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある + # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips type: LoadBalancer loadBalancerIP: 192.168.0.133 resources: diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml index 2d65d9036..6f643b563 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml @@ -19,6 +19,9 @@ spec: notify-keyspace-events "Eg$x" master: service: + # loadBalancerIP has been deprecated in k8s v1.24 + # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある + # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips type: LoadBalancer loadBalancerIP: 192.168.0.132 resources: