From 1d183b04426d1726cc031949f9fef1fa6b9a88cb Mon Sep 17 00:00:00 2001 From: unchama <11990197+unchama@users.noreply.github.com> Date: Sun, 3 Dec 2023 21:08:26 +0900 Subject: [PATCH] =?UTF-8?q?kubeadm=20join=E3=81=AB=E5=BF=85=E8=A6=81?= =?UTF-8?q?=E3=81=AAconfig=E3=82=92ansible=E3=81=AB=E4=BD=9C=E3=82=89?= =?UTF-8?q?=E3=81=9B=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hosts/k8s-servers/group_vars/all.yaml | 5 ++ .../ansible/hosts/k8s-servers/inventory | 72 ++++++++++++------- .../roles/11-kubeadm-join-cp/tasks/main.yaml | 7 +- .../templates/join_kubeadm_cp.yaml | 21 ++++++ .../roles/12-kubeadm-join-wk/tasks/main.yaml | 6 +- .../tasks/templates/join_kubeadm_wk.yaml | 16 +++++ .../scripts/nodes/k8s-node-setup.sh | 47 +----------- 7 files changed, 96 insertions(+), 78 deletions(-) create mode 100644 seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml create mode 100644 seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml create mode 100644 seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml new file mode 100644 index 000000000..03e870245 --- /dev/null +++ b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml @@ -0,0 +1,5 @@ +--- +ansible_python_interpreter: /usr/bin/python3 +ansible_port: 22 +ansible_user: cloudinit +kube_api_server_vip: 192.168.18.100 diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory index ce812b376..0fc0ad0f8 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory +++ b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory @@ -1,56 +1,74 @@ -[k8s-servers] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 -seichi-onp-k8s-wk-1 ansible_host=192.168.8.21 -seichi-onp-k8s-wk-2 ansible_host=192.168.8.22 -seichi-onp-k8s-wk-3 ansible_host=192.168.8.23 +[seichi-onp-k8s-cp-1] +cp-1 ansible_host=192.168.0.11 host_addr_srv=192.168.0.11 host_addr_san=192.168.18.11 host_addr_cls=192.168.32.11 + +[seichi-onp-k8s-cp-2] +cp-2 ansible_host=192.168.0.12 host_addr_srv=192.168.0.12 host_addr_san=192.168.18.12 host_addr_cls=192.168.32.12 + +[seichi-onp-k8s-cp-3] +cp-3 ansible_host=192.168.0.13 host_addr_srv=192.168.0.13 host_addr_san=192.168.18.13 host_addr_cls=192.168.32.13 + +[seichi-onp-k8s-wk-1] +wk-1 ansible_host=192.168.0.21 host_addr_srv=192.168.0.21 host_addr_san=192.168.18.21 host_addr_cls=192.168.32.21 + +[seichi-onp-k8s-wk-2] +wk-2 ansible_host=192.168.0.22 host_addr_srv=192.168.0.22 host_addr_san=192.168.18.22 host_addr_cls=192.168.32.22 + +[seichi-onp-k8s-wk-3] +wk-3 ansible_host=192.168.0.23 host_addr_srv=192.168.0.23 host_addr_san=192.168.18.23 host_addr_cls=192.168.32.23 + + +[k8s-servers:children] +seichi-onp-k8s-cp-1 +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 +seichi-onp-k8s-wk-1 +seichi-onp-k8s-wk-2 +seichi-onp-k8s-wk-3 [k8s-servers:vars] ansible_ssh_pass=zaq12wsx [k8s-servers-with-ssh:children] -k8s-servers +seichi-onp-k8s-cp-1 +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 +seichi-onp-k8s-wk-1 +seichi-onp-k8s-wk-2 +seichi-onp-k8s-wk-3 [k8s-servers-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-cp-with-ssh] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 +[k8s-servers-cp-with-ssh:children] +seichi-onp-k8s-cp-1 +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 [k8s-servers-cp-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-cp-leader-with-ssh] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 +[k8s-servers-cp-leader-with-ssh:children] +seichi-onp-k8s-cp-1 [k8s-servers-cp-leader-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-cp-follower-with-ssh] -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 +[k8s-servers-cp-follower-with-ssh:children] +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 [k8s-servers-cp-follower-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-wk-with-ssh] -seichi-onp-k8s-wk-1 ansible_host=192.168.8.21 -seichi-onp-k8s-wk-2 ansible_host=192.168.8.22 -seichi-onp-k8s-wk-3 ansible_host=192.168.8.23 +[k8s-servers-wk-with-ssh:children] +seichi-onp-k8s-wk-1 +seichi-onp-k8s-wk-2 +seichi-onp-k8s-wk-3 [k8s-servers-wk-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 - - -[all:vars] -ansible_python_interpreter=/usr/bin/python3 -ansible_port=22 -ansible_user=cloudinit \ No newline at end of file diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml index 121469655..dd8af67c0 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml @@ -1,10 +1,9 @@ -- name: Copy file +- name: Deploy kubeadm join config file become: yes - ansible.builtin.copy: - src: /root/join_kubeadm_cp.yaml + template: + src: /home/cloudinit/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml dest: /root/join_kubeadm_cp.yaml - name: Execute kubeadm join command become: yes shell: "kubeadm join --config /root/join_kubeadm_cp.yaml" - \ No newline at end of file diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml new file mode 100644 index 000000000..331655b2d --- /dev/null +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml @@ -0,0 +1,21 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: "systemd" +protectKernelDefaults: true +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: JoinConfiguration +nodeRegistration: + criSocket: "unix:///var/run/containerd/containerd.sock" + kubeletExtraArgs: + node-ip: "{{ host_addr_cls }}" +localAPIEndpoint: + advertiseAddress: "{{ host_addr_cls }}" + bindPort: 6443 +discovery: + bootstrapToken: + apiServerEndpoint: "{{ kube_api_server_vip }}:8443" + token: "{{ kubeadm_bootstrap_token }}" + unsafeSkipCAVerification: true +controlPlane: + certificateKey: "{{ kubeadm_uploaded_certs }}" diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml index 06da29d94..99a8841ed 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml @@ -1,7 +1,7 @@ -- name: Copy file +- name: Deploy kubeadm join config file become: yes - ansible.builtin.copy: - src: /root/join_kubeadm_wk.yaml + template: + src: /home/cloudinit/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/templates/join_kubeadm_wk.yaml dest: /root/join_kubeadm_wk.yaml - name: Execute kubeadm join command diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml new file mode 100644 index 000000000..1db412aa0 --- /dev/null +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml @@ -0,0 +1,16 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: "systemd" +protectKernelDefaults: true +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: JoinConfiguration +nodeRegistration: + criSocket: "unix:///var/run/containerd/containerd.sock" + kubeletExtraArgs: + node-ip: "{{ host_addr_cls }}" +discovery: + bootstrapToken: + apiServerEndpoint: "{{ kube_api_server_vip }}:8443" + token: "{{ kubeadm_bootstrap_token }}" + unsafeSkipCAVerification: true diff --git a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh index 0295f4bed..c4b89518c 100644 --- a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh +++ b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh @@ -343,50 +343,9 @@ helm install cilium cilium/cilium \ # Generate control plane certificate KUBEADM_UPLOADED_CERTS=$(kubeadm init phase upload-certs --upload-certs | tail -n 1) -# Set join configuration for other control plane nodes -cat > "$HOME"/join_kubeadm_cp.yaml < "$HOME"/join_kubeadm_wk.yaml <> "$HOME"/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml +echo "kubeadm_uploaded_certs: $KUBEADM_UPLOADED_CERTS" >> "$HOME"/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml # install ansible sudo apt-get install -y ansible git sshpass