From 2b63105055a7c0c34a25876c510af5e3c0ff643b Mon Sep 17 00:00:00 2001 From: "eric.domeier" Date: Fri, 24 May 2024 22:59:32 -0400 Subject: [PATCH 1/7] Added tasks to change config file server to fixed registration address --- .gitignore | 5 +++-- inventory/sample/group_vars/rke2_servers.yml | 3 +++ roles/rke2_server/defaults/main.yml | 2 +- roles/rke2_server/tasks/fixed-registration.yml | 10 ++++++++++ roles/rke2_server/tasks/main.yml | 6 ++++++ roles/rke2_server/tasks/other_servers.yml | 11 +++++++++++ 6 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 roles/rke2_server/tasks/fixed-registration.yml diff --git a/.gitignore b/.gitignore index 782a0c73..2cde3bc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ *.retry .cache/ - +.vscode/ +files/ venv/ - +inventory/homelab/* test_inventory* rke2-images.linux-amd64.tar.gz diff --git a/inventory/sample/group_vars/rke2_servers.yml b/inventory/sample/group_vars/rke2_servers.yml index d451b625..61cbd654 100644 --- a/inventory/sample/group_vars/rke2_servers.yml +++ b/inventory/sample/group_vars/rke2_servers.yml @@ -33,6 +33,9 @@ rke2_config: {} # # write-kubeconfig-mode: "0640" +# See https://docs.rke2.io/install/ha +# Add a fixed registration address, such as a load balancer +# fixed_registration_address: 192.168.1.1 # See https://kubernetes.io/docs/tasks/debug-application-cluster/audit/ # Add a policy configuration file by specifying the file path on the control host diff --git a/roles/rke2_server/defaults/main.yml b/roles/rke2_server/defaults/main.yml index ae927959..173bdff8 100644 --- a/roles/rke2_server/defaults/main.yml +++ b/roles/rke2_server/defaults/main.yml @@ -1,2 +1,2 @@ --- -kubernetes_api_server_host: "{{ hostvars[groups['rke2_servers'][0]].inventory_hostname }}" +kubernetes_api_server_host: "{{ hostvars[groups['rke2_servers'][0]].inventory_hostname }}" \ No newline at end of file diff --git a/roles/rke2_server/tasks/fixed-registration.yml b/roles/rke2_server/tasks/fixed-registration.yml new file mode 100644 index 00000000..09f0cdd6 --- /dev/null +++ b/roles/rke2_server/tasks/fixed-registration.yml @@ -0,0 +1,10 @@ +--- +- name: Add fixed registration url to config file + ansible.builtin.lineinfile: + dest: /etc/rancher/rke2/config.yaml + line: "server: https://{{ fixed_registration_address }}:9345" + state: present + insertbefore: BOF + when: + - '"server:" not in server_url_check.stdout' + - fixed_registration_address is defined \ No newline at end of file diff --git a/roles/rke2_server/tasks/main.yml b/roles/rke2_server/tasks/main.yml index ef402d14..d73e4872 100644 --- a/roles/rke2_server/tasks/main.yml +++ b/roles/rke2_server/tasks/main.yml @@ -18,5 +18,11 @@ ansible.builtin.include_tasks: other_servers.yml when: inventory_hostname in groups['rke2_servers'][1:] +- name: Add first server to HA Setup + ansible.builtin.include_tasks: fixed-registration.yml + when: + - inventory_hostname in groups['rke2_servers'][0] + - fixed_registration_address is defined + - name: Configure Utilities ansible.builtin.include_tasks: utilities.yml diff --git a/roles/rke2_server/tasks/other_servers.yml b/roles/rke2_server/tasks/other_servers.yml index c075b058..466e7a95 100644 --- a/roles/rke2_server/tasks/other_servers.yml +++ b/roles/rke2_server/tasks/other_servers.yml @@ -29,6 +29,17 @@ insertbefore: BOF when: - '"server:" not in server_url_check.stdout' + - fixed_registration_address is undefined + +- name: Add fixed registration url to config file + ansible.builtin.lineinfile: + dest: /etc/rancher/rke2/config.yaml + line: "server: https://{{ fixed_registration_address }}:9345" + state: present + insertbefore: BOF + when: + - '"server:" not in server_url_check.stdout' + - fixed_registration_address is defined - name: Start rke2-server throttle: 1 From 66398dbb71b811c0407ce00e74d782a10b06db63 Mon Sep 17 00:00:00 2001 From: Eric-Domeier Date: Sat, 25 May 2024 18:11:55 -0400 Subject: [PATCH 2/7] Added tasks to allow a fixed registration address, fixed a CIS Hardening task to account for /opt install --- roles/rke2_agent/tasks/main.yml | 10 ++++++++++ roles/rke2_common/tasks/cis-hardening.yml | 19 ++++++++++++++++++- roles/rke2_server/tasks/other_servers.yml | 9 +++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/roles/rke2_agent/tasks/main.yml b/roles/rke2_agent/tasks/main.yml index 4d9cfdeb..2e7be2a3 100644 --- a/roles/rke2_agent/tasks/main.yml +++ b/roles/rke2_agent/tasks/main.yml @@ -37,6 +37,16 @@ when: - '"server:" not in server_url_check.stdout' +- name: Add fixed registration url to config file + ansible.builtin.lineinfile: + dest: /etc/rancher/rke2/config.yaml + line: "server: https://{{ fixed_registration_address }}:9345" + state: present + insertbefore: BOF + when: + - '"server:" not in server_url_check.stdout' + - fixed_registration_address is defined + - name: Start rke2-agent ansible.builtin.systemd: name: rke2-agent.service diff --git a/roles/rke2_common/tasks/cis-hardening.yml b/roles/rke2_common/tasks/cis-hardening.yml index 67a12bb6..9654e094 100644 --- a/roles/rke2_common/tasks/cis-hardening.yml +++ b/roles/rke2_common/tasks/cis-hardening.yml @@ -16,6 +16,11 @@ shell: /usr/sbin/nologin group: etcd create_home: false + + - name: "Check for rke2-cis-sysctl.conf in the /opt directory" + ansible.builtin.stat: + path: "/opt/rke2/share/rke2/rke2-cis-sysctl.conf" + register: rke2_cis_conf_in_opt - name: Copy systemctl file for kernel hardening for yum installs ansible.builtin.copy: @@ -28,6 +33,7 @@ - ansible_os_family == 'RedHat' or ansible_os_family == 'Rocky' - not rke2_binary_tarball_check.stat.exists - rke2_tarball_url is not defined or rke2_tarball_url == "" + - not rke2_cis_conf_in_opt.stat.exists - name: Copy systemctl file for kernel hardening for non-yum installs ansible.builtin.copy: @@ -37,16 +43,27 @@ mode: 0600 register: sysctl_operation_tarball when: >- + not rke2_cis_conf_in_opt.stat.exists and (ansible_facts['os_family'] != 'RedHat' and ansible_facts['os_family'] != 'Rocky') or rke2_binary_tarball_check.stat.exists or (rke2_tarball_url is defined and rke2_tarball_url != "") + - name: Copy systemctl file for kernel hardening for other + ansible.builtin.copy: + src: /opt/rke2/share/rke2/rke2-cis-sysctl.conf + dest: /etc/sysctl.d/60-rke2-cis.conf + remote_src: true + mode: 0600 + register: sysctl_operation_other + when: + - rke2_cis_conf_in_opt.stat.exists + - name: Restart systemd-sysctl ansible.builtin.service: state: restarted name: systemd-sysctl - when: sysctl_operation_yum.changed or sysctl_operation_tarball.changed + when: sysctl_operation_yum.changed or sysctl_operation_tarball.changed or sysctl_operation_other.changed # Per CIS hardening guide, if Kubernetes is already running, making changes to sysctl can result in unexpected # side-effects. Rebooting node if RKE2 is already running to prevent potential issues whereas before we were diff --git a/roles/rke2_server/tasks/other_servers.yml b/roles/rke2_server/tasks/other_servers.yml index 466e7a95..c6d26f8e 100644 --- a/roles/rke2_server/tasks/other_servers.yml +++ b/roles/rke2_server/tasks/other_servers.yml @@ -41,6 +41,15 @@ - '"server:" not in server_url_check.stdout' - fixed_registration_address is defined +- name: Wait for k8s apiserver reachability when use fixed registration address + ansible.builtin.wait_for: + host: "{{ fixed_registration_address }}" + port: "6443" + state: present + timeout: 300 + when: + - fixed_registration_address is defined + - name: Start rke2-server throttle: 1 ansible.builtin.systemd: From fa0899cdedcc342d4956c64ff8727a2b6c86a934 Mon Sep 17 00:00:00 2001 From: Eric-Domeier Date: Sun, 26 May 2024 00:16:24 -0400 Subject: [PATCH 3/7] Added handler to start rke2 service --- roles/rke2_server/handlers/main.yml | 13 +++++++++++++ roles/rke2_server/tasks/other_servers.yml | 18 +++--------------- 2 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 roles/rke2_server/handlers/main.yml diff --git a/roles/rke2_server/handlers/main.yml b/roles/rke2_server/handlers/main.yml new file mode 100644 index 00000000..4b8b3253 --- /dev/null +++ b/roles/rke2_server/handlers/main.yml @@ -0,0 +1,13 @@ +--- +- name: Ensure the RKE2 Service is started + block: + - name: Start RKE2 + ansible.builtin.systemd: + name: rke2-server + enabled: yes + register: rke2_service_start + retries: 20 + delay: 10 + until: rke2_service_start is succeeded + + \ No newline at end of file diff --git a/roles/rke2_server/tasks/other_servers.yml b/roles/rke2_server/tasks/other_servers.yml index c6d26f8e..b09345ad 100644 --- a/roles/rke2_server/tasks/other_servers.yml +++ b/roles/rke2_server/tasks/other_servers.yml @@ -40,22 +40,10 @@ when: - '"server:" not in server_url_check.stdout' - fixed_registration_address is defined + notify: Start RKE2 -- name: Wait for k8s apiserver reachability when use fixed registration address - ansible.builtin.wait_for: - host: "{{ fixed_registration_address }}" - port: "6443" - state: present - timeout: 300 - when: - - fixed_registration_address is defined - -- name: Start rke2-server - throttle: 1 - ansible.builtin.systemd: - name: rke2-server - state: started - enabled: yes +- name: Flush handlers + meta: flush_handlers - name: Wait for k8s apiserver reachability ansible.builtin.wait_for: From 299f973a4f6ef1ba71297327712d7d3f66cc94b4 Mon Sep 17 00:00:00 2001 From: Eric-Domeier Date: Tue, 28 May 2024 17:37:42 -0400 Subject: [PATCH 4/7] Increased retries/delay, fixed handlers --- roles/rke2_server/handlers/main.yml | 16 ++++++++++++++-- roles/rke2_server/tasks/other_servers.yml | 6 +++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/roles/rke2_server/handlers/main.yml b/roles/rke2_server/handlers/main.yml index 4b8b3253..b36ac062 100644 --- a/roles/rke2_server/handlers/main.yml +++ b/roles/rke2_server/handlers/main.yml @@ -1,13 +1,25 @@ --- - name: Ensure the RKE2 Service is started block: - - name: Start RKE2 + - name: Attempt to start RKE2 ansible.builtin.systemd: name: rke2-server enabled: yes + state: started register: rke2_service_start retries: 20 delay: 10 until: rke2_service_start is succeeded + listen: Start RKE2 - \ No newline at end of file + - name: Wait to ensure the service started correctly + ansible.builtin.pause: + seconds: 20 + listen: Start RKE2 + + - name: Verify rke2-server started + ansible.builtin.systemd: + name: rke2-server + state: started + failed_when: rke2_service_start is failed + listen: Start RKE2 \ No newline at end of file diff --git a/roles/rke2_server/tasks/other_servers.yml b/roles/rke2_server/tasks/other_servers.yml index b09345ad..ee372755 100644 --- a/roles/rke2_server/tasks/other_servers.yml +++ b/roles/rke2_server/tasks/other_servers.yml @@ -50,7 +50,7 @@ host: "{{ kubernetes_api_server_host }}" port: "6443" state: present - timeout: 300 + timeout: 600 - name: Wait for kubelet process to be present on host ansible.builtin.command: >- @@ -58,7 +58,7 @@ register: kubelet_check until: kubelet_check.rc == 0 retries: 20 - delay: 10 + delay: 30 changed_when: false - name: Extract the hostname-override parameter from the kubelet process @@ -75,5 +75,5 @@ register: status_result until: status_result.stdout.find("True") != -1 retries: 20 - delay: 10 + delay: 30 changed_when: false From fe7a534cf2347b79553c8c006a178e30ea87b610 Mon Sep 17 00:00:00 2001 From: Eric-Domeier Date: Tue, 28 May 2024 22:33:36 -0400 Subject: [PATCH 5/7] Changed adding fixed address with regexp, added create argument to .bashrc task for when it doesn't exist --- roles/rke2_server/tasks/fixed-registration.yml | 2 +- roles/rke2_server/tasks/main.yml | 2 +- roles/rke2_server/tasks/utilities.yml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/rke2_server/tasks/fixed-registration.yml b/roles/rke2_server/tasks/fixed-registration.yml index 09f0cdd6..7c46714d 100644 --- a/roles/rke2_server/tasks/fixed-registration.yml +++ b/roles/rke2_server/tasks/fixed-registration.yml @@ -5,6 +5,6 @@ line: "server: https://{{ fixed_registration_address }}:9345" state: present insertbefore: BOF + regexp: '^server:' when: - - '"server:" not in server_url_check.stdout' - fixed_registration_address is defined \ No newline at end of file diff --git a/roles/rke2_server/tasks/main.yml b/roles/rke2_server/tasks/main.yml index d73e4872..21be4b26 100644 --- a/roles/rke2_server/tasks/main.yml +++ b/roles/rke2_server/tasks/main.yml @@ -18,7 +18,7 @@ ansible.builtin.include_tasks: other_servers.yml when: inventory_hostname in groups['rke2_servers'][1:] -- name: Add first server to HA Setup +- name: Add first server to fixed registration address (High availability) ansible.builtin.include_tasks: fixed-registration.yml when: - inventory_hostname in groups['rke2_servers'][0] diff --git a/roles/rke2_server/tasks/utilities.yml b/roles/rke2_server/tasks/utilities.yml index a1a0bb70..226c5481 100644 --- a/roles/rke2_server/tasks/utilities.yml +++ b/roles/rke2_server/tasks/utilities.yml @@ -4,6 +4,7 @@ ansible.builtin.lineinfile: dest: "/root/.bashrc" line: 'PATH=$PATH:/var/lib/rancher/rke2/bin' + create: true insertafter: EOF - name: Symlink crictl config to /etc/crictl.yaml From 7379ca744ec19e4a5a72993b918b046a0a721e12 Mon Sep 17 00:00:00 2001 From: Eric-Domeier Date: Wed, 29 May 2024 23:33:23 -0400 Subject: [PATCH 6/7] Added tasks to account for possible /opt/rke2 directory install --- .../tasks/calculate_rke2_version.yml | 3 +-- roles/rke2_common/tasks/previous_install.yml | 25 +++++++++++++++++-- roles/rke2_common/tasks/tarball_install.yml | 4 +-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/roles/rke2_common/tasks/calculate_rke2_version.yml b/roles/rke2_common/tasks/calculate_rke2_version.yml index e18ae9c5..3aba7d80 100644 --- a/roles/rke2_common/tasks/calculate_rke2_version.yml +++ b/roles/rke2_common/tasks/calculate_rke2_version.yml @@ -24,8 +24,7 @@ - name: Set rke2_full_version fact ansible.builtin.set_fact: - rke2_full_version: "{{ rke2_full_version.stdout if ((install_rke2_version is not defined) or - (install_rke2_version | length == 0)) else install_rke2_version }}" + rke2_full_version: "{{ rke2_full_version.stdout if ((install_rke2_version is not defined) or (install_rke2_version | length == 0)) else install_rke2_version }}" - name: Set dot version ansible.builtin.shell: diff --git a/roles/rke2_common/tasks/previous_install.yml b/roles/rke2_common/tasks/previous_install.yml index ea1b9c3a..b24f6d3f 100644 --- a/roles/rke2_common/tasks/previous_install.yml +++ b/roles/rke2_common/tasks/previous_install.yml @@ -33,18 +33,39 @@ path: /usr/local/bin/rke2 register: rke2_binary +- name: Check for the rke2 binary + ansible.builtin.stat: + path: /opt/rke2/bin/rke2 + register: rke2_binary + when: not rke2_binary.stat.exists + - name: Get current RKE2 version if already installed ansible.builtin.shell: set -o pipefail && /usr/local/bin/rke2 -v | awk '$1 ~ /rke2/ { print $3 }' register: installed_rke2_version_tmp changed_when: false args: executable: /usr/bin/bash - when: rke2_binary.stat.exists + when: + - rke2_binary.stat.exists + - rke2_binary.stat.path == '/usr/local/bin/rke2' + failed_when: > + (installed_rke2_version_tmp.rc != 141) and + (installed_rke2_version_tmp.rc != 0) + +- name: Get current RKE2 version if already installed + ansible.builtin.shell: set -o pipefail && /opt/rke2/bin/rke2 -v | awk '$1 ~ /rke2/ { print $3 }' + register: installed_rke2_version_tmp + changed_when: false + args: + executable: /usr/bin/bash + when: + - rke2_binary.stat.exists + - rke2_binary.stat.path == '/opt/rke2/bin/rke2' failed_when: > (installed_rke2_version_tmp.rc != 141) and (installed_rke2_version_tmp.rc != 0) -- name: Determine if current version differs what what is being installed +- name: Determine if current version differs from what is being installed ansible.builtin.set_fact: installed_rke2_version: "{{ installed_rke2_version_tmp.stdout }}" when: rke2_binary.stat.exists diff --git a/roles/rke2_common/tasks/tarball_install.yml b/roles/rke2_common/tasks/tarball_install.yml index ca0d3f5f..9df5635f 100644 --- a/roles/rke2_common/tasks/tarball_install.yml +++ b/roles/rke2_common/tasks/tarball_install.yml @@ -37,7 +37,7 @@ - rke2_tarball_url != "" -- name: Determine if current version differs what what is being installed +- name: Determine if current version differs from what is being installed ansible.builtin.set_fact: rke2_version_changed: true when: @@ -82,7 +82,7 @@ ansible.builtin.set_fact: tarball_rke2_version: "{{ tarball_rke2_version_tmp.stdout }}" - - name: Determine if current version differs what what is being installed + - name: Determine if current version differs from what is being installed ansible.builtin.set_fact: rke2_version_changed: true when: From 15107fec0b1873e9abafa60bd89bba71f0ab1690 Mon Sep 17 00:00:00 2001 From: Eric-Domeier Date: Sun, 9 Jun 2024 22:09:11 -0400 Subject: [PATCH 7/7] Changed delay timer --- inventory/sample/group_vars/rke2_servers.yml | 4 ---- roles/rke2_agent/tasks/main.yml | 10 ---------- roles/rke2_server/tasks/fixed-registration.yml | 10 ---------- roles/rke2_server/tasks/main.yml | 6 ------ roles/rke2_server/tasks/other_servers.yml | 13 +------------ 5 files changed, 1 insertion(+), 42 deletions(-) delete mode 100644 roles/rke2_server/tasks/fixed-registration.yml diff --git a/inventory/sample/group_vars/rke2_servers.yml b/inventory/sample/group_vars/rke2_servers.yml index 61cbd654..4275d10a 100644 --- a/inventory/sample/group_vars/rke2_servers.yml +++ b/inventory/sample/group_vars/rke2_servers.yml @@ -33,10 +33,6 @@ rke2_config: {} # # write-kubeconfig-mode: "0640" -# See https://docs.rke2.io/install/ha -# Add a fixed registration address, such as a load balancer -# fixed_registration_address: 192.168.1.1 - # See https://kubernetes.io/docs/tasks/debug-application-cluster/audit/ # Add a policy configuration file by specifying the file path on the control host # audit_policy_config_file_path: "{{ playbook_dir }}/sample_files/audit-policy.yaml" diff --git a/roles/rke2_agent/tasks/main.yml b/roles/rke2_agent/tasks/main.yml index 2e7be2a3..4d9cfdeb 100644 --- a/roles/rke2_agent/tasks/main.yml +++ b/roles/rke2_agent/tasks/main.yml @@ -37,16 +37,6 @@ when: - '"server:" not in server_url_check.stdout' -- name: Add fixed registration url to config file - ansible.builtin.lineinfile: - dest: /etc/rancher/rke2/config.yaml - line: "server: https://{{ fixed_registration_address }}:9345" - state: present - insertbefore: BOF - when: - - '"server:" not in server_url_check.stdout' - - fixed_registration_address is defined - - name: Start rke2-agent ansible.builtin.systemd: name: rke2-agent.service diff --git a/roles/rke2_server/tasks/fixed-registration.yml b/roles/rke2_server/tasks/fixed-registration.yml deleted file mode 100644 index 7c46714d..00000000 --- a/roles/rke2_server/tasks/fixed-registration.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Add fixed registration url to config file - ansible.builtin.lineinfile: - dest: /etc/rancher/rke2/config.yaml - line: "server: https://{{ fixed_registration_address }}:9345" - state: present - insertbefore: BOF - regexp: '^server:' - when: - - fixed_registration_address is defined \ No newline at end of file diff --git a/roles/rke2_server/tasks/main.yml b/roles/rke2_server/tasks/main.yml index 21be4b26..ef402d14 100644 --- a/roles/rke2_server/tasks/main.yml +++ b/roles/rke2_server/tasks/main.yml @@ -18,11 +18,5 @@ ansible.builtin.include_tasks: other_servers.yml when: inventory_hostname in groups['rke2_servers'][1:] -- name: Add first server to fixed registration address (High availability) - ansible.builtin.include_tasks: fixed-registration.yml - when: - - inventory_hostname in groups['rke2_servers'][0] - - fixed_registration_address is defined - - name: Configure Utilities ansible.builtin.include_tasks: utilities.yml diff --git a/roles/rke2_server/tasks/other_servers.yml b/roles/rke2_server/tasks/other_servers.yml index ee372755..04248afb 100644 --- a/roles/rke2_server/tasks/other_servers.yml +++ b/roles/rke2_server/tasks/other_servers.yml @@ -29,17 +29,6 @@ insertbefore: BOF when: - '"server:" not in server_url_check.stdout' - - fixed_registration_address is undefined - -- name: Add fixed registration url to config file - ansible.builtin.lineinfile: - dest: /etc/rancher/rke2/config.yaml - line: "server: https://{{ fixed_registration_address }}:9345" - state: present - insertbefore: BOF - when: - - '"server:" not in server_url_check.stdout' - - fixed_registration_address is defined notify: Start RKE2 - name: Flush handlers @@ -74,6 +63,6 @@ -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' register: status_result until: status_result.stdout.find("True") != -1 - retries: 20 + retries: 60 delay: 30 changed_when: false