From 459fcaeba7d3498a46584abbdb29cbae551f63cb Mon Sep 17 00:00:00 2001 From: Alexander Olofsson Date: Tue, 11 Apr 2023 17:26:08 +0200 Subject: [PATCH 1/8] Add a wait online class to improve first-run By default, the first run may fail to apply some resources as the apiserver takes a few seconds before it opens its communication socket when started. This adds a simple exec that waits up to 30s for the apiserver to be ready before any kubectl_apply is allowed to occur. --- REFERENCE.md | 5 ++++ lib/puppet/type/kubectl_apply.rb | 5 +++- manifests/server.pp | 1 + manifests/server/apiserver.pp | 3 +- manifests/server/wait_online.pp | 15 ++++++++++ spec/classes/server/wait_online_spec.rb | 38 +++++++++++++++++++++++++ 6 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 manifests/server/wait_online.pp create mode 100644 spec/classes/server/wait_online_spec.rb diff --git a/REFERENCE.md b/REFERENCE.md index e2f33d8..53ea928 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -32,6 +32,7 @@ Uses the cni-plugins bridge binary to create a bridge interface to connect the c * [`k8s::server::resources::kube_proxy`](#k8s--server--resources--kube_proxy): Generates and deploys the default kube-proxy service for Kubernetes * [`k8s::server::scheduler`](#k8s--server--scheduler): Installs and configures a Kubernetes scheduler * [`k8s::server::tls`](#k8s--server--tls): Generates the necessary Kubernetes certificates for a server +* [`k8s::server::wait_online`](#k8s--server--wait_online): Creates a dummy exec to allow deferring applies until the Kubernetes API server has started ### Defined types @@ -2869,6 +2870,10 @@ Data type: `Stdlib::Unixpath` Default value: `$k8s::server::aggregator_ca_cert` +### `k8s::server::wait_online` + +Creates a dummy exec to allow deferring applies until the Kubernetes API server has started + ## Defined types ### `k8s::binary` diff --git a/lib/puppet/type/kubectl_apply.rb b/lib/puppet/type/kubectl_apply.rb index 028af3f..35e68ad 100644 --- a/lib/puppet/type/kubectl_apply.rb +++ b/lib/puppet/type/kubectl_apply.rb @@ -148,7 +148,10 @@ def retrieve [self[:kubeconfig]] end autorequire(:service) do - ['k8s-apiserver'] + ['kube-apiserver'] + end + autorequire(:exec) do + ['k8s-apiserver wait online'] end autorequire(:file) do [ diff --git a/manifests/server.pp b/manifests/server.pp index 394cb9b..a4edead 100644 --- a/manifests/server.pp +++ b/manifests/server.pp @@ -71,6 +71,7 @@ } if $manage_components { include k8s::server::apiserver + include k8s::server::wait_online # XXX Think of a better way to do this if $master == 'https://kubernetes:6443' { diff --git a/manifests/server/apiserver.pp b/manifests/server/apiserver.pp index c13bb96..14a97fc 100644 --- a/manifests/server/apiserver.pp +++ b/manifests/server/apiserver.pp @@ -250,6 +250,7 @@ }, }), } + # TODO: Create a dummy kube-apiserver service that just requires kubelet } else { $_sysconfig_path = pick($k8s::sysconfig_path, '/etc/sysconfig') file { "${_sysconfig_path}/kube-apiserver": @@ -286,7 +287,7 @@ subscribe => K8s::Binary['kube-apiserver'], } - Service['kube-apiserver'] -> Kubectl_apply<| |> ['kube-apiserver', 'front-proxy-client', 'apiserver-kubelet-client'].each |$cert| { + ['kube-apiserver', 'front-proxy-client', 'apiserver-kubelet-client'].each |$cert| { if defined(K8s::Server::Tls::Cert[$cert]) { K8s::Server::Tls::Cert[$cert] ~> Service['kube-apiserver'] } diff --git a/manifests/server/wait_online.pp b/manifests/server/wait_online.pp new file mode 100644 index 0000000..3ad88ae --- /dev/null +++ b/manifests/server/wait_online.pp @@ -0,0 +1,15 @@ +# @summary Creates a dummy exec to allow deferring applies until the Kubernetes API server has started +class k8s::server::wait_online { + # Wait up to 30 seconds for kube-apiserver to start + exec { 'k8s-apiserver wait online': + command => 'kubectl version', + path => $facts['path'], + tries => 15, + try_sleep => 2, + } + + # Require possibly managed components before checking online state + Kubeconfig <| title == '/root/.kube/config' |> -> Exec['k8s-apiserver wait online'] + K8s::Binary <| title == 'kubectl' |> -> Exec['k8s-apiserver wait online'] + Service <| title == 'kube-apiserver' |> -> Exec['k8s-apiserver wait online'] +} diff --git a/spec/classes/server/wait_online_spec.rb b/spec/classes/server/wait_online_spec.rb new file mode 100644 index 0000000..36cf68b --- /dev/null +++ b/spec/classes/server/wait_online_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 'k8s::server::wait_online' do + let(:pre_condition) do + <<~PUPPET + function assert_private() {} + + include ::k8s + class { '::k8s::server': + manage_etcd => false, + manage_certs => true, + manage_components => false, + manage_resources => false, + node_on_server => false, + } + class { '::k8s::server::apiserver': + etcd_servers => [], + } + PUPPET + end + + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + it { is_expected.to compile } + + it do + is_expected.to contain_exec('k8s-apiserver wait online'). + that_requires('Kubeconfig[/root/.kube/config]'). + that_requires('K8s::Binary[kubectl]'). + that_requires('Service[kube-apiserver]') + end + end + end +end From 65cfb8c33b09f6bff4502c7966e81618708dbcd5 Mon Sep 17 00:00:00 2001 From: Alexander Olofsson Date: Wed, 12 Apr 2023 13:57:23 +0200 Subject: [PATCH 2/8] Switch to a refreshonly exec for wait online --- manifests/server/wait_online.pp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/manifests/server/wait_online.pp b/manifests/server/wait_online.pp index 3ad88ae..ba72d23 100644 --- a/manifests/server/wait_online.pp +++ b/manifests/server/wait_online.pp @@ -2,14 +2,15 @@ class k8s::server::wait_online { # Wait up to 30 seconds for kube-apiserver to start exec { 'k8s-apiserver wait online': - command => 'kubectl version', - path => $facts['path'], - tries => 15, - try_sleep => 2, + command => 'kubectl version', + path => $facts['path'], + refreshonly => true, + tries => 15, + try_sleep => 2, } # Require possibly managed components before checking online state Kubeconfig <| title == '/root/.kube/config' |> -> Exec['k8s-apiserver wait online'] K8s::Binary <| title == 'kubectl' |> -> Exec['k8s-apiserver wait online'] - Service <| title == 'kube-apiserver' |> -> Exec['k8s-apiserver wait online'] + Service <| title == 'kube-apiserver' |> ~> Exec['k8s-apiserver wait online'] } From 4e61f20f7bda4c2a823b894020eee1fc4509a1d9 Mon Sep 17 00:00:00 2001 From: Alexander Olofsson Date: Wed, 12 Apr 2023 14:50:46 +0200 Subject: [PATCH 3/8] Require etcd before kube-apiserver - if managed --- manifests/server/apiserver.pp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manifests/server/apiserver.pp b/manifests/server/apiserver.pp index 14a97fc..b65ffd6 100644 --- a/manifests/server/apiserver.pp +++ b/manifests/server/apiserver.pp @@ -281,7 +281,8 @@ ], notify => Service['kube-apiserver'], } - service { 'kube-apiserver': + Service <| title == 'etcd' |> + -> service { 'kube-apiserver': ensure => stdlib::ensure($ensure, 'service'), enable => true, subscribe => K8s::Binary['kube-apiserver'], From 5d79abcca39fc621dfc727a547e762b8a18ec080 Mon Sep 17 00:00:00 2001 From: Alexander Olofsson Date: Wed, 12 Apr 2023 15:55:36 +0200 Subject: [PATCH 4/8] Give br_netfilter a moment to load before sysctls --- manifests/node/kubelet.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/manifests/node/kubelet.pp b/manifests/node/kubelet.pp index 9ac5d61..ff32af3 100644 --- a/manifests/node/kubelet.pp +++ b/manifests/node/kubelet.pp @@ -193,6 +193,13 @@ if $manage_kernel_modules { Kmod::Load['br_netfilter'] + ~> exec { 'Wait for br_netfilter to load before kubelet sysctls': + command => 'sysctl -aNr net.bridge | grep nf-call-iptables', + path => $facts['path'], + refreshonly => true, + tries => 5, + try_sleep => 1, + } -> [ Sysctl['net.bridge.bridge-nf-call-iptables'], Sysctl['net.bridge.bridge-nf-call-ip6tables'] From 2aa8c651c7766e044af31fda873726e57c805019 Mon Sep 17 00:00:00 2001 From: Alexander Olofsson Date: Wed, 12 Apr 2023 16:50:18 +0200 Subject: [PATCH 5/8] Hardcode kubeconfig path for wait-online Puppet doesn't run with the $HOME environment variable set, so kubectl won't check the path on its own --- manifests/server/wait_online.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/server/wait_online.pp b/manifests/server/wait_online.pp index ba72d23..4934ef0 100644 --- a/manifests/server/wait_online.pp +++ b/manifests/server/wait_online.pp @@ -2,7 +2,7 @@ class k8s::server::wait_online { # Wait up to 30 seconds for kube-apiserver to start exec { 'k8s-apiserver wait online': - command => 'kubectl version', + command => 'kubectl --kubeconfig /root/.kube/config version', path => $facts['path'], refreshonly => true, tries => 15, From 49118e1fe606af6927403bfc167aa434b33de588 Mon Sep 17 00:00:00 2001 From: Alexander Olofsson Date: Wed, 12 Apr 2023 18:41:36 +0200 Subject: [PATCH 6/8] Remove superfluous dash in exec name --- manifests/server/wait_online.pp | 8 ++++---- spec/classes/server/wait_online_spec.rb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/manifests/server/wait_online.pp b/manifests/server/wait_online.pp index 4934ef0..4cd2d1c 100644 --- a/manifests/server/wait_online.pp +++ b/manifests/server/wait_online.pp @@ -1,7 +1,7 @@ # @summary Creates a dummy exec to allow deferring applies until the Kubernetes API server has started class k8s::server::wait_online { # Wait up to 30 seconds for kube-apiserver to start - exec { 'k8s-apiserver wait online': + exec { 'k8s apiserver wait online': command => 'kubectl --kubeconfig /root/.kube/config version', path => $facts['path'], refreshonly => true, @@ -10,7 +10,7 @@ } # Require possibly managed components before checking online state - Kubeconfig <| title == '/root/.kube/config' |> -> Exec['k8s-apiserver wait online'] - K8s::Binary <| title == 'kubectl' |> -> Exec['k8s-apiserver wait online'] - Service <| title == 'kube-apiserver' |> ~> Exec['k8s-apiserver wait online'] + Kubeconfig <| title == '/root/.kube/config' |> -> Exec['k8s apiserver wait online'] + K8s::Binary <| title == 'kubectl' |> -> Exec['k8s apiserver wait online'] + Service <| title == 'kube-apiserver' |> ~> Exec['k8s apiserver wait online'] } diff --git a/spec/classes/server/wait_online_spec.rb b/spec/classes/server/wait_online_spec.rb index 36cf68b..5e6be7c 100644 --- a/spec/classes/server/wait_online_spec.rb +++ b/spec/classes/server/wait_online_spec.rb @@ -28,7 +28,7 @@ class { '::k8s::server::apiserver': it { is_expected.to compile } it do - is_expected.to contain_exec('k8s-apiserver wait online'). + is_expected.to contain_exec('k8s apiserver wait online'). that_requires('Kubeconfig[/root/.kube/config]'). that_requires('K8s::Binary[kubectl]'). that_requires('Service[kube-apiserver]') From 1e8ad6e2c44db3b192c08a6ea6ca17f9c7e54a65 Mon Sep 17 00:00:00 2001 From: Alexander Olofsson Date: Wed, 12 Apr 2023 18:51:15 +0200 Subject: [PATCH 7/8] Allow missing sysctl keys The augeas provider prefetches them before the module has a chance to load, so use the silent parameter instead. --- manifests/node/kubelet.pp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/manifests/node/kubelet.pp b/manifests/node/kubelet.pp index ff32af3..d69015b 100644 --- a/manifests/node/kubelet.pp +++ b/manifests/node/kubelet.pp @@ -181,6 +181,7 @@ sysctl { default: ensure => $ensure, + silent => true, value => '1'; 'net.bridge.bridge-nf-call-iptables': @@ -193,13 +194,6 @@ if $manage_kernel_modules { Kmod::Load['br_netfilter'] - ~> exec { 'Wait for br_netfilter to load before kubelet sysctls': - command => 'sysctl -aNr net.bridge | grep nf-call-iptables', - path => $facts['path'], - refreshonly => true, - tries => 5, - try_sleep => 1, - } -> [ Sysctl['net.bridge.bridge-nf-call-iptables'], Sysctl['net.bridge.bridge-nf-call-ip6tables'] From decd957b4689b5b5a9931019650d4fa3f7774d8e Mon Sep 17 00:00:00 2001 From: Alexander Olofsson Date: Wed, 12 Apr 2023 19:28:52 +0200 Subject: [PATCH 8/8] Fix incorrect wait online name for kubectl_apply --- lib/puppet/type/kubectl_apply.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/puppet/type/kubectl_apply.rb b/lib/puppet/type/kubectl_apply.rb index 35e68ad..6a9fc15 100644 --- a/lib/puppet/type/kubectl_apply.rb +++ b/lib/puppet/type/kubectl_apply.rb @@ -151,7 +151,7 @@ def retrieve ['kube-apiserver'] end autorequire(:exec) do - ['k8s-apiserver wait online'] + ['k8s apiserver wait online'] end autorequire(:file) do [