From e2158c0e594afae61451aa6a9285de2f74f293b7 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sat, 1 Feb 2020 16:19:08 -0500
Subject: [PATCH 01/29] fix k3s server token

---
 jobs/k3s-server/templates/bin/ctl.erb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 1cb1f873..6b21ec52 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -15,6 +15,7 @@ case $1 in
       
     exec  /var/vcap/packages/k3s/k3s server \
     -v <%= p('k3s.v') %> \
+    --token=<%= p('k3s.token') %> \
     --data-dir=/var/vcap/store/k3s-server \
     --default-local-storage-path=/var/vcap/store/k3s-server/local-storage-path \
     --private-registry=/var/vcap/jobs/k3s-server/config/registries.yaml \

From c8f769457a8fa74d4fe6a890c74c3fb00a7604b8 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sun, 2 Feb 2020 14:18:22 -0500
Subject: [PATCH 02/29] adapt agent for registries configuration

---
 jobs/k3s-agent/spec                           | 27 ++++++++++++++++++-
 jobs/k3s-agent/templates/bin/ctl.erb          |  1 +
 .../templates/config/registries.yaml.erb      | 14 ++++++++++
 .../templates/config/registry.ca.erb          |  1 +
 .../templates/config/registry.cert.erb        |  1 +
 .../templates/config/registry.key.erb         |  1 +
 6 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100644 jobs/k3s-agent/templates/config/registries.yaml.erb
 create mode 100644 jobs/k3s-agent/templates/config/registry.ca.erb
 create mode 100644 jobs/k3s-agent/templates/config/registry.cert.erb
 create mode 100644 jobs/k3s-agent/templates/config/registry.key.erb

diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec
index 9d70943c..e3c5d314 100644
--- a/jobs/k3s-agent/spec
+++ b/jobs/k3s-agent/spec
@@ -11,6 +11,10 @@ templates:
   bin/pre-stop.erb: bin/pre-stop
   bin/post-deploy.erb: bin/post-deploy
   bin/ctl.erb: bin/ctl
+  config/registries.yaml.erb: config/registries.yaml
+  config/registry.ca.erb: config/registry.ca
+  config/registry.cert.erb: config/registry.cert
+  config/registry.key.erb: config/registry.key 
 
 
 # Documentation https://bosh.io/docs/links.html
@@ -26,13 +30,34 @@ provides:
   type: k3s-agent
 
 properties:
+
+  registry.mirrors.endpoint:
+    description: private registry endpoint
+    default: http://localhost:5000
+
+  registry.mirrors.auth.username:
+    description: private registry user
+
+  registry.mirrors.auth.password:
+    description: private registry password
+
+  registry.mirrors.tls.cert:
+    description: private registry certificate
+
+  registry.mirrors.tls.key:
+    description: private registry private key
+
+  registry.mirrors.tls.ca:
+    description: private registry ca
+
+
   k3s.v:
     description: "(logging) Number for the log level verbosity (default: 0)"
     default: 0
   k3s.bind-address value:
     description: "(listener) k3s bind address (default: 0.0.0.0)"
     default: 0.0.0.0
-  k3s.flannel-backend:                    
+  k3s.flannel-backend:
     description: (networking) One of 'none', 'vxlan', 'ipsec', or 'wireguard'
     default: vxlan
   k3s.token:
diff --git a/jobs/k3s-agent/templates/bin/ctl.erb b/jobs/k3s-agent/templates/bin/ctl.erb
index ac113b12..adbb350c 100755
--- a/jobs/k3s-agent/templates/bin/ctl.erb
+++ b/jobs/k3s-agent/templates/bin/ctl.erb
@@ -19,6 +19,7 @@ case $1 in
     -v <%= p('k3s.v') %> \
     --token=<%= p('k3s.token') %> \
     --data-dir=/var/vcap/store/k3s-agent \
+    --default-local-storage-path=/var/vcap/store/k3s-agent/local-storage-path \
     --private-registry=/var/vcap/jobs/k3s-agent/config/registries.yaml \
       $servers \
       >>  $LOG_DIR/k3s-agent.stdout.log \
diff --git a/jobs/k3s-agent/templates/config/registries.yaml.erb b/jobs/k3s-agent/templates/config/registries.yaml.erb
new file mode 100644
index 00000000..04261af1
--- /dev/null
+++ b/jobs/k3s-agent/templates/config/registries.yaml.erb
@@ -0,0 +1,14 @@
+---
+mirrors:
+  docker.io:
+    endpoint:
+      - <%= p('registry.mirrors.endpoint') %>
+configs:
+  customreg:
+    auth:
+      username: <%= p('registry.mirrors.auth.username') %> # this is the registry username
+      password: <%= p('registry.mirrors.auth.password') %> # this is the registry password
+    tls:
+      cert_file: /var/vcap/jobs/k3s-server/config/registry.cert
+      key_file:  /var/vcap/jobs/k3s-server/config/registry.key
+      ca_file: /var/vcap/jobs/k3s-server/config/registry.ca
diff --git a/jobs/k3s-agent/templates/config/registry.ca.erb b/jobs/k3s-agent/templates/config/registry.ca.erb
new file mode 100644
index 00000000..f06e2b5d
--- /dev/null
+++ b/jobs/k3s-agent/templates/config/registry.ca.erb
@@ -0,0 +1 @@
+<%= p('registry.mirrors.tls.ca') %>
\ No newline at end of file
diff --git a/jobs/k3s-agent/templates/config/registry.cert.erb b/jobs/k3s-agent/templates/config/registry.cert.erb
new file mode 100644
index 00000000..9106d00c
--- /dev/null
+++ b/jobs/k3s-agent/templates/config/registry.cert.erb
@@ -0,0 +1 @@
+<%= p('registry.mirrors.tls.cert') %>
\ No newline at end of file
diff --git a/jobs/k3s-agent/templates/config/registry.key.erb b/jobs/k3s-agent/templates/config/registry.key.erb
new file mode 100644
index 00000000..55077003
--- /dev/null
+++ b/jobs/k3s-agent/templates/config/registry.key.erb
@@ -0,0 +1 @@
+<%= p('registry.mirrors.tls.key') %>
\ No newline at end of file

From c1ab90ad7b40d15fb291f22fd543fcf96417428f Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sun, 2 Feb 2020 15:44:36 -0500
Subject: [PATCH 03/29] default local storage not available on k3s worker nodes

---
 jobs/k3s-agent/templates/bin/ctl.erb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/jobs/k3s-agent/templates/bin/ctl.erb b/jobs/k3s-agent/templates/bin/ctl.erb
index adbb350c..ac113b12 100755
--- a/jobs/k3s-agent/templates/bin/ctl.erb
+++ b/jobs/k3s-agent/templates/bin/ctl.erb
@@ -19,7 +19,6 @@ case $1 in
     -v <%= p('k3s.v') %> \
     --token=<%= p('k3s.token') %> \
     --data-dir=/var/vcap/store/k3s-agent \
-    --default-local-storage-path=/var/vcap/store/k3s-agent/local-storage-path \
     --private-registry=/var/vcap/jobs/k3s-agent/config/registries.yaml \
       $servers \
       >>  $LOG_DIR/k3s-agent.stdout.log \

From bd522d89cf4d0b1137c9c1dbaec95d3f806c3f1e Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sun, 2 Feb 2020 15:56:54 -0500
Subject: [PATCH 04/29] add k9s cli in bosh ssh users context

---
 jobs/k3s-server/spec                         |  5 ++++-
 jobs/k3s-server/templates/bin/ctl.erb        |  3 +++
 jobs/k3s-server/templates/bin/envrc          |  5 +++++
 jobs/k3s-server/templates/bin/setup-user-env | 15 +++++++++++++++
 4 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 jobs/k3s-server/templates/bin/envrc
 create mode 100644 jobs/k3s-server/templates/bin/setup-user-env

diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index fd1b5712..9dd591ea 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -14,7 +14,10 @@ templates:
   config/registries.yaml.erb: config/registries.yaml
   config/registry.ca.erb: config/registry.ca
   config/registry.cert.erb: config/registry.cert
-  config/registry.key.erb: config/registry.key 
+  config/registry.key.erb: config/registry.key
+  bin/envrc: bin/envrc
+  bin/setup-user-env: bin/setup-user-env
+
 
 # Documentation https://bosh.io/docs/links.html
 # Tutorial      https://gist.github.com/Amit-PivotalLabs/c39528248b8cdc4ba8e347f8aa68abb6
diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 6b21ec52..45af06af 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -9,6 +9,9 @@ case $1 in
   start)
     mkdir -p $RUN_DIR $LOG_DIR
     chown -R vcap:vcap $RUN_DIR $LOG_DIR
+    
+    # Setup ssh env vars
+    ${JOB_DIR}/bin/setup-user-env
 
     echo $$ > $PIDFILE
 	
diff --git a/jobs/k3s-server/templates/bin/envrc b/jobs/k3s-server/templates/bin/envrc
new file mode 100644
index 00000000..097b4260
--- /dev/null
+++ b/jobs/k3s-server/templates/bin/envrc
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+export PATH=$PATH:/var/vcap/packages/k3s
+export PATH=$PATH:/var/vcap/packages/k9s
+export KUBECONFIG=/var/vcap/store/k3s-server/kubeconfig.yml
diff --git a/jobs/k3s-server/templates/bin/setup-user-env b/jobs/k3s-server/templates/bin/setup-user-env
new file mode 100644
index 00000000..b0e0948d
--- /dev/null
+++ b/jobs/k3s-server/templates/bin/setup-user-env
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+checkenv() {
+  trap "exit 0" INT TERM QUIT
+
+  command=". ${JOB_DIR}/bin/envrc"
+  for profile in /root/.profile /etc/skel/.profile; do
+    # toolbelt.auto is disabled; stripping envrc from .profiles
+    if ! grep -q "^${command}\$" ${profile} >/dev/null 2>&1; then
+      echo "${command}" >> ${profile}
+    fi
+  done
+}
+
+checkenv

From adaadc5e900eaf4219ebf844c68f64668230144b Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sun, 2 Feb 2020 18:45:13 -0500
Subject: [PATCH 05/29] add kubelet-arg support

---
 jobs/k3s-server/spec                  | 6 ++++++
 jobs/k3s-server/templates/bin/ctl.erb | 1 +
 2 files changed, 7 insertions(+)

diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index 9dd591ea..69b9ffce 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -72,6 +72,12 @@ properties:
     description: (agent/node) Registering kubelet with set of taints
     default: ""
 
+  k3s.kubelet-arg: 
+    description: (agent/flags) Customized flag for kubelet process
+    default: ""
+
+
+
 #done    
 #   -v value                                   (logging) Number for the log level verbosity (default: 0)
 #   --bind-address value                       (listener) k3s bind address (default: 0.0.0.0)
diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 45af06af..177d3dda 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -19,6 +19,7 @@ case $1 in
     exec  /var/vcap/packages/k3s/k3s server \
     -v <%= p('k3s.v') %> \
     --token=<%= p('k3s.token') %> \
+    --kubelet-arg=<%= p('k3s.kubelet-arg' %> \
     --data-dir=/var/vcap/store/k3s-server \
     --default-local-storage-path=/var/vcap/store/k3s-server/local-storage-path \
     --private-registry=/var/vcap/jobs/k3s-server/config/registries.yaml \

From 43574f85a8fe490c6faedd3e10242a6a9f45070b Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Mon, 3 Feb 2020 00:20:31 +0000
Subject: [PATCH 06/29] define k3s resolv.conf

---
 jobs/k3s-agent/templates/bin/ctl.erb  | 1 +
 jobs/k3s-server/templates/bin/ctl.erb | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/jobs/k3s-agent/templates/bin/ctl.erb b/jobs/k3s-agent/templates/bin/ctl.erb
index ac113b12..f4a3daa1 100755
--- a/jobs/k3s-agent/templates/bin/ctl.erb
+++ b/jobs/k3s-agent/templates/bin/ctl.erb
@@ -20,6 +20,7 @@ case $1 in
     --token=<%= p('k3s.token') %> \
     --data-dir=/var/vcap/store/k3s-agent \
     --private-registry=/var/vcap/jobs/k3s-agent/config/registries.yaml \
+    --resolv-conf=/etc/resolv.conf \
       $servers \
       >>  $LOG_DIR/k3s-agent.stdout.log \
       2>> $LOG_DIR/k3s-agent.stderr.log
diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 177d3dda..75a4ebb6 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -19,7 +19,7 @@ case $1 in
     exec  /var/vcap/packages/k3s/k3s server \
     -v <%= p('k3s.v') %> \
     --token=<%= p('k3s.token') %> \
-    --kubelet-arg=<%= p('k3s.kubelet-arg' %> \
+    --resolv-conf=/etc/resolv.conf \
     --data-dir=/var/vcap/store/k3s-server \
     --default-local-storage-path=/var/vcap/store/k3s-server/local-storage-path \
     --private-registry=/var/vcap/jobs/k3s-server/config/registries.yaml \

From 78e55c67c2984f72c4e9215097053eb6cbac7839 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Thu, 6 Feb 2020 07:05:46 -0500
Subject: [PATCH 07/29] additional infos in README.md

---
 README.md | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ff7ff364..6bc06b29 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,23 @@
 # BOSH release for k3s
 
-This BOSH release and deployment manifest deploy a cluster of k3s.
+This BOSH release and deployment manifest deploy a cluster of k3s
+
+Lightweight Kubernetes. 5 less than k8s. https://k3s.io.
+
+
+## design overview
+
+Provide a lightweight bosh packaging of Rancher k3s kubernetes distribution
+
+Includes
+- Rancher k3s binary
+- k9S binary
+
+
+The bosh release offers 2 jobs to build a full k3s bosh deployment:
+- k3s-server job.
+- k3s-agent job.
+
 
 ## Usage
 

From 9916f46c069ed439260e6138da0b5a8fc57231bc Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sat, 8 Feb 2020 13:00:27 +0000
Subject: [PATCH 08/29] use kubelet arg hostname-override to set ip as hostname

---
 jobs/k3s-agent/templates/bin/ctl.erb  | 1 +
 jobs/k3s-server/templates/bin/ctl.erb | 1 +
 2 files changed, 2 insertions(+)

diff --git a/jobs/k3s-agent/templates/bin/ctl.erb b/jobs/k3s-agent/templates/bin/ctl.erb
index f4a3daa1..e3f48b45 100755
--- a/jobs/k3s-agent/templates/bin/ctl.erb
+++ b/jobs/k3s-agent/templates/bin/ctl.erb
@@ -21,6 +21,7 @@ case $1 in
     --data-dir=/var/vcap/store/k3s-agent \
     --private-registry=/var/vcap/jobs/k3s-agent/config/registries.yaml \
     --resolv-conf=/etc/resolv.conf \
+    --kubelet-arg="--hostname-override=<%= spec.ip %>
       $servers \
       >>  $LOG_DIR/k3s-agent.stdout.log \
       2>> $LOG_DIR/k3s-agent.stderr.log
diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 75a4ebb6..c8271049 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -25,6 +25,7 @@ case $1 in
     --private-registry=/var/vcap/jobs/k3s-server/config/registries.yaml \
     --write-kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml \
     --write-kubeconfig-mode=755 \
+    --kubelet-arg="--hostname-override=<%= spec.ip %>"
       >>  $LOG_DIR/k3s-server.stdout.log \
       2>> $LOG_DIR/k3s-server.stderr.log
 

From 69f3ae94dad5e61dd926c69083b7d4d5bf4b0d23 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Tue, 4 Feb 2020 22:53:13 +0000
Subject: [PATCH 09/29] upload blobs

---
 config/blobs.yml                      | 3 +++
 jobs/k3s-server/templates/bin/ctl.erb | 5 ++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/config/blobs.yml b/config/blobs.yml
index 310fc6e7..3cbc596e 100644
--- a/config/blobs.yml
+++ b/config/blobs.yml
@@ -1,9 +1,12 @@
 k3s-images/k3s-airgap-images-amd64.tar:
   size: 349377024
+  object_id: 5d8b906c-3af4-40fe-7133-969761fd335d
   sha: sha256:e1d140575deb3043cb0102d1ae510ab3273875a275662e1ca84a89c955471189
 k3s/k3s:
   size: 52273152
+  object_id: 950df9db-5901-4b4c-6d31-30ac5f5d5428
   sha: sha256:b283533e78a2bdec3b6b20b92729b3618221e1e89321dbce73ac1fb466ea4f1a
 k9s/k9s:
   size: 42287104
+  object_id: 803cae2b-1a1e-401e-432f-8b00a007920f
   sha: sha256:241b52c6373d46bb4fc076c5f656618fd0cbb728823030880e0d89d840278050
diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index c8271049..694fedcb 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -1,5 +1,6 @@
 #!/bin/bash
 
+JOB_DIR=/var/vcap/jobs/k3s-server
 RUN_DIR=/var/vcap/sys/run/k3s-server
 LOG_DIR=/var/vcap/sys/log/k3s-server
 PIDFILE=${RUN_DIR}/pid
@@ -25,10 +26,12 @@ case $1 in
     --private-registry=/var/vcap/jobs/k3s-server/config/registries.yaml \
     --write-kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml \
     --write-kubeconfig-mode=755 \
-    --kubelet-arg="--hostname-override=<%= spec.ip %>"
       >>  $LOG_DIR/k3s-server.stdout.log \
       2>> $LOG_DIR/k3s-server.stderr.log
 
+
+    #TODO --kubelet-arg=" --hostname-override=<%= spec.ip %>" \
+
     ;;
 
   stop)

From a2e58bc8f9c6ca36beebc81a146586caf8fe8dbd Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sat, 17 Oct 2020 09:28:54 +0000
Subject: [PATCH 10/29] upload blobs for k3s 1.19

---
 addblob.sh                           | 11 ++++++-----
 config/blobs.yml                     | 18 +++++++++---------
 jobs/k3s-agent/templates/bin/ctl.erb |  1 -
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/addblob.sh b/addblob.sh
index 4cbe8aca..9cf748e2 100755
--- a/addblob.sh
+++ b/addblob.sh
@@ -1,13 +1,14 @@
 #!/bin/bash
-wget https://github.com/rancher/k3s/releases/download/v1.17.2-alpha3%2Bk3s1/k3s -O src/k3s
+wget https://github.com/rancher/k3s/releases/download/v1.19.3%2Bk3s1/k3s -O src/k3s
 bosh add-blob src/k3s k3s/k3s
 
-wget https://github.com/rancher/k3s/releases/download/v1.17.2-alpha3%2Bk3s1/k3s-airgap-images-amd64.tar -O src/k3s-airgap-images-amd64.tar
+wget https://github.com/rancher/k3s/releases/download/v1.19.3%2Bk3s1/k3s-airgap-images-amd64.tar -O src/k3s-airgap-images-amd64.tar
 bosh add-blob src/k3s-airgap-images-amd64.tar k3s-images/k3s-airgap-images-amd64.tar
 
-wget https://github.com/derailed/k9s/releases/download/v0.13.6/k9s_0.13.6_Linux_x86_64.tar.gz -O src/k9s_0.13.6_Linux_x86_64.tar.gz
+wget https://github.com/derailed/k9s/releases/download/v0.22.1/k9s_Linux_x86_64.tar.gz -O src/k9s_Linux_x86_64.tar.gz
 cd src
-tar xfv k9s_0.13.6_Linux_x86_64.tar.gz
+tar xfv ./k9s_Linux_x86_64.tar.gz
 cd ..
 bosh add-blob src/k9s k9s/k9s
-# rm src/k9s_0.13.6_Linux_x86_64.tar.gz
+
+
diff --git a/config/blobs.yml b/config/blobs.yml
index 3cbc596e..5bceda00 100644
--- a/config/blobs.yml
+++ b/config/blobs.yml
@@ -1,12 +1,12 @@
 k3s-images/k3s-airgap-images-amd64.tar:
-  size: 349377024
-  object_id: 5d8b906c-3af4-40fe-7133-969761fd335d
-  sha: sha256:e1d140575deb3043cb0102d1ae510ab3273875a275662e1ca84a89c955471189
+  size: 369606656
+  object_id: 172301a2-56a0-4496-7b48-aa79d399a3e8
+  sha: sha256:9d3013742f011bd08225344ae5fd85407bd2bcd88886261ba58e1bff5b00f08d
 k3s/k3s:
-  size: 52273152
-  object_id: 950df9db-5901-4b4c-6d31-30ac5f5d5428
-  sha: sha256:b283533e78a2bdec3b6b20b92729b3618221e1e89321dbce73ac1fb466ea4f1a
+  size: 53424128
+  object_id: d928a091-fd50-41a4-5b18-d61bfa41fe92
+  sha: sha256:3b031d78f9edeed6718b5cd1070e4bd46524faa90a82d64f6f28008f6192c5dc
 k9s/k9s:
-  size: 42287104
-  object_id: 803cae2b-1a1e-401e-432f-8b00a007920f
-  sha: sha256:241b52c6373d46bb4fc076c5f656618fd0cbb728823030880e0d89d840278050
+  size: 49139712
+  object_id: ce7272d4-d76c-4d67-68a2-4ccd59366344
+  sha: sha256:04b457ba48879f64ec97ca513ad573328939b2c9b3591cb1171d7bd989e3eaef
diff --git a/jobs/k3s-agent/templates/bin/ctl.erb b/jobs/k3s-agent/templates/bin/ctl.erb
index e3f48b45..f4a3daa1 100755
--- a/jobs/k3s-agent/templates/bin/ctl.erb
+++ b/jobs/k3s-agent/templates/bin/ctl.erb
@@ -21,7 +21,6 @@ case $1 in
     --data-dir=/var/vcap/store/k3s-agent \
     --private-registry=/var/vcap/jobs/k3s-agent/config/registries.yaml \
     --resolv-conf=/etc/resolv.conf \
-    --kubelet-arg="--hostname-override=<%= spec.ip %>
       $servers \
       >>  $LOG_DIR/k3s-agent.stdout.log \
       2>> $LOG_DIR/k3s-agent.stderr.log

From 4316991c9e644e354eca93cb777fc95433a2fd19 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sat, 17 Oct 2020 10:06:23 -0400
Subject: [PATCH 11/29] add k3s-killall.sh, to use with bosh drain

---
 jobs/k3s-agent/spec                          |  4 +
 jobs/k3s-agent/templates/bin/k3s-killall.sh  | 77 ++++++++++++++++++++
 jobs/k3s-server/spec                         |  1 +
 jobs/k3s-server/templates/bin/k3s-killall.sh | 77 ++++++++++++++++++++
 4 files changed, 159 insertions(+)
 create mode 100755 jobs/k3s-agent/templates/bin/k3s-killall.sh
 create mode 100755 jobs/k3s-server/templates/bin/k3s-killall.sh

diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec
index e3c5d314..6403c10e 100644
--- a/jobs/k3s-agent/spec
+++ b/jobs/k3s-agent/spec
@@ -11,6 +11,10 @@ templates:
   bin/pre-stop.erb: bin/pre-stop
   bin/post-deploy.erb: bin/post-deploy
   bin/ctl.erb: bin/ctl
+  bin/k3s-killall.sh: bin/k3s-killall.sh
+
+
+
   config/registries.yaml.erb: config/registries.yaml
   config/registry.ca.erb: config/registry.ca
   config/registry.cert.erb: config/registry.cert
diff --git a/jobs/k3s-agent/templates/bin/k3s-killall.sh b/jobs/k3s-agent/templates/bin/k3s-killall.sh
new file mode 100755
index 00000000..50543fea
--- /dev/null
+++ b/jobs/k3s-agent/templates/bin/k3s-killall.sh
@@ -0,0 +1,77 @@
+#!/bin/sh
+[ $(id -u) -eq 0 ] || exec sudo $0 $@
+
+for bin in /var/lib/rancher/k3s/data/**/bin/; do
+    [ -d $bin ] && export PATH=$bin:$PATH
+done
+
+set -x
+
+for service in /etc/systemd/system/k3s*.service; do
+    [ -s $service ] && systemctl stop $(basename $service)
+done
+
+for service in /etc/init.d/k3s*; do
+    [ -x $service ] && $service stop
+done
+
+pschildren() {
+    ps -e -o ppid= -o pid= | \
+    sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \
+    grep -w "^$1" | \
+    cut -f2
+}
+
+pstree() {
+    for pid in $@; do
+        echo $pid
+        for child in $(pschildren $pid); do
+            pstree $child
+        done
+    done
+}
+
+killtree() {
+    kill -9 $(
+        { set +x; } 2>/dev/null;
+        pstree $@;
+        set -x;
+    ) 2>/dev/null
+}
+
+getshims() {
+    lsof | sed -e 's/^[^0-9]*//g; s/  */\t/g' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 | sort -n -u
+}
+
+killtree $({ set +x; } 2>/dev/null; getshims; set -x)
+
+do_unmount() {
+    { set +x; } 2>/dev/null
+    MOUNTS=
+    while read ignore mount ignore; do
+        MOUNTS="$mount\n$MOUNTS"
+    done </proc/self/mounts
+    MOUNTS=$(printf $MOUNTS | grep "^$1" | sort -r)
+    if [ -n "${MOUNTS}" ]; then
+        set -x
+        umount ${MOUNTS}
+    else
+        set -x
+    fi
+}
+
+do_unmount '/run/k3s'
+do_unmount '/var/lib/rancher/k3s'
+do_unmount '/var/lib/kubelet/pods'
+do_unmount '/run/netns/cni-'
+
+# Delete network interface(s) that match 'master cni0'
+ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do
+    iface=${iface%%@*}
+    [ -z "$iface" ] || ip link delete $iface
+done
+ip link delete cni0
+ip link delete flannel.1
+rm -rf /var/lib/cni/
+iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
+
diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index 69b9ffce..5deaec23 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -17,6 +17,7 @@ templates:
   config/registry.key.erb: config/registry.key
   bin/envrc: bin/envrc
   bin/setup-user-env: bin/setup-user-env
+  bin/k3s-killall.sh: bin/k3s-killall.sh
 
 
 # Documentation https://bosh.io/docs/links.html
diff --git a/jobs/k3s-server/templates/bin/k3s-killall.sh b/jobs/k3s-server/templates/bin/k3s-killall.sh
new file mode 100755
index 00000000..50543fea
--- /dev/null
+++ b/jobs/k3s-server/templates/bin/k3s-killall.sh
@@ -0,0 +1,77 @@
+#!/bin/sh
+[ $(id -u) -eq 0 ] || exec sudo $0 $@
+
+for bin in /var/lib/rancher/k3s/data/**/bin/; do
+    [ -d $bin ] && export PATH=$bin:$PATH
+done
+
+set -x
+
+for service in /etc/systemd/system/k3s*.service; do
+    [ -s $service ] && systemctl stop $(basename $service)
+done
+
+for service in /etc/init.d/k3s*; do
+    [ -x $service ] && $service stop
+done
+
+pschildren() {
+    ps -e -o ppid= -o pid= | \
+    sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \
+    grep -w "^$1" | \
+    cut -f2
+}
+
+pstree() {
+    for pid in $@; do
+        echo $pid
+        for child in $(pschildren $pid); do
+            pstree $child
+        done
+    done
+}
+
+killtree() {
+    kill -9 $(
+        { set +x; } 2>/dev/null;
+        pstree $@;
+        set -x;
+    ) 2>/dev/null
+}
+
+getshims() {
+    lsof | sed -e 's/^[^0-9]*//g; s/  */\t/g' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 | sort -n -u
+}
+
+killtree $({ set +x; } 2>/dev/null; getshims; set -x)
+
+do_unmount() {
+    { set +x; } 2>/dev/null
+    MOUNTS=
+    while read ignore mount ignore; do
+        MOUNTS="$mount\n$MOUNTS"
+    done </proc/self/mounts
+    MOUNTS=$(printf $MOUNTS | grep "^$1" | sort -r)
+    if [ -n "${MOUNTS}" ]; then
+        set -x
+        umount ${MOUNTS}
+    else
+        set -x
+    fi
+}
+
+do_unmount '/run/k3s'
+do_unmount '/var/lib/rancher/k3s'
+do_unmount '/var/lib/kubelet/pods'
+do_unmount '/run/netns/cni-'
+
+# Delete network interface(s) that match 'master cni0'
+ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do
+    iface=${iface%%@*}
+    [ -z "$iface" ] || ip link delete $iface
+done
+ip link delete cni0
+ip link delete flannel.1
+rm -rf /var/lib/cni/
+iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
+

From 5caf45e316dfef99a26325d67b1c8a7a6a77131d Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sat, 17 Oct 2020 10:13:41 -0400
Subject: [PATCH 12/29] add drain script

---
 jobs/k3s-agent/spec                                          | 1 +
 jobs/k3s-agent/templates/bin/drain.erb                       | 0
 jobs/k3s-server/spec                                         | 3 ++-
 jobs/k3s-server/templates/bin/drain.erb                      | 0
 jobs/k3s-server/templates/bin/envrc                          | 2 ++
 jobs/k3s-server/templates/bin/pre-start.erb                  | 5 ++---
 .../templates/bin/{setup-user-env => setup-user-env.erb}     | 2 +-
 7 files changed, 8 insertions(+), 5 deletions(-)
 create mode 100644 jobs/k3s-agent/templates/bin/drain.erb
 create mode 100644 jobs/k3s-server/templates/bin/drain.erb
 rename jobs/k3s-server/templates/bin/{setup-user-env => setup-user-env.erb} (97%)

diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec
index 6403c10e..7f609310 100644
--- a/jobs/k3s-agent/spec
+++ b/jobs/k3s-agent/spec
@@ -10,6 +10,7 @@ templates:
   bin/post-start.erb: bin/post-start  
   bin/pre-stop.erb: bin/pre-stop
   bin/post-deploy.erb: bin/post-deploy
+  bin/drain.erb: bin/drain
   bin/ctl.erb: bin/ctl
   bin/k3s-killall.sh: bin/k3s-killall.sh
 
diff --git a/jobs/k3s-agent/templates/bin/drain.erb b/jobs/k3s-agent/templates/bin/drain.erb
new file mode 100644
index 00000000..e69de29b
diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index 5deaec23..0e8c9c8b 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -10,13 +10,14 @@ templates:
   bin/post-start.erb: bin/post-start  
   bin/pre-stop.erb: bin/pre-stop
   bin/post-deploy.erb: bin/post-deploy
+  bin/drain.erb: bin/drain
   bin/ctl.erb: bin/ctl
   config/registries.yaml.erb: config/registries.yaml
   config/registry.ca.erb: config/registry.ca
   config/registry.cert.erb: config/registry.cert
   config/registry.key.erb: config/registry.key
   bin/envrc: bin/envrc
-  bin/setup-user-env: bin/setup-user-env
+  bin/setup-user-env.erb: bin/setup-user-env
   bin/k3s-killall.sh: bin/k3s-killall.sh
 
 
diff --git a/jobs/k3s-server/templates/bin/drain.erb b/jobs/k3s-server/templates/bin/drain.erb
new file mode 100644
index 00000000..e69de29b
diff --git a/jobs/k3s-server/templates/bin/envrc b/jobs/k3s-server/templates/bin/envrc
index 097b4260..71f07953 100644
--- a/jobs/k3s-server/templates/bin/envrc
+++ b/jobs/k3s-server/templates/bin/envrc
@@ -2,4 +2,6 @@
 
 export PATH=$PATH:/var/vcap/packages/k3s
 export PATH=$PATH:/var/vcap/packages/k9s
+export PATH=$PATH:/var/vcap/packages/kubectl
+
 export KUBECONFIG=/var/vcap/store/k3s-server/kubeconfig.yml
diff --git a/jobs/k3s-server/templates/bin/pre-start.erb b/jobs/k3s-server/templates/bin/pre-start.erb
index c0180c83..a85cc0c5 100644
--- a/jobs/k3s-server/templates/bin/pre-start.erb
+++ b/jobs/k3s-server/templates/bin/pre-start.erb
@@ -1,8 +1,7 @@
 #!/bin/sh
 /var/vcap/packages/k3s/k3s check-config
 
-#bpm pre start recommandation
-#sysctl -e -w net.ipv4.tcp_fin_timeout=10
-#sysctl -e -w net.ipv4.tcp_tw_reuse=1
+# Setup ssh env vars
+${JOB_DIR}/bin/setup-user-env
 
 exit 0
diff --git a/jobs/k3s-server/templates/bin/setup-user-env b/jobs/k3s-server/templates/bin/setup-user-env.erb
similarity index 97%
rename from jobs/k3s-server/templates/bin/setup-user-env
rename to jobs/k3s-server/templates/bin/setup-user-env.erb
index b0e0948d..d6ac5f81 100644
--- a/jobs/k3s-server/templates/bin/setup-user-env
+++ b/jobs/k3s-server/templates/bin/setup-user-env.erb
@@ -12,4 +12,4 @@ checkenv() {
   done
 }
 
-checkenv
+checkenv
\ No newline at end of file

From ef7d88be997ce9cab5928e574fe3d3b538334b5b Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sun, 18 Oct 2020 21:29:55 +0000
Subject: [PATCH 13/29] add kubectl package

---
 addblob.sh                     |  4 ++++
 config/blobs.yml               |  3 +++
 jobs/k3s-agent/spec            |  1 +
 jobs/k3s-server/spec           |  1 +
 packages/kubectl-k3s/packaging | 12 ++++++++++++
 packages/kubectl-k3s/spec      |  5 +++++
 6 files changed, 26 insertions(+)
 create mode 100644 packages/kubectl-k3s/packaging
 create mode 100644 packages/kubectl-k3s/spec

diff --git a/addblob.sh b/addblob.sh
index 9cf748e2..8dc06074 100755
--- a/addblob.sh
+++ b/addblob.sh
@@ -11,4 +11,8 @@ tar xfv ./k9s_Linux_x86_64.tar.gz
 cd ..
 bosh add-blob src/k9s k9s/k9s
 
+curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl -O src/kubectl
+
+bosh add-blob src/kubectl kubectl/kubectl
+
 
diff --git a/config/blobs.yml b/config/blobs.yml
index 5bceda00..b2e5e529 100644
--- a/config/blobs.yml
+++ b/config/blobs.yml
@@ -10,3 +10,6 @@ k9s/k9s:
   size: 49139712
   object_id: ce7272d4-d76c-4d67-68a2-4ccd59366344
   sha: sha256:04b457ba48879f64ec97ca513ad573328939b2c9b3591cb1171d7bd989e3eaef
+kubectl/kubectl:
+  size: 43003904
+  sha: sha256:79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f
diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec
index 7f609310..956fe171 100644
--- a/jobs/k3s-agent/spec
+++ b/jobs/k3s-agent/spec
@@ -3,6 +3,7 @@ name: k3s-agent
 
 packages:
 - k3s
+- kubectl-k3s
 - k3s-images
 templates:
   config/bpm.yml: config/bpm.yml
diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index 0e8c9c8b..ed31453d 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -4,6 +4,7 @@ name: k3s-server
 packages:
 - k3s
 - k9s
+- kubectl-k3s
 templates:
   config/bpm.yml: config/bpm.yml
   bin/pre-start.erb: bin/pre-start
diff --git a/packages/kubectl-k3s/packaging b/packages/kubectl-k3s/packaging
new file mode 100644
index 00000000..da51de43
--- /dev/null
+++ b/packages/kubectl-k3s/packaging
@@ -0,0 +1,12 @@
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+# Detect # of CPUs so make jobs can be parallelized
+CPUS=$(grep -c ^processor /proc/cpuinfo)
+ # Available variables
+# $BOSH_COMPILE_TARGET - where this package & spec'd source files are available
+# $BOSH_INSTALL_TARGET - where you copy/install files to be included in package
+export HOME=/var/vcap
+cp ${BOSH_COMPILE_TARGET}/kubectl/kubectl ${BOSH_INSTALL_TARGET}/kubectl
+chmod ugo+x ${BOSH_INSTALL_TARGET}/kubectl
+
diff --git a/packages/kubectl-k3s/spec b/packages/kubectl-k3s/spec
new file mode 100644
index 00000000..2af0713d
--- /dev/null
+++ b/packages/kubectl-k3s/spec
@@ -0,0 +1,5 @@
+---
+name: kubectl-k3s
+dependencies: []
+files:
+- kubectl/kubectl

From e16d3a446f48e90e5447bfd35c5058457aebfff5 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sun, 18 Oct 2020 21:31:28 +0000
Subject: [PATCH 14/29] uploaded kubectl blob

---
 config/blobs.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/blobs.yml b/config/blobs.yml
index b2e5e529..3c02a9a2 100644
--- a/config/blobs.yml
+++ b/config/blobs.yml
@@ -12,4 +12,5 @@ k9s/k9s:
   sha: sha256:04b457ba48879f64ec97ca513ad573328939b2c9b3591cb1171d7bd989e3eaef
 kubectl/kubectl:
   size: 43003904
+  object_id: bb931f61-9bd1-4cca-71a0-bf1139913a72
   sha: sha256:79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f

From ddcdb06b96a0f57b2c0bd6f121bc7349cc8f6781 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Sun, 18 Oct 2020 22:15:16 +0000
Subject: [PATCH 15/29] use spec.ip as k3s node name

---
 jobs/k3s-agent/templates/bin/ctl.erb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/jobs/k3s-agent/templates/bin/ctl.erb b/jobs/k3s-agent/templates/bin/ctl.erb
index f4a3daa1..7bf5af6d 100755
--- a/jobs/k3s-agent/templates/bin/ctl.erb
+++ b/jobs/k3s-agent/templates/bin/ctl.erb
@@ -14,7 +14,9 @@ case $1 in
 	
 	export servers="<% masters = link('k3s-server') %><% masters.instances.each do |instance| %> --server=https://<%= instance.address %>:6443 <% end %>"
 	
-	
+
+	export K3S_NODE_NAME=<%= spec.ip %>
+
     exec  /var/vcap/packages/k3s/k3s agent \
     -v <%= p('k3s.v') %> \
     --token=<%= p('k3s.token') %> \

From d6f61f0757eb099979c13ab0bd8444c55e9edbdd Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Mon, 19 Oct 2020 04:52:29 -0400
Subject: [PATCH 16/29] map bosh drain to k8s drain

---
 jobs/k3s-agent/templates/bin/drain.erb  | 6 ++++++
 jobs/k3s-server/templates/bin/ctl.erb   | 4 ----
 jobs/k3s-server/templates/bin/drain.erb | 6 ++++++
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/jobs/k3s-agent/templates/bin/drain.erb b/jobs/k3s-agent/templates/bin/drain.erb
index e69de29b..3fbdb852 100644
--- a/jobs/k3s-agent/templates/bin/drain.erb
+++ b/jobs/k3s-agent/templates/bin/drain.erb
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml get pods --all-namespaces
+
+#drain
+var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml drain <%= spec.ip %>
diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 694fedcb..4684ce5c 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -11,11 +11,7 @@ case $1 in
     mkdir -p $RUN_DIR $LOG_DIR
     chown -R vcap:vcap $RUN_DIR $LOG_DIR
     
-    # Setup ssh env vars
-    ${JOB_DIR}/bin/setup-user-env
-
     echo $$ > $PIDFILE
-	
       
     exec  /var/vcap/packages/k3s/k3s server \
     -v <%= p('k3s.v') %> \
diff --git a/jobs/k3s-server/templates/bin/drain.erb b/jobs/k3s-server/templates/bin/drain.erb
index e69de29b..3fbdb852 100644
--- a/jobs/k3s-server/templates/bin/drain.erb
+++ b/jobs/k3s-server/templates/bin/drain.erb
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml get pods --all-namespaces
+
+#drain
+var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml drain <%= spec.ip %>

From 107f88849743b0bc8ca8d56b7706424b0d43c5ac Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Mon, 19 Oct 2020 08:56:27 +0000
Subject: [PATCH 17/29] use ip as master identifier

---
 jobs/k3s-server/templates/bin/ctl.erb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 4684ce5c..6970cb52 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -11,6 +11,8 @@ case $1 in
     mkdir -p $RUN_DIR $LOG_DIR
     chown -R vcap:vcap $RUN_DIR $LOG_DIR
     
+    export K3S_NODE_NAME=<%= spec.ip %>
+
     echo $$ > $PIDFILE
       
     exec  /var/vcap/packages/k3s/k3s server \

From e51f5e962a92a383550034329c11ff8b25d21285 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Mon, 19 Oct 2020 10:16:24 -0400
Subject: [PATCH 18/29] conditional templating of custom docker registry

---
 jobs/k3s-agent/templates/config/registries.yaml.erb  | 7 +++++++
 jobs/k3s-agent/templates/config/registry.ca.erb      | 4 +++-
 jobs/k3s-agent/templates/config/registry.cert.erb    | 4 +++-
 jobs/k3s-agent/templates/config/registry.key.erb     | 4 +++-
 jobs/k3s-server/templates/config/registries.yaml.erb | 7 +++++++
 jobs/k3s-server/templates/config/registry.ca.erb     | 4 +++-
 jobs/k3s-server/templates/config/registry.cert.erb   | 4 +++-
 jobs/k3s-server/templates/config/registry.key.erb    | 4 +++-
 8 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/jobs/k3s-agent/templates/config/registries.yaml.erb b/jobs/k3s-agent/templates/config/registries.yaml.erb
index 04261af1..63abc057 100644
--- a/jobs/k3s-agent/templates/config/registries.yaml.erb
+++ b/jobs/k3s-agent/templates/config/registries.yaml.erb
@@ -1,4 +1,8 @@
 ---
+
+<% if_p('registry.mirrors.endpoint') do |value| %>
+
+
 mirrors:
   docker.io:
     endpoint:
@@ -12,3 +16,6 @@ configs:
       cert_file: /var/vcap/jobs/k3s-server/config/registry.cert
       key_file:  /var/vcap/jobs/k3s-server/config/registry.key
       ca_file: /var/vcap/jobs/k3s-server/config/registry.ca
+
+<% end %>
+      
\ No newline at end of file
diff --git a/jobs/k3s-agent/templates/config/registry.ca.erb b/jobs/k3s-agent/templates/config/registry.ca.erb
index f06e2b5d..619a604c 100644
--- a/jobs/k3s-agent/templates/config/registry.ca.erb
+++ b/jobs/k3s-agent/templates/config/registry.ca.erb
@@ -1 +1,3 @@
-<%= p('registry.mirrors.tls.ca') %>
\ No newline at end of file
+<% if_p('registry.mirrors.endpoint') do |value| %>
+<%= p('registry.mirrors.tls.ca') %>
+<<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-agent/templates/config/registry.cert.erb b/jobs/k3s-agent/templates/config/registry.cert.erb
index 9106d00c..cd6730a0 100644
--- a/jobs/k3s-agent/templates/config/registry.cert.erb
+++ b/jobs/k3s-agent/templates/config/registry.cert.erb
@@ -1 +1,3 @@
-<%= p('registry.mirrors.tls.cert') %>
\ No newline at end of file
+<% if_p('registry.mirrors.endpoint') do |value| %>
+<%= p('registry.mirrors.tls.cert') %>
+<<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-agent/templates/config/registry.key.erb b/jobs/k3s-agent/templates/config/registry.key.erb
index 55077003..fa221211 100644
--- a/jobs/k3s-agent/templates/config/registry.key.erb
+++ b/jobs/k3s-agent/templates/config/registry.key.erb
@@ -1 +1,3 @@
-<%= p('registry.mirrors.tls.key') %>
\ No newline at end of file
+<% if_p('registry.mirrors.endpoint') do |value| %>
+<%= p('registry.mirrors.tls.key') %>
+<<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-server/templates/config/registries.yaml.erb b/jobs/k3s-server/templates/config/registries.yaml.erb
index 04261af1..63abc057 100644
--- a/jobs/k3s-server/templates/config/registries.yaml.erb
+++ b/jobs/k3s-server/templates/config/registries.yaml.erb
@@ -1,4 +1,8 @@
 ---
+
+<% if_p('registry.mirrors.endpoint') do |value| %>
+
+
 mirrors:
   docker.io:
     endpoint:
@@ -12,3 +16,6 @@ configs:
       cert_file: /var/vcap/jobs/k3s-server/config/registry.cert
       key_file:  /var/vcap/jobs/k3s-server/config/registry.key
       ca_file: /var/vcap/jobs/k3s-server/config/registry.ca
+
+<% end %>
+      
\ No newline at end of file
diff --git a/jobs/k3s-server/templates/config/registry.ca.erb b/jobs/k3s-server/templates/config/registry.ca.erb
index f06e2b5d..619a604c 100644
--- a/jobs/k3s-server/templates/config/registry.ca.erb
+++ b/jobs/k3s-server/templates/config/registry.ca.erb
@@ -1 +1,3 @@
-<%= p('registry.mirrors.tls.ca') %>
\ No newline at end of file
+<% if_p('registry.mirrors.endpoint') do |value| %>
+<%= p('registry.mirrors.tls.ca') %>
+<<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-server/templates/config/registry.cert.erb b/jobs/k3s-server/templates/config/registry.cert.erb
index 9106d00c..cd6730a0 100644
--- a/jobs/k3s-server/templates/config/registry.cert.erb
+++ b/jobs/k3s-server/templates/config/registry.cert.erb
@@ -1 +1,3 @@
-<%= p('registry.mirrors.tls.cert') %>
\ No newline at end of file
+<% if_p('registry.mirrors.endpoint') do |value| %>
+<%= p('registry.mirrors.tls.cert') %>
+<<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-server/templates/config/registry.key.erb b/jobs/k3s-server/templates/config/registry.key.erb
index 55077003..fa221211 100644
--- a/jobs/k3s-server/templates/config/registry.key.erb
+++ b/jobs/k3s-server/templates/config/registry.key.erb
@@ -1 +1,3 @@
-<%= p('registry.mirrors.tls.key') %>
\ No newline at end of file
+<% if_p('registry.mirrors.endpoint') do |value| %>
+<%= p('registry.mirrors.tls.key') %>
+<<% end %>
\ No newline at end of file

From 2694ca1e6250ee6f940f49a6f138d5a1ee340e42 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Mon, 19 Oct 2020 10:17:10 -0400
Subject: [PATCH 19/29] init k8s drain properties

---
 jobs/k3s-server/spec | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index ed31453d..22bb14c2 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -79,6 +79,44 @@ properties:
     description: (agent/flags) Customized flag for kubelet process
     default: ""
 
+  k3s.drain.delete-local-data:
+    description:  continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).
+    default: false
+
+  k3s.drain.disable-eviction: 
+    description: force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution
+    default: false
+
+
+  k3s.drain.grace-period:
+    description: period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used.
+    default: -1
+
+
+  k3s.drain.ignore-daemonsets:
+    description: Ignore DaemonSet-managed pods.
+    default: false
+
+
+  k3s.drain.pod-selector:
+    description:  Label selector to filter pods on the node
+    default: xx #no default
+
+
+  k3s.drain.selector:
+    description: Selector (label query) to filter on
+    default: xx #no default
+
+
+  k3s.drain.skip-wait-for-delete-timeout:
+    description: If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.
+    default: 0
+
+
+  k3s.drain.timeout:
+    description: The length of time to wait before giving up, zero means infinite
+    default: 0
+
 
 
 #done    

From 4fcd20769550899fee4dd4b706696f3b882c016e Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Mon, 19 Oct 2020 11:19:23 -0400
Subject: [PATCH 20/29] fix mirroring configuration with 3 level (uri, basic
 auth, certificates)

---
 jobs/k3s-agent/spec                                  | 3 +--
 jobs/k3s-agent/templates/config/registries.yaml.erb  | 5 ++++-
 jobs/k3s-agent/templates/config/registry.ca.erb      | 2 +-
 jobs/k3s-agent/templates/config/registry.cert.erb    | 2 +-
 jobs/k3s-agent/templates/config/registry.key.erb     | 2 +-
 jobs/k3s-server/spec                                 | 3 +--
 jobs/k3s-server/templates/config/registries.yaml.erb | 5 ++++-
 jobs/k3s-server/templates/config/registry.ca.erb     | 2 +-
 jobs/k3s-server/templates/config/registry.cert.erb   | 2 +-
 jobs/k3s-server/templates/config/registry.key.erb    | 2 +-
 10 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec
index 956fe171..8e63f331 100644
--- a/jobs/k3s-agent/spec
+++ b/jobs/k3s-agent/spec
@@ -38,8 +38,7 @@ provides:
 properties:
 
   registry.mirrors.endpoint:
-    description: private registry endpoint
-    default: http://localhost:5000
+    description: private registry endpoint, eg http://localhost:5000
 
   registry.mirrors.auth.username:
     description: private registry user
diff --git a/jobs/k3s-agent/templates/config/registries.yaml.erb b/jobs/k3s-agent/templates/config/registries.yaml.erb
index 63abc057..fed67d8c 100644
--- a/jobs/k3s-agent/templates/config/registries.yaml.erb
+++ b/jobs/k3s-agent/templates/config/registries.yaml.erb
@@ -9,13 +9,16 @@ mirrors:
       - <%= p('registry.mirrors.endpoint') %>
 configs:
   customreg:
+<% if_p('registry.mirrors.auth.username') do |auth| %>
     auth:
       username: <%= p('registry.mirrors.auth.username') %> # this is the registry username
       password: <%= p('registry.mirrors.auth.password') %> # this is the registry password
+<% end %>
+<% if_p('registry.mirrors.tls.cert') do |value| %>
     tls:
       cert_file: /var/vcap/jobs/k3s-server/config/registry.cert
       key_file:  /var/vcap/jobs/k3s-server/config/registry.key
       ca_file: /var/vcap/jobs/k3s-server/config/registry.ca
-
+<% end %>
 <% end %>
       
\ No newline at end of file
diff --git a/jobs/k3s-agent/templates/config/registry.ca.erb b/jobs/k3s-agent/templates/config/registry.ca.erb
index 619a604c..eaa090a8 100644
--- a/jobs/k3s-agent/templates/config/registry.ca.erb
+++ b/jobs/k3s-agent/templates/config/registry.ca.erb
@@ -1,3 +1,3 @@
-<% if_p('registry.mirrors.endpoint') do |value| %>
+<% if_p('registry.mirrors.tls.cert') do |value| %>
 <%= p('registry.mirrors.tls.ca') %>
 <<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-agent/templates/config/registry.cert.erb b/jobs/k3s-agent/templates/config/registry.cert.erb
index cd6730a0..5c704d20 100644
--- a/jobs/k3s-agent/templates/config/registry.cert.erb
+++ b/jobs/k3s-agent/templates/config/registry.cert.erb
@@ -1,3 +1,3 @@
-<% if_p('registry.mirrors.endpoint') do |value| %>
+<% if_p('registry.mirrors.tls.cert') do |value| %>
 <%= p('registry.mirrors.tls.cert') %>
 <<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-agent/templates/config/registry.key.erb b/jobs/k3s-agent/templates/config/registry.key.erb
index fa221211..e5f4c7fb 100644
--- a/jobs/k3s-agent/templates/config/registry.key.erb
+++ b/jobs/k3s-agent/templates/config/registry.key.erb
@@ -1,3 +1,3 @@
-<% if_p('registry.mirrors.endpoint') do |value| %>
+<% if_p('registry.mirrors.tls.cert') do |value| %>
 <%= p('registry.mirrors.tls.key') %>
 <<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index 22bb14c2..7fe68787 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -36,8 +36,7 @@ provides:
 properties:
 
   registry.mirrors.endpoint:
-    description: private registry endpoint
-    default: http://localhost:5000
+    description: private registry endpoint, eg http://localhost:5000
 
   registry.mirrors.auth.username:
     description: private registry user
diff --git a/jobs/k3s-server/templates/config/registries.yaml.erb b/jobs/k3s-server/templates/config/registries.yaml.erb
index 63abc057..fed67d8c 100644
--- a/jobs/k3s-server/templates/config/registries.yaml.erb
+++ b/jobs/k3s-server/templates/config/registries.yaml.erb
@@ -9,13 +9,16 @@ mirrors:
       - <%= p('registry.mirrors.endpoint') %>
 configs:
   customreg:
+<% if_p('registry.mirrors.auth.username') do |auth| %>
     auth:
       username: <%= p('registry.mirrors.auth.username') %> # this is the registry username
       password: <%= p('registry.mirrors.auth.password') %> # this is the registry password
+<% end %>
+<% if_p('registry.mirrors.tls.cert') do |value| %>
     tls:
       cert_file: /var/vcap/jobs/k3s-server/config/registry.cert
       key_file:  /var/vcap/jobs/k3s-server/config/registry.key
       ca_file: /var/vcap/jobs/k3s-server/config/registry.ca
-
+<% end %>
 <% end %>
       
\ No newline at end of file
diff --git a/jobs/k3s-server/templates/config/registry.ca.erb b/jobs/k3s-server/templates/config/registry.ca.erb
index 619a604c..eaa090a8 100644
--- a/jobs/k3s-server/templates/config/registry.ca.erb
+++ b/jobs/k3s-server/templates/config/registry.ca.erb
@@ -1,3 +1,3 @@
-<% if_p('registry.mirrors.endpoint') do |value| %>
+<% if_p('registry.mirrors.tls.cert') do |value| %>
 <%= p('registry.mirrors.tls.ca') %>
 <<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-server/templates/config/registry.cert.erb b/jobs/k3s-server/templates/config/registry.cert.erb
index cd6730a0..5c704d20 100644
--- a/jobs/k3s-server/templates/config/registry.cert.erb
+++ b/jobs/k3s-server/templates/config/registry.cert.erb
@@ -1,3 +1,3 @@
-<% if_p('registry.mirrors.endpoint') do |value| %>
+<% if_p('registry.mirrors.tls.cert') do |value| %>
 <%= p('registry.mirrors.tls.cert') %>
 <<% end %>
\ No newline at end of file
diff --git a/jobs/k3s-server/templates/config/registry.key.erb b/jobs/k3s-server/templates/config/registry.key.erb
index fa221211..e5f4c7fb 100644
--- a/jobs/k3s-server/templates/config/registry.key.erb
+++ b/jobs/k3s-server/templates/config/registry.key.erb
@@ -1,3 +1,3 @@
-<% if_p('registry.mirrors.endpoint') do |value| %>
+<% if_p('registry.mirrors.tls.cert') do |value| %>
 <%= p('registry.mirrors.tls.key') %>
 <<% end %>
\ No newline at end of file

From 316e8fe500d1d71a27f3aa0799ef8c005479b437 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Mon, 19 Oct 2020 11:58:04 -0400
Subject: [PATCH 21/29] bosh post-stop will issue a k3s-killall.sh

---
 jobs/k3s-agent/spec                         | 1 +
 jobs/k3s-agent/templates/bin/drain.erb      | 4 ++--
 jobs/k3s-agent/templates/bin/post-stop.erb  | 4 ++++
 jobs/k3s-server/spec                        | 1 +
 jobs/k3s-server/templates/bin/drain.erb     | 4 ++--
 jobs/k3s-server/templates/bin/post-stop.erb | 4 ++++
 6 files changed, 14 insertions(+), 4 deletions(-)
 create mode 100644 jobs/k3s-agent/templates/bin/post-stop.erb
 create mode 100644 jobs/k3s-server/templates/bin/post-stop.erb

diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec
index 8e63f331..9821f008 100644
--- a/jobs/k3s-agent/spec
+++ b/jobs/k3s-agent/spec
@@ -10,6 +10,7 @@ templates:
   bin/pre-start.erb: bin/pre-start
   bin/post-start.erb: bin/post-start  
   bin/pre-stop.erb: bin/pre-stop
+  bin/post-stop.erb: bin/post-stop
   bin/post-deploy.erb: bin/post-deploy
   bin/drain.erb: bin/drain
   bin/ctl.erb: bin/ctl
diff --git a/jobs/k3s-agent/templates/bin/drain.erb b/jobs/k3s-agent/templates/bin/drain.erb
index 3fbdb852..09b206db 100644
--- a/jobs/k3s-agent/templates/bin/drain.erb
+++ b/jobs/k3s-agent/templates/bin/drain.erb
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml get pods --all-namespaces
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml get pods --all-namespaces
 
 #drain
-var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml drain <%= spec.ip %>
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml drain <%= spec.ip %>
diff --git a/jobs/k3s-agent/templates/bin/post-stop.erb b/jobs/k3s-agent/templates/bin/post-stop.erb
new file mode 100644
index 00000000..be97a81c
--- /dev/null
+++ b/jobs/k3s-agent/templates/bin/post-stop.erb
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "post-stop: k3s-killall.sh to clean the host"
+/var/vcap/jobs/k3s-agent/bin/k3s-killall.sh
diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index 7fe68787..d830f3dc 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -10,6 +10,7 @@ templates:
   bin/pre-start.erb: bin/pre-start
   bin/post-start.erb: bin/post-start  
   bin/pre-stop.erb: bin/pre-stop
+  bin/pre-stop.erb: bin/pre-stop
   bin/post-deploy.erb: bin/post-deploy
   bin/drain.erb: bin/drain
   bin/ctl.erb: bin/ctl
diff --git a/jobs/k3s-server/templates/bin/drain.erb b/jobs/k3s-server/templates/bin/drain.erb
index 3fbdb852..09b206db 100644
--- a/jobs/k3s-server/templates/bin/drain.erb
+++ b/jobs/k3s-server/templates/bin/drain.erb
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml get pods --all-namespaces
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml get pods --all-namespaces
 
 #drain
-var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml drain <%= spec.ip %>
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml drain <%= spec.ip %>
diff --git a/jobs/k3s-server/templates/bin/post-stop.erb b/jobs/k3s-server/templates/bin/post-stop.erb
new file mode 100644
index 00000000..ccdc3d0b
--- /dev/null
+++ b/jobs/k3s-server/templates/bin/post-stop.erb
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "post-stop: k3s-killall.sh to clean the host"
+/var/vcap/jobs/k3s-server/bin/k3s-killall.sh
\ No newline at end of file

From 22c873756096a522f60e91e00013acd241e7b1db Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Mon, 19 Oct 2020 13:01:48 -0400
Subject: [PATCH 22/29] implement drain with all spec properties

---
 jobs/k3s-agent/spec                     | 32 +++++++++++++++++++++++++
 jobs/k3s-agent/templates/bin/drain.erb  | 11 +++++++--
 jobs/k3s-server/spec                    | 20 ++++------------
 jobs/k3s-server/templates/bin/drain.erb | 12 ++++++++--
 4 files changed, 55 insertions(+), 20 deletions(-)

diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec
index 9821f008..15bc66d6 100644
--- a/jobs/k3s-agent/spec
+++ b/jobs/k3s-agent/spec
@@ -74,3 +74,35 @@ properties:
   k3s.node-taint: 
     description: (agent/node) Registering kubelet with set of taints
     default: ""
+
+  k3s.drain.delete-local-data:
+    description:  continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).
+    default: false
+
+  k3s.drain.disable-eviction: 
+    description: force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution
+    default: false
+
+  k3s.drain.grace-period:
+    description: period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used.
+    default: -1
+
+  k3s.drain.ignore-daemonsets:
+    description: Ignore DaemonSet-managed pods.
+    default: false
+
+
+  k3s.drain.skip-wait-for-delete-timeout:
+    description: If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.
+    default: 0
+
+  k3s.drain.timeout:
+    description: The length of time to wait before giving up, zero means infinite
+    default: 0
+
+  k3s.drain.pod-selector: #Not implemented
+    description:  Label selector to filter pods on the node
+
+  k3s.drain.selector:  #Not implemented
+    description: Selector (label query) to filter on
+
diff --git a/jobs/k3s-agent/templates/bin/drain.erb b/jobs/k3s-agent/templates/bin/drain.erb
index 09b206db..0b4077ce 100644
--- a/jobs/k3s-agent/templates/bin/drain.erb
+++ b/jobs/k3s-agent/templates/bin/drain.erb
@@ -1,6 +1,13 @@
 #!/bin/bash
 
-/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml get pods --all-namespaces
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-agent/kubeconfig.yml get pods --all-namespaces
 
 #drain
-/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml drain <%= spec.ip %>
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-agent/kubeconfig.yml  drain <%= spec.ip %> \
+--delete-local-data=<%= p('k3s.drain.delete-local-data') %> \
+--disable-eviction=<%= p('k3s.drain.disable-eviction') %> \
+--drain.grace-period=<%= p('k3s.drain.grace-period') %> \
+--ignore-daemonsets=<%= p('k3s.drain.ignore-daemonsets') %> \
+--skip-wait-for-delete-timeout=<%= p('k3s.drain.skip-wait-for-delete-timeout') %> \
+--drain.timeout=<%= p('k3s.drain.timeout') %> 
+
diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index d830f3dc..fa95be2a 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -54,9 +54,6 @@ properties:
   registry.mirrors.tls.ca:
     description: private registry ca
 
-  k3s.xxx:
-    description:
-
   k3s.v:
     description: "(logging) Number for the log level verbosity (default: 0)"
     default: 0
@@ -87,37 +84,28 @@ properties:
     description: force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution
     default: false
 
-
   k3s.drain.grace-period:
     description: period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used.
     default: -1
 
-
   k3s.drain.ignore-daemonsets:
     description: Ignore DaemonSet-managed pods.
     default: false
 
 
-  k3s.drain.pod-selector:
-    description:  Label selector to filter pods on the node
-    default: xx #no default
-
-
-  k3s.drain.selector:
-    description: Selector (label query) to filter on
-    default: xx #no default
-
-
   k3s.drain.skip-wait-for-delete-timeout:
     description: If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.
     default: 0
 
-
   k3s.drain.timeout:
     description: The length of time to wait before giving up, zero means infinite
     default: 0
 
+  k3s.drain.pod-selector: #Not implemented
+    description:  Label selector to filter pods on the node
 
+  k3s.drain.selector:  #Not implemented
+    description: Selector (label query) to filter on
 
 #done    
 #   -v value                                   (logging) Number for the log level verbosity (default: 0)
diff --git a/jobs/k3s-server/templates/bin/drain.erb b/jobs/k3s-server/templates/bin/drain.erb
index 09b206db..8b64893c 100644
--- a/jobs/k3s-server/templates/bin/drain.erb
+++ b/jobs/k3s-server/templates/bin/drain.erb
@@ -1,6 +1,14 @@
 #!/bin/bash
 
-/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml get pods --all-namespaces
+
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml get pods --all-namespaces
 
 #drain
-/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=./kubeconfig.yml drain <%= spec.ip %>
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml drain <%= spec.ip %> \
+--delete-local-data=<%= p('k3s.drain.delete-local-data') %> \
+--disable-eviction=<%= p('k3s.drain.disable-eviction') %> \
+--drain.grace-period=<%= p('k3s.drain.grace-period') %> \
+--ignore-daemonsets=<%= p('k3s.drain.ignore-daemonsets') %> \
+--skip-wait-for-delete-timeout=<%= p('k3s.drain.skip-wait-for-delete-timeout') %> \
+--drain.timeout=<%= p('k3s.drain.timeout') %> 
+

From 3d01b366886d5db15df8a6f1a37dbae0a5029865 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Mon, 19 Oct 2020 17:12:26 +0000
Subject: [PATCH 23/29] do not drain daemonset pods as default

---
 jobs/k3s-agent/spec                     | 2 +-
 jobs/k3s-agent/templates/bin/drain.erb  | 9 +++++++--
 jobs/k3s-server/spec                    | 2 +-
 jobs/k3s-server/templates/bin/drain.erb | 9 +++++++--
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec
index 15bc66d6..a017a867 100644
--- a/jobs/k3s-agent/spec
+++ b/jobs/k3s-agent/spec
@@ -89,7 +89,7 @@ properties:
 
   k3s.drain.ignore-daemonsets:
     description: Ignore DaemonSet-managed pods.
-    default: false
+    default: true 
 
 
   k3s.drain.skip-wait-for-delete-timeout:
diff --git a/jobs/k3s-agent/templates/bin/drain.erb b/jobs/k3s-agent/templates/bin/drain.erb
index 0b4077ce..c523b9da 100644
--- a/jobs/k3s-agent/templates/bin/drain.erb
+++ b/jobs/k3s-agent/templates/bin/drain.erb
@@ -1,13 +1,18 @@
 #!/bin/bash
 
+LOG_DIR=/var/vcap/sys/log/k3s-agent
+
 /var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-agent/kubeconfig.yml get pods --all-namespaces
 
 #drain
 /var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-agent/kubeconfig.yml  drain <%= spec.ip %> \
 --delete-local-data=<%= p('k3s.drain.delete-local-data') %> \
 --disable-eviction=<%= p('k3s.drain.disable-eviction') %> \
---drain.grace-period=<%= p('k3s.drain.grace-period') %> \
+--grace-period=<%= p('k3s.drain.grace-period') %> \
 --ignore-daemonsets=<%= p('k3s.drain.ignore-daemonsets') %> \
 --skip-wait-for-delete-timeout=<%= p('k3s.drain.skip-wait-for-delete-timeout') %> \
---drain.timeout=<%= p('k3s.drain.timeout') %> 
+--timeout=<%= p('k3s.drain.timeout') %>  \
+>> $JOB_DIR/drain.log \
+2>> $JOB_DIR/drain-stderr.log
+
 
diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec
index fa95be2a..bc20c8f8 100644
--- a/jobs/k3s-server/spec
+++ b/jobs/k3s-server/spec
@@ -90,7 +90,7 @@ properties:
 
   k3s.drain.ignore-daemonsets:
     description: Ignore DaemonSet-managed pods.
-    default: false
+    default: true
 
 
   k3s.drain.skip-wait-for-delete-timeout:
diff --git a/jobs/k3s-server/templates/bin/drain.erb b/jobs/k3s-server/templates/bin/drain.erb
index 8b64893c..24b0acc2 100644
--- a/jobs/k3s-server/templates/bin/drain.erb
+++ b/jobs/k3s-server/templates/bin/drain.erb
@@ -1,14 +1,19 @@
 #!/bin/bash
 
 
+LOG_DIR=/var/vcap/sys/log/k3s-server
+
 /var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml get pods --all-namespaces
 
 #drain
 /var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml drain <%= spec.ip %> \
 --delete-local-data=<%= p('k3s.drain.delete-local-data') %> \
 --disable-eviction=<%= p('k3s.drain.disable-eviction') %> \
---drain.grace-period=<%= p('k3s.drain.grace-period') %> \
+--grace-period=<%= p('k3s.drain.grace-period') %> \
 --ignore-daemonsets=<%= p('k3s.drain.ignore-daemonsets') %> \
 --skip-wait-for-delete-timeout=<%= p('k3s.drain.skip-wait-for-delete-timeout') %> \
---drain.timeout=<%= p('k3s.drain.timeout') %> 
+--timeout=<%= p('k3s.drain.timeout') %> \
+>> $JOB_DIR/drain.log \
+2>> $JOB_DIR/drain-stderr.log
+
 

From a100b3be4f5f19643a8af4c26f012f013f3324ef Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Tue, 20 Oct 2020 05:45:53 -0400
Subject: [PATCH 24/29] explicit set external-ip, and tls-san on ip

---
 jobs/k3s-server/templates/bin/ctl.erb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 6970cb52..3305107d 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -24,6 +24,9 @@ case $1 in
     --private-registry=/var/vcap/jobs/k3s-server/config/registries.yaml \
     --write-kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml \
     --write-kubeconfig-mode=755 \
+    --tls-san=<%= spec.ip %> \
+    --node-external-ip=<%= spec.ip %> \
+    --external-hostname=<%= spec.ip %> \
       >>  $LOG_DIR/k3s-server.stdout.log \
       2>> $LOG_DIR/k3s-server.stderr.log
 

From 8bebda228ebb73811ab86ad0d1cb024f50006ca1 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Tue, 20 Oct 2020 05:47:49 -0400
Subject: [PATCH 25/29] disable agent level drain

---
 jobs/k3s-agent/templates/bin/drain.erb | 3 +++
 jobs/k3s-server/templates/bin/ctl.erb  | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/jobs/k3s-agent/templates/bin/drain.erb b/jobs/k3s-agent/templates/bin/drain.erb
index c523b9da..575db19b 100644
--- a/jobs/k3s-agent/templates/bin/drain.erb
+++ b/jobs/k3s-agent/templates/bin/drain.erb
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+#FIXME: check how a node can drain itself (which kubeconfig ?)
+exit 0
+
 LOG_DIR=/var/vcap/sys/log/k3s-agent
 
 /var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-agent/kubeconfig.yml get pods --all-namespaces
diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 3305107d..3c34d417 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -26,7 +26,6 @@ case $1 in
     --write-kubeconfig-mode=755 \
     --tls-san=<%= spec.ip %> \
     --node-external-ip=<%= spec.ip %> \
-    --external-hostname=<%= spec.ip %> \
       >>  $LOG_DIR/k3s-server.stdout.log \
       2>> $LOG_DIR/k3s-server.stderr.log
 

From 931bede4910f6d93f2e90e8b725507a8107e292d Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Tue, 20 Oct 2020 06:06:43 -0400
Subject: [PATCH 26/29] bosh post-start uncordon server (previous drain might
 have cordon it)

---
 jobs/k3s-server/templates/bin/envrc          | 2 +-
 jobs/k3s-server/templates/bin/post-start.erb | 7 ++++++-
 jobs/k3s-server/templates/bin/pre-start.erb  | 2 ++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/jobs/k3s-server/templates/bin/envrc b/jobs/k3s-server/templates/bin/envrc
index 71f07953..2f2f8119 100644
--- a/jobs/k3s-server/templates/bin/envrc
+++ b/jobs/k3s-server/templates/bin/envrc
@@ -2,6 +2,6 @@
 
 export PATH=$PATH:/var/vcap/packages/k3s
 export PATH=$PATH:/var/vcap/packages/k9s
-export PATH=$PATH:/var/vcap/packages/kubectl
+export PATH=$PATH:/var/vcap/packages/kubectl-k3s
 
 export KUBECONFIG=/var/vcap/store/k3s-server/kubeconfig.yml
diff --git a/jobs/k3s-server/templates/bin/post-start.erb b/jobs/k3s-server/templates/bin/post-start.erb
index c3c3f3f5..e94a6cd7 100644
--- a/jobs/k3s-server/templates/bin/post-start.erb
+++ b/jobs/k3s-server/templates/bin/post-start.erb
@@ -1,2 +1,7 @@
 #!/bin/bash
-exit 0
\ No newline at end of file
+
+
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml get pods --all-namespaces
+
+#uncordon
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml uncordon <%= spec.ip %>
diff --git a/jobs/k3s-server/templates/bin/pre-start.erb b/jobs/k3s-server/templates/bin/pre-start.erb
index a85cc0c5..0b1e54e2 100644
--- a/jobs/k3s-server/templates/bin/pre-start.erb
+++ b/jobs/k3s-server/templates/bin/pre-start.erb
@@ -1,4 +1,6 @@
 #!/bin/sh
+export JOB_DIR=/var/vcap/jobs/k3s-server
+
 /var/vcap/packages/k3s/k3s check-config
 
 # Setup ssh env vars

From e74ce4b6e581b0beed528ca9c9a4029d180b1361 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Tue, 20 Oct 2020 12:02:44 -0400
Subject: [PATCH 27/29] adapt kubeconfig file for remote access (server ip
 instead of 127.0.0.1)

---
 jobs/k3s-server/templates/bin/ctl.erb        | 2 +-
 jobs/k3s-server/templates/bin/post-start.erb | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb
index 3c34d417..317a1e82 100755
--- a/jobs/k3s-server/templates/bin/ctl.erb
+++ b/jobs/k3s-server/templates/bin/ctl.erb
@@ -22,7 +22,7 @@ case $1 in
     --data-dir=/var/vcap/store/k3s-server \
     --default-local-storage-path=/var/vcap/store/k3s-server/local-storage-path \
     --private-registry=/var/vcap/jobs/k3s-server/config/registries.yaml \
-    --write-kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml \
+    --write-kubeconfig=/var/vcap/store/k3s-server/k3s.yaml \
     --write-kubeconfig-mode=755 \
     --tls-san=<%= spec.ip %> \
     --node-external-ip=<%= spec.ip %> \
diff --git a/jobs/k3s-server/templates/bin/post-start.erb b/jobs/k3s-server/templates/bin/post-start.erb
index e94a6cd7..7894ebb3 100644
--- a/jobs/k3s-server/templates/bin/post-start.erb
+++ b/jobs/k3s-server/templates/bin/post-start.erb
@@ -5,3 +5,8 @@
 
 #uncordon
 /var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-server/kubeconfig.yml uncordon <%= spec.ip %>
+
+
+#prepare kubeconfig for remote access 
+
+cat /var/vcap/store/k3s-server/k3s.yaml |sed -r 's/(\b[0-9]{1,3}\.){3}[0-9]{1,3}\b'/"<%= spec.ip %>"/ > /var/vcap/store/k3s-server/kubeconfig.yml

From 183066e386fa9d5d1524521a429fb9ec664b7e04 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Tue, 20 Oct 2020 12:13:35 -0400
Subject: [PATCH 28/29] symlink /etc/rancher/node to /var/vcap/store for
 password persistence

---
 jobs/k3s-agent/templates/bin/pre-start.erb  | 7 ++++---
 jobs/k3s-server/templates/bin/pre-start.erb | 5 +++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/jobs/k3s-agent/templates/bin/pre-start.erb b/jobs/k3s-agent/templates/bin/pre-start.erb
index d05df279..a40eca94 100644
--- a/jobs/k3s-agent/templates/bin/pre-start.erb
+++ b/jobs/k3s-agent/templates/bin/pre-start.erb
@@ -1,8 +1,9 @@
 #!/bin/sh
 /var/vcap/packages/k3s/k3s check-config
 
-#bpm pre start recommandation
-sysctl -e -w net.ipv4.tcp_fin_timeout 10
-sysctl -e -w net.ipv4.tcp_tw_reuse 1
+#prepare a persistent directory so /etc/rancher/node paswword file is kept on bosh recreate
+mkdir -p /etc/rancher
+mkdir -p /var/vcap/store/k3s-agent/etc/rancher/node
+ln -sf /var/vcap/store/k3s-agent/etc/rancher/node /etc/rancher/node
 
 exit 0
diff --git a/jobs/k3s-server/templates/bin/pre-start.erb b/jobs/k3s-server/templates/bin/pre-start.erb
index 0b1e54e2..be32a04d 100644
--- a/jobs/k3s-server/templates/bin/pre-start.erb
+++ b/jobs/k3s-server/templates/bin/pre-start.erb
@@ -6,4 +6,9 @@ export JOB_DIR=/var/vcap/jobs/k3s-server
 # Setup ssh env vars
 ${JOB_DIR}/bin/setup-user-env
 
+#prepare a persistent directory so /etc/rancher/node paswword file is kept on bosh recreate
+mkdir -p /etc/rancher
+mkdir -p /var/vcap/store/k3s-agent/etc/rancher/node
+ln -sf /var/vcap/store/k3s-agent/etc/rancher/node /etc/rancher/node
+
 exit 0

From ae774324de3ae7d7871fa95d1b88ed294a264ac4 Mon Sep 17 00:00:00 2001
From: Pierre Oblin <pierre.oblin@orange.com>
Date: Tue, 20 Oct 2020 18:52:29 +0000
Subject: [PATCH 29/29] version 4

---
 .final_builds/jobs/k3s-agent/index.yml       |  4 ++
 .final_builds/jobs/k3s-server/index.yml      |  4 ++
 .final_builds/packages/k3s-images/index.yml  |  4 ++
 .final_builds/packages/k3s/index.yml         |  4 ++
 .final_builds/packages/k9s/index.yml         |  4 ++
 .final_builds/packages/kubectl-k3s/index.yml |  6 +++
 releases/k3s/index.yml                       |  2 +
 releases/k3s/k3s-4.yml                       | 52 ++++++++++++++++++++
 8 files changed, 80 insertions(+)
 create mode 100644 .final_builds/packages/kubectl-k3s/index.yml
 create mode 100644 releases/k3s/k3s-4.yml

diff --git a/.final_builds/jobs/k3s-agent/index.yml b/.final_builds/jobs/k3s-agent/index.yml
index 3a04e1f4..f1284031 100644
--- a/.final_builds/jobs/k3s-agent/index.yml
+++ b/.final_builds/jobs/k3s-agent/index.yml
@@ -3,4 +3,8 @@ builds:
     version: 56423cc75a6c653b3d5998e459a50bba4bc7d8303445b2286b6daeb6572c6be5
     blobstore_id: f1d3412c-a43c-49fb-5a92-9da7b20d8294
     sha1: sha256:3c218e1734d2b229ab2ba71f14f4ce3be6c09e6f8c6d814ba9ea5e242518af96
+  e75d3a2bafa394c6eed9cdd360a93f2c31092bea3db75f96790a8b07dbca7e2e:
+    version: e75d3a2bafa394c6eed9cdd360a93f2c31092bea3db75f96790a8b07dbca7e2e
+    blobstore_id: 25212fb8-33f5-4785-4d6b-0226f5580352
+    sha1: sha256:97285787230e13113fb3a677a5171722cb25931791d96921a6ee6e8983c7121f
 format-version: "2"
diff --git a/.final_builds/jobs/k3s-server/index.yml b/.final_builds/jobs/k3s-server/index.yml
index 735a401c..31f18c1c 100644
--- a/.final_builds/jobs/k3s-server/index.yml
+++ b/.final_builds/jobs/k3s-server/index.yml
@@ -7,4 +7,8 @@ builds:
     version: 55f25ac5d2e2e76db11009b64baaa73fcca6c28cc4020baf80541263dfa995d2
     blobstore_id: 6bcba8f2-e19b-4279-619c-f1acec3ad431
     sha1: sha256:e531cc557ea80d79c40d81c3a3c026127de222f688d5c699d733288ef3142fdd
+  85cbbe9198ed988a83185dbf9ddf52e6979f18b78a902198289df16b44e37a6f:
+    version: 85cbbe9198ed988a83185dbf9ddf52e6979f18b78a902198289df16b44e37a6f
+    blobstore_id: 21b746aa-39a6-40f4-5021-622a56913fb7
+    sha1: sha256:3417e6aa91f950e880ec5b286ffded241c6bfef48a025a56ce69e8d65300986a
 format-version: "2"
diff --git a/.final_builds/packages/k3s-images/index.yml b/.final_builds/packages/k3s-images/index.yml
index e8ea8621..b4ddc31f 100644
--- a/.final_builds/packages/k3s-images/index.yml
+++ b/.final_builds/packages/k3s-images/index.yml
@@ -7,4 +7,8 @@ builds:
     version: 142d2d012e1a3668783c2108e90c9bc86641483cf985f8ced9cb63ea043d5627
     blobstore_id: 240c674a-fb99-445c-52a9-273b8a630148
     sha1: sha256:d409b5244a1307951d4d6ad365976c9d8eb60d77d060f0870e2c432db246138a
+  5c931aa45f328bcf860adaa2a3eaf32f58b53b38d375792be32c540084b0ce27:
+    version: 5c931aa45f328bcf860adaa2a3eaf32f58b53b38d375792be32c540084b0ce27
+    blobstore_id: d3faf4cb-ab64-4110-71f8-4f203af5b47c
+    sha1: sha256:3e4460c7fafbb2029c5ec4da0a58c449354aa0c9c174796a29cf022a3e866419
 format-version: "2"
diff --git a/.final_builds/packages/k3s/index.yml b/.final_builds/packages/k3s/index.yml
index 4be4e907..ca18bed0 100644
--- a/.final_builds/packages/k3s/index.yml
+++ b/.final_builds/packages/k3s/index.yml
@@ -1,4 +1,8 @@
 builds:
+  79950a7a1c37a84c0cda3cd8ed316bb4e4dcdbeea8a079cdcf5e5009bbac58f9:
+    version: 79950a7a1c37a84c0cda3cd8ed316bb4e4dcdbeea8a079cdcf5e5009bbac58f9
+    blobstore_id: d379e0a2-1a6d-4046-5e00-6587f5634ca9
+    sha1: sha256:9abed7bfee5a719b098bd4b0d8ae5aba11775b5ab088bf4fb8328cb40c35f348
   bdaf3648d5da639116068ebc75c43e5198401374b08e8d6268a8204897b189e5:
     version: bdaf3648d5da639116068ebc75c43e5198401374b08e8d6268a8204897b189e5
     blobstore_id: 0eec279f-3ac8-4776-47a1-3b4c38ca2260
diff --git a/.final_builds/packages/k9s/index.yml b/.final_builds/packages/k9s/index.yml
index 38943936..9857d1bd 100644
--- a/.final_builds/packages/k9s/index.yml
+++ b/.final_builds/packages/k9s/index.yml
@@ -3,4 +3,8 @@ builds:
     version: fe4389249af0fe3ced7d677e7008beadf39351d0bcdb0efff0890c3723a67aad
     blobstore_id: 7e62b352-50de-4e5c-66f4-cf223b78befd
     sha1: sha256:8ec51482f19107e038350a4488e02975f7e9a355ba876aa7d2df52fbaaab9f03
+  fe5117ab40ba6d673737299c5f28c16425991d67ebb1c375099da7b9b6ceea17:
+    version: fe5117ab40ba6d673737299c5f28c16425991d67ebb1c375099da7b9b6ceea17
+    blobstore_id: e5ac57fa-0c16-4785-6ca1-658422e3b212
+    sha1: sha256:e070762b81da8733d9df2ea01990ee7c86aaa9c1c67611af99de136607bdbe65
 format-version: "2"
diff --git a/.final_builds/packages/kubectl-k3s/index.yml b/.final_builds/packages/kubectl-k3s/index.yml
new file mode 100644
index 00000000..043f6fca
--- /dev/null
+++ b/.final_builds/packages/kubectl-k3s/index.yml
@@ -0,0 +1,6 @@
+builds:
+  41fe074fc5fe879b1909bdc82b3050c42b94e2d4aa945de2ce4e5f5737feb8dc:
+    version: 41fe074fc5fe879b1909bdc82b3050c42b94e2d4aa945de2ce4e5f5737feb8dc
+    blobstore_id: 7c6d7770-354c-44b0-6150-82614409dfb2
+    sha1: sha256:79ec01ab7c166b7121052ee44ce0109bb0318bb8e14e5e23c7b1d9dc5f619910
+format-version: "2"
diff --git a/releases/k3s/index.yml b/releases/k3s/index.yml
index c0e5ac4a..3c132937 100644
--- a/releases/k3s/index.yml
+++ b/releases/k3s/index.yml
@@ -1,4 +1,6 @@
 builds:
+  30048db2-3d9d-40cc-4dd7-5ca02ff12b66:
+    version: "4"
   4e64a643-4d01-48fa-794b-6564cef26f31:
     version: "1"
   7f7ad91b-e067-4862-4b39-b2fb7b241259:
diff --git a/releases/k3s/k3s-4.yml b/releases/k3s/k3s-4.yml
new file mode 100644
index 00000000..bc82ec9b
--- /dev/null
+++ b/releases/k3s/k3s-4.yml
@@ -0,0 +1,52 @@
+name: k3s
+version: "4"
+commit_hash: 183066e
+uncommitted_changes: false
+jobs:
+- name: k3s-agent
+  version: e75d3a2bafa394c6eed9cdd360a93f2c31092bea3db75f96790a8b07dbca7e2e
+  fingerprint: e75d3a2bafa394c6eed9cdd360a93f2c31092bea3db75f96790a8b07dbca7e2e
+  sha1: sha256:97285787230e13113fb3a677a5171722cb25931791d96921a6ee6e8983c7121f
+  packages:
+  - k3s
+  - kubectl-k3s
+  - k3s-images
+- name: k3s-airgaped-images
+  version: 99126a51b26e8df4e7009e55869d8167bc9f554aed95f3eef688a1e13f669f2b
+  fingerprint: 99126a51b26e8df4e7009e55869d8167bc9f554aed95f3eef688a1e13f669f2b
+  sha1: sha256:9c4a0f047fa7c3f4a9a1d55f6686378979a292158147746296ed4b94eb8dd5ba
+  packages:
+  - k3s-images
+- name: k3s-server
+  version: 85cbbe9198ed988a83185dbf9ddf52e6979f18b78a902198289df16b44e37a6f
+  fingerprint: 85cbbe9198ed988a83185dbf9ddf52e6979f18b78a902198289df16b44e37a6f
+  sha1: sha256:3417e6aa91f950e880ec5b286ffded241c6bfef48a025a56ce69e8d65300986a
+  packages:
+  - k3s
+  - k9s
+  - kubectl-k3s
+packages:
+- name: k3s
+  version: 79950a7a1c37a84c0cda3cd8ed316bb4e4dcdbeea8a079cdcf5e5009bbac58f9
+  fingerprint: 79950a7a1c37a84c0cda3cd8ed316bb4e4dcdbeea8a079cdcf5e5009bbac58f9
+  sha1: sha256:9abed7bfee5a719b098bd4b0d8ae5aba11775b5ab088bf4fb8328cb40c35f348
+  dependencies: []
+- name: k3s-images
+  version: 5c931aa45f328bcf860adaa2a3eaf32f58b53b38d375792be32c540084b0ce27
+  fingerprint: 5c931aa45f328bcf860adaa2a3eaf32f58b53b38d375792be32c540084b0ce27
+  sha1: sha256:3e4460c7fafbb2029c5ec4da0a58c449354aa0c9c174796a29cf022a3e866419
+  dependencies: []
+- name: k9s
+  version: fe5117ab40ba6d673737299c5f28c16425991d67ebb1c375099da7b9b6ceea17
+  fingerprint: fe5117ab40ba6d673737299c5f28c16425991d67ebb1c375099da7b9b6ceea17
+  sha1: sha256:e070762b81da8733d9df2ea01990ee7c86aaa9c1c67611af99de136607bdbe65
+  dependencies: []
+- name: kubectl-k3s
+  version: 41fe074fc5fe879b1909bdc82b3050c42b94e2d4aa945de2ce4e5f5737feb8dc
+  fingerprint: 41fe074fc5fe879b1909bdc82b3050c42b94e2d4aa945de2ce4e5f5737feb8dc
+  sha1: sha256:79ec01ab7c166b7121052ee44ce0109bb0318bb8e14e5e23c7b1d9dc5f619910
+  dependencies: []
+license:
+  version: cf0ca1fe65dee9607c7bb9460a3f20835e582cbb7f5b2c3625fba2e21d7e91b9
+  fingerprint: cf0ca1fe65dee9607c7bb9460a3f20835e582cbb7f5b2c3625fba2e21d7e91b9
+  sha1: sha256:cbe3175489dfaec91e9f392399fbfb68e008885feb91a3b8d493a35d2fc5a8fa