Merge pull request #7 from poblin-orange/develop

k3s-1.19

.final_builds/jobs/k3s-agent/index.yml

@@ -3,4 +3,8 @@ builds:
     version: 56423cc75a6c653b3d5998e459a50bba4bc7d8303445b2286b6daeb6572c6be5
     blobstore_id: f1d3412c-a43c-49fb-5a92-9da7b20d8294
     sha1: sha256:3c218e1734d2b229ab2ba71f14f4ce3be6c09e6f8c6d814ba9ea5e242518af96
+  e75d3a2bafa394c6eed9cdd360a93f2c31092bea3db75f96790a8b07dbca7e2e:
+    version: e75d3a2bafa394c6eed9cdd360a93f2c31092bea3db75f96790a8b07dbca7e2e
+    blobstore_id: 25212fb8-33f5-4785-4d6b-0226f5580352
+    sha1: sha256:97285787230e13113fb3a677a5171722cb25931791d96921a6ee6e8983c7121f
 format-version: "2"

.final_builds/jobs/k3s-server/index.yml

@@ -7,4 +7,8 @@ builds:
     version: 55f25ac5d2e2e76db11009b64baaa73fcca6c28cc4020baf80541263dfa995d2
     blobstore_id: 6bcba8f2-e19b-4279-619c-f1acec3ad431
     sha1: sha256:e531cc557ea80d79c40d81c3a3c026127de222f688d5c699d733288ef3142fdd
+  85cbbe9198ed988a83185dbf9ddf52e6979f18b78a902198289df16b44e37a6f:
+    version: 85cbbe9198ed988a83185dbf9ddf52e6979f18b78a902198289df16b44e37a6f
+    blobstore_id: 21b746aa-39a6-40f4-5021-622a56913fb7
+    sha1: sha256:3417e6aa91f950e880ec5b286ffded241c6bfef48a025a56ce69e8d65300986a
 format-version: "2"

.final_builds/packages/k3s-images/index.yml

@@ -7,4 +7,8 @@ builds:
     version: 142d2d012e1a3668783c2108e90c9bc86641483cf985f8ced9cb63ea043d5627
     blobstore_id: 240c674a-fb99-445c-52a9-273b8a630148
     sha1: sha256:d409b5244a1307951d4d6ad365976c9d8eb60d77d060f0870e2c432db246138a
+  5c931aa45f328bcf860adaa2a3eaf32f58b53b38d375792be32c540084b0ce27:
+    version: 5c931aa45f328bcf860adaa2a3eaf32f58b53b38d375792be32c540084b0ce27
+    blobstore_id: d3faf4cb-ab64-4110-71f8-4f203af5b47c
+    sha1: sha256:3e4460c7fafbb2029c5ec4da0a58c449354aa0c9c174796a29cf022a3e866419
 format-version: "2"

.final_builds/packages/k3s/index.yml

@@ -1,4 +1,8 @@
 builds:
+  79950a7a1c37a84c0cda3cd8ed316bb4e4dcdbeea8a079cdcf5e5009bbac58f9:
+    version: 79950a7a1c37a84c0cda3cd8ed316bb4e4dcdbeea8a079cdcf5e5009bbac58f9
+    blobstore_id: d379e0a2-1a6d-4046-5e00-6587f5634ca9
+    sha1: sha256:9abed7bfee5a719b098bd4b0d8ae5aba11775b5ab088bf4fb8328cb40c35f348
   bdaf3648d5da639116068ebc75c43e5198401374b08e8d6268a8204897b189e5:
     version: bdaf3648d5da639116068ebc75c43e5198401374b08e8d6268a8204897b189e5
     blobstore_id: 0eec279f-3ac8-4776-47a1-3b4c38ca2260

.final_builds/packages/k9s/index.yml

@@ -3,4 +3,8 @@ builds:
     version: fe4389249af0fe3ced7d677e7008beadf39351d0bcdb0efff0890c3723a67aad
     blobstore_id: 7e62b352-50de-4e5c-66f4-cf223b78befd
     sha1: sha256:8ec51482f19107e038350a4488e02975f7e9a355ba876aa7d2df52fbaaab9f03
+  fe5117ab40ba6d673737299c5f28c16425991d67ebb1c375099da7b9b6ceea17:
+    version: fe5117ab40ba6d673737299c5f28c16425991d67ebb1c375099da7b9b6ceea17
+    blobstore_id: e5ac57fa-0c16-4785-6ca1-658422e3b212
+    sha1: sha256:e070762b81da8733d9df2ea01990ee7c86aaa9c1c67611af99de136607bdbe65
 format-version: "2"

.final_builds/packages/kubectl-k3s/index.yml

@@ -0,0 +1,6 @@
+builds:
+  41fe074fc5fe879b1909bdc82b3050c42b94e2d4aa945de2ce4e5f5737feb8dc:
+    version: 41fe074fc5fe879b1909bdc82b3050c42b94e2d4aa945de2ce4e5f5737feb8dc
+    blobstore_id: 7c6d7770-354c-44b0-6150-82614409dfb2
+    sha1: sha256:79ec01ab7c166b7121052ee44ce0109bb0318bb8e14e5e23c7b1d9dc5f619910
+format-version: "2"

README.md

@@ -1,6 +1,23 @@
 # BOSH release for k3s
 
-This BOSH release and deployment manifest deploy a cluster of k3s.
+This BOSH release and deployment manifest deploy a cluster of k3s
+
+Lightweight Kubernetes. 5 less than k8s. https://k3s.io.
+
+
+## design overview
+
+Provide a lightweight bosh packaging of Rancher k3s kubernetes distribution
+
+Includes
+- Rancher k3s binary
+- k9S binary
+
+
+The bosh release offers 2 jobs to build a full k3s bosh deployment:
+- k3s-server job.
+- k3s-agent job.
+
 
 ## Usage
 

addblob.sh

@@ -1,13 +1,18 @@
 #!/bin/bash
-wget https://github.com/rancher/k3s/releases/download/v1.17.2-alpha3%2Bk3s1/k3s -O src/k3s
+wget https://github.com/rancher/k3s/releases/download/v1.19.3%2Bk3s1/k3s -O src/k3s
 bosh add-blob src/k3s k3s/k3s
 
-wget https://github.com/rancher/k3s/releases/download/v1.17.2-alpha3%2Bk3s1/k3s-airgap-images-amd64.tar -O src/k3s-airgap-images-amd64.tar
+wget https://github.com/rancher/k3s/releases/download/v1.19.3%2Bk3s1/k3s-airgap-images-amd64.tar -O src/k3s-airgap-images-amd64.tar
 bosh add-blob src/k3s-airgap-images-amd64.tar k3s-images/k3s-airgap-images-amd64.tar
 
-wget https://github.com/derailed/k9s/releases/download/v0.13.6/k9s_0.13.6_Linux_x86_64.tar.gz -O src/k9s_0.13.6_Linux_x86_64.tar.gz
+wget https://github.com/derailed/k9s/releases/download/v0.22.1/k9s_Linux_x86_64.tar.gz -O src/k9s_Linux_x86_64.tar.gz
 cd src
-tar xfv k9s_0.13.6_Linux_x86_64.tar.gz
+tar xfv ./k9s_Linux_x86_64.tar.gz
 cd ..
 bosh add-blob src/k9s k9s/k9s
-# rm src/k9s_0.13.6_Linux_x86_64.tar.gz
+
+curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl -O src/kubectl
+
+bosh add-blob src/kubectl kubectl/kubectl
+
+

config/blobs.yml

@@ -1,9 +1,16 @@
 k3s-images/k3s-airgap-images-amd64.tar:
-  size: 349377024
-  sha: sha256:e1d140575deb3043cb0102d1ae510ab3273875a275662e1ca84a89c955471189
+  size: 369606656
+  object_id: 172301a2-56a0-4496-7b48-aa79d399a3e8
+  sha: sha256:9d3013742f011bd08225344ae5fd85407bd2bcd88886261ba58e1bff5b00f08d
 k3s/k3s:
-  size: 52273152
-  sha: sha256:b283533e78a2bdec3b6b20b92729b3618221e1e89321dbce73ac1fb466ea4f1a
+  size: 53424128
+  object_id: d928a091-fd50-41a4-5b18-d61bfa41fe92
+  sha: sha256:3b031d78f9edeed6718b5cd1070e4bd46524faa90a82d64f6f28008f6192c5dc
 k9s/k9s:
-  size: 42287104
-  sha: sha256:241b52c6373d46bb4fc076c5f656618fd0cbb728823030880e0d89d840278050
+  size: 49139712
+  object_id: ce7272d4-d76c-4d67-68a2-4ccd59366344
+  sha: sha256:04b457ba48879f64ec97ca513ad573328939b2c9b3591cb1171d7bd989e3eaef
+kubectl/kubectl:
+  size: 43003904
+  object_id: bb931f61-9bd1-4cca-71a0-bf1139913a72
+  sha: sha256:79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f

jobs/k3s-agent/spec

@@ -3,14 +3,25 @@ name: k3s-agent
 
 packages:
 - k3s
+- kubectl-k3s
 - k3s-images
 templates:
   config/bpm.yml: config/bpm.yml
   bin/pre-start.erb: bin/pre-start
   bin/post-start.erb: bin/post-start  
   bin/pre-stop.erb: bin/pre-stop
+  bin/post-stop.erb: bin/post-stop
   bin/post-deploy.erb: bin/post-deploy
+  bin/drain.erb: bin/drain
   bin/ctl.erb: bin/ctl
+  bin/k3s-killall.sh: bin/k3s-killall.sh
+
+
+
+  config/registries.yaml.erb: config/registries.yaml
+  config/registry.ca.erb: config/registry.ca
+  config/registry.cert.erb: config/registry.cert
+  config/registry.key.erb: config/registry.key 
 
 
 # Documentation https://bosh.io/docs/links.html
@@ -26,13 +37,33 @@ provides:
   type: k3s-agent
 
 properties:
+
+  registry.mirrors.endpoint:
+    description: private registry endpoint, eg http://localhost:5000
+
+  registry.mirrors.auth.username:
+    description: private registry user
+
+  registry.mirrors.auth.password:
+    description: private registry password
+
+  registry.mirrors.tls.cert:
+    description: private registry certificate
+
+  registry.mirrors.tls.key:
+    description: private registry private key
+
+  registry.mirrors.tls.ca:
+    description: private registry ca
+
+
   k3s.v:
     description: "(logging) Number for the log level verbosity (default: 0)"
     default: 0
   k3s.bind-address value:
     description: "(listener) k3s bind address (default: 0.0.0.0)"
     default: 0.0.0.0
-  k3s.flannel-backend:                    
+  k3s.flannel-backend:
     description: (networking) One of 'none', 'vxlan', 'ipsec', or 'wireguard'
     default: vxlan
   k3s.token:
@@ -43,3 +74,35 @@ properties:
   k3s.node-taint: 
     description: (agent/node) Registering kubelet with set of taints
     default: ""
+
+  k3s.drain.delete-local-data:
+    description:  continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).
+    default: false
+
+  k3s.drain.disable-eviction: 
+    description: force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution
+    default: false
+
+  k3s.drain.grace-period:
+    description: period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used.
+    default: -1
+
+  k3s.drain.ignore-daemonsets:
+    description: Ignore DaemonSet-managed pods.
+    default: true 
+
+
+  k3s.drain.skip-wait-for-delete-timeout:
+    description: If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.
+    default: 0
+
+  k3s.drain.timeout:
+    description: The length of time to wait before giving up, zero means infinite
+    default: 0
+
+  k3s.drain.pod-selector: #Not implemented
+    description:  Label selector to filter pods on the node
+
+  k3s.drain.selector:  #Not implemented
+    description: Selector (label query) to filter on
+

jobs/k3s-agent/templates/bin/ctl.erb

@@ -14,12 +14,15 @@ case $1 in
 
 	export servers="<% masters = link('k3s-server') %><% masters.instances.each do |instance| %> --server=https://<%= instance.address %>:6443 <% end %>"
 
-
+
+	export K3S_NODE_NAME=<%= spec.ip %>
+
     exec  /var/vcap/packages/k3s/k3s agent \
     -v <%= p('k3s.v') %> \
     --token=<%= p('k3s.token') %> \
     --data-dir=/var/vcap/store/k3s-agent \
     --private-registry=/var/vcap/jobs/k3s-agent/config/registries.yaml \
+    --resolv-conf=/etc/resolv.conf \
       $servers \
       >>  $LOG_DIR/k3s-agent.stdout.log \
       2>> $LOG_DIR/k3s-agent.stderr.log

jobs/k3s-agent/templates/bin/drain.erb

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+#FIXME: check how a node can drain itself (which kubeconfig ?)
+exit 0
+
+LOG_DIR=/var/vcap/sys/log/k3s-agent
+
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-agent/kubeconfig.yml get pods --all-namespaces
+
+#drain
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-agent/kubeconfig.yml  drain <%= spec.ip %> \
+--delete-local-data=<%= p('k3s.drain.delete-local-data') %> \
+--disable-eviction=<%= p('k3s.drain.disable-eviction') %> \
+--grace-period=<%= p('k3s.drain.grace-period') %> \
+--ignore-daemonsets=<%= p('k3s.drain.ignore-daemonsets') %> \
+--skip-wait-for-delete-timeout=<%= p('k3s.drain.skip-wait-for-delete-timeout') %> \
+--timeout=<%= p('k3s.drain.timeout') %>  \
+>> $JOB_DIR/drain.log \
+2>> $JOB_DIR/drain-stderr.log
+
+

jobs/k3s-agent/templates/bin/k3s-killall.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+[ $(id -u) -eq 0 ] || exec sudo $0 $@
+
+for bin in /var/lib/rancher/k3s/data/**/bin/; do
+    [ -d $bin ] && export PATH=$bin:$PATH
+done
+
+set -x
+
+for service in /etc/systemd/system/k3s*.service; do
+    [ -s $service ] && systemctl stop $(basename $service)
+done
+
+for service in /etc/init.d/k3s*; do
+    [ -x $service ] && $service stop
+done
+
+pschildren() {
+    ps -e -o ppid= -o pid= | \
+    sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \
+    grep -w "^$1" | \
+    cut -f2
+}
+
+pstree() {
+    for pid in $@; do
+        echo $pid
+        for child in $(pschildren $pid); do
+            pstree $child
+        done
+    done
+}
+
+killtree() {
+    kill -9 $(
+        { set +x; } 2>/dev/null;
+        pstree $@;
+        set -x;
+    ) 2>/dev/null
+}
+
+getshims() {
+    lsof | sed -e 's/^[^0-9]*//g; s/  */\t/g' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 | sort -n -u
+}
+
+killtree $({ set +x; } 2>/dev/null; getshims; set -x)
+
+do_unmount() {
+    { set +x; } 2>/dev/null
+    MOUNTS=
+    while read ignore mount ignore; do
+        MOUNTS="$mount\n$MOUNTS"
+    done </proc/self/mounts
+    MOUNTS=$(printf $MOUNTS | grep "^$1" | sort -r)
+    if [ -n "${MOUNTS}" ]; then
+        set -x
+        umount ${MOUNTS}
+    else
+        set -x
+    fi
+}
+
+do_unmount '/run/k3s'
+do_unmount '/var/lib/rancher/k3s'
+do_unmount '/var/lib/kubelet/pods'
+do_unmount '/run/netns/cni-'
+
+# Delete network interface(s) that match 'master cni0'
+ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do
+    iface=${iface%%@*}
+    [ -z "$iface" ] || ip link delete $iface
+done
+ip link delete cni0
+ip link delete flannel.1
+rm -rf /var/lib/cni/
+iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
+

jobs/k3s-agent/templates/bin/post-stop.erb

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "post-stop: k3s-killall.sh to clean the host"
+/var/vcap/jobs/k3s-agent/bin/k3s-killall.sh

jobs/k3s-agent/templates/bin/pre-start.erb

@@ -1,8 +1,9 @@
 #!/bin/sh
 /var/vcap/packages/k3s/k3s check-config
 
-#bpm pre start recommandation
-sysctl -e -w net.ipv4.tcp_fin_timeout 10
-sysctl -e -w net.ipv4.tcp_tw_reuse 1
+#prepare a persistent directory so /etc/rancher/node paswword file is kept on bosh recreate
+mkdir -p /etc/rancher
+mkdir -p /var/vcap/store/k3s-agent/etc/rancher/node
+ln -sf /var/vcap/store/k3s-agent/etc/rancher/node /etc/rancher/node
 
 exit 0

jobs/k3s-agent/templates/config/registries.yaml.erb

@@ -0,0 +1,24 @@
+---
+
+<% if_p('registry.mirrors.endpoint') do |value| %>
+
+
+mirrors:
+  docker.io:
+    endpoint:
+      - <%= p('registry.mirrors.endpoint') %>
+configs:
+  customreg:
+<% if_p('registry.mirrors.auth.username') do |auth| %>
+    auth:
+      username: <%= p('registry.mirrors.auth.username') %> # this is the registry username
+      password: <%= p('registry.mirrors.auth.password') %> # this is the registry password
+<% end %>
+<% if_p('registry.mirrors.tls.cert') do |value| %>
+    tls:
+      cert_file: /var/vcap/jobs/k3s-server/config/registry.cert
+      key_file:  /var/vcap/jobs/k3s-server/config/registry.key
+      ca_file: /var/vcap/jobs/k3s-server/config/registry.ca
+<% end %>
+<% end %>
+

jobs/k3s-agent/templates/config/registry.ca.erb

@@ -0,0 +1,3 @@
+<% if_p('registry.mirrors.tls.cert') do |value| %>
+<%= p('registry.mirrors.tls.ca') %>
+<<% end %>

jobs/k3s-agent/templates/config/registry.cert.erb

@@ -0,0 +1,3 @@
+<% if_p('registry.mirrors.tls.cert') do |value| %>
+<%= p('registry.mirrors.tls.cert') %>
+<<% end %>

jobs/k3s-agent/templates/config/registry.key.erb

@@ -0,0 +1,3 @@
+<% if_p('registry.mirrors.tls.cert') do |value| %>
+<%= p('registry.mirrors.tls.key') %>
+<<% end %>