Merge pull request #18 from poblin-orange/develop

version 8

.final_builds/jobs/k3s-agent/index.yml

@@ -1,6 +1,22 @@
 builds:
+  12fa7f8a92464c3b5407109f87d962c8cf5f6023dfbc21fd466ea57de011bfb1:
+    version: 12fa7f8a92464c3b5407109f87d962c8cf5f6023dfbc21fd466ea57de011bfb1
+    blobstore_id: 41c89ae7-8b26-46bc-43e5-e272541ceb3f
+    sha1: sha256:044bcfc366acb02f7c888a5549c9bb06c0bf23666835f7ebb826c9cef14f65c0
+  31331644ddf17d3128cc2c5be631f950c381d85f31b0da879bb3e7ec87c7425d:
+    version: 31331644ddf17d3128cc2c5be631f950c381d85f31b0da879bb3e7ec87c7425d
+    blobstore_id: 6a9f3709-73d3-4141-5fd5-8039866d0591
+    sha1: sha256:f9765052a793d1f650ef88beb2362ff851ead6c0367d47b5c99b88907b7b82a9
   56423cc75a6c653b3d5998e459a50bba4bc7d8303445b2286b6daeb6572c6be5:
     version: 56423cc75a6c653b3d5998e459a50bba4bc7d8303445b2286b6daeb6572c6be5
     blobstore_id: f1d3412c-a43c-49fb-5a92-9da7b20d8294
     sha1: sha256:3c218e1734d2b229ab2ba71f14f4ce3be6c09e6f8c6d814ba9ea5e242518af96
+  c17a8adc1a8f595743f58228497c767a8bf5faab71e184c7a48c45bbeac1a843:
+    version: c17a8adc1a8f595743f58228497c767a8bf5faab71e184c7a48c45bbeac1a843
+    blobstore_id: 6d5c4e0a-e31a-49a8-5037-96aa51036a49
+    sha1: sha256:71dd8c68e6c5aee6f2a1a20740fd45ba458b909dec44fe2f3a44ee7e6084da6b
+  e75d3a2bafa394c6eed9cdd360a93f2c31092bea3db75f96790a8b07dbca7e2e:
+    version: e75d3a2bafa394c6eed9cdd360a93f2c31092bea3db75f96790a8b07dbca7e2e
+    blobstore_id: 25212fb8-33f5-4785-4d6b-0226f5580352
+    sha1: sha256:97285787230e13113fb3a677a5171722cb25931791d96921a6ee6e8983c7121f
 format-version: "2"

.final_builds/jobs/k3s-server/index.yml

@@ -1,6 +1,30 @@
 builds:
+  1c42d77e460f3ca17fdec8951313d07b62c84342f503738e1f401a054bb5e8c6:
+    version: 1c42d77e460f3ca17fdec8951313d07b62c84342f503738e1f401a054bb5e8c6
+    blobstore_id: d0b95461-ce50-4c96-5af4-2ba09e2c2d65
+    sha1: sha256:f3e6d95056dad68b94ad309d2dbfabb297a97a12a5546fce5230e32b3687d556
+  1f38608d21ee257455920d16f1087120eba3ab91d50c23186e74ba93ed2b1674:
+    version: 1f38608d21ee257455920d16f1087120eba3ab91d50c23186e74ba93ed2b1674
+    blobstore_id: acbdbcae-027d-4678-7128-746c02246279
+    sha1: sha256:20d70fe44211a579629452ca9f26fb06570b048901eacde26881c804fc81e8cf
+  3ac37bb6d183cbe54221590838324785fc40f95182254acbf03ef4a6c7cef37e:
+    version: 3ac37bb6d183cbe54221590838324785fc40f95182254acbf03ef4a6c7cef37e
+    blobstore_id: e9de980f-5d77-45a9-5a89-e4cb79562737
+    sha1: sha256:6a821829144cd2ae18b63a696168e94da17701a4590fa0a89fa5c66f3f73867c
+  4dd0b4c15a58fceae095811a02039cab981e346dab1771cc20bc7b18bdc994fc:
+    version: 4dd0b4c15a58fceae095811a02039cab981e346dab1771cc20bc7b18bdc994fc
+    blobstore_id: 5d065403-e29c-428f-7728-a03e13bffc31
+    sha1: sha256:d409f3b559600c00d83e30e32fab313a4abc433c34c5b80575e9dbfaae45224a
   55f25ac5d2e2e76db11009b64baaa73fcca6c28cc4020baf80541263dfa995d2:
     version: 55f25ac5d2e2e76db11009b64baaa73fcca6c28cc4020baf80541263dfa995d2
     blobstore_id: 6bcba8f2-e19b-4279-619c-f1acec3ad431
     sha1: sha256:e531cc557ea80d79c40d81c3a3c026127de222f688d5c699d733288ef3142fdd
+  85cbbe9198ed988a83185dbf9ddf52e6979f18b78a902198289df16b44e37a6f:
+    version: 85cbbe9198ed988a83185dbf9ddf52e6979f18b78a902198289df16b44e37a6f
+    blobstore_id: 21b746aa-39a6-40f4-5021-622a56913fb7
+    sha1: sha256:3417e6aa91f950e880ec5b286ffded241c6bfef48a025a56ce69e8d65300986a
+  a39c16432a77d74db4b196076656a69b6a5eee052e39105dadce4cda8909d81d:
+    version: a39c16432a77d74db4b196076656a69b6a5eee052e39105dadce4cda8909d81d
+    blobstore_id: 2668a214-d655-4bbc-6bfe-7711499fee14
+    sha1: sha256:5c2b073b2a84b5f0e7a6b2da642d3ab21f68608f1550913b1e2da56417f178ec
 format-version: "2"

.final_builds/packages/k3s-images/index.yml

@@ -3,4 +3,16 @@ builds:
     version: 0d29cc086527b5e91253d1f85dcfcc2deb88f0c956c216563facbe62ffc8a6c4
     blobstore_id: 4cc54da2-799b-4f82-5984-fdd12d71fdf3
     sha1: sha256:80c9c4abea2d72c2d0fda022cac1cee308ca7751f809a4a983595069175bb88b
+  142d2d012e1a3668783c2108e90c9bc86641483cf985f8ced9cb63ea043d5627:
+    version: 142d2d012e1a3668783c2108e90c9bc86641483cf985f8ced9cb63ea043d5627
+    blobstore_id: 240c674a-fb99-445c-52a9-273b8a630148
+    sha1: sha256:d409b5244a1307951d4d6ad365976c9d8eb60d77d060f0870e2c432db246138a
+  5c931aa45f328bcf860adaa2a3eaf32f58b53b38d375792be32c540084b0ce27:
+    version: 5c931aa45f328bcf860adaa2a3eaf32f58b53b38d375792be32c540084b0ce27
+    blobstore_id: d3faf4cb-ab64-4110-71f8-4f203af5b47c
+    sha1: sha256:3e4460c7fafbb2029c5ec4da0a58c449354aa0c9c174796a29cf022a3e866419
+  c78a8c2e8cd3d8f69ec70c3f38db944e40cdd94f79ce9a32eef8325cf9c08311:
+    version: c78a8c2e8cd3d8f69ec70c3f38db944e40cdd94f79ce9a32eef8325cf9c08311
+    blobstore_id: 33d21678-bde1-4b43-5283-d49ef23d4274
+    sha1: sha256:359a920d70cd47fa7ce7176a2fa4bb647c17c03b8f5191f7a22ab9be65b56a9c
 format-version: "2"

.final_builds/packages/k3s/index.yml

@@ -1,4 +1,16 @@
 builds:
+  40aa439634b35ba7dd9b6e706ca437d8e191d1cad1463aae7078e0fcca1c2ad8:
+    version: 40aa439634b35ba7dd9b6e706ca437d8e191d1cad1463aae7078e0fcca1c2ad8
+    blobstore_id: f66c41c1-6304-40a5-6920-edd8a5e7dcdf
+    sha1: sha256:b6ec3a2decfa0e16b84887e3c2fae96bdce20520a305f8613c5d16d45f984e53
+  79950a7a1c37a84c0cda3cd8ed316bb4e4dcdbeea8a079cdcf5e5009bbac58f9:
+    version: 79950a7a1c37a84c0cda3cd8ed316bb4e4dcdbeea8a079cdcf5e5009bbac58f9
+    blobstore_id: d379e0a2-1a6d-4046-5e00-6587f5634ca9
+    sha1: sha256:9abed7bfee5a719b098bd4b0d8ae5aba11775b5ab088bf4fb8328cb40c35f348
+  bdaf3648d5da639116068ebc75c43e5198401374b08e8d6268a8204897b189e5:
+    version: bdaf3648d5da639116068ebc75c43e5198401374b08e8d6268a8204897b189e5
+    blobstore_id: 0eec279f-3ac8-4776-47a1-3b4c38ca2260
+    sha1: sha256:09da263a24f791993810d6195a16fac1dd6d875c364a8bcbfa103d615c2e289d
   deafc4b71013993801c97617b8e57c5215062ae3d9a9a38d7efcfe038424d4a7:
     version: deafc4b71013993801c97617b8e57c5215062ae3d9a9a38d7efcfe038424d4a7
     blobstore_id: 5f822676-9e90-4024-688f-8a2b5475495f

.final_builds/packages/k9s/index.yml

@@ -0,0 +1,14 @@
+builds:
+  aace0b38859c00e13f6c985d0711fff0213d5d77b894912554ab020ec775a88b:
+    version: aace0b38859c00e13f6c985d0711fff0213d5d77b894912554ab020ec775a88b
+    blobstore_id: ba66175f-21d3-4a23-43cc-30924c1c28b6
+    sha1: sha256:a7ab0bb41adee47313137197d558273451c435a44ddc95d61f5f733bb9ad4db1
+  fe4389249af0fe3ced7d677e7008beadf39351d0bcdb0efff0890c3723a67aad:
+    version: fe4389249af0fe3ced7d677e7008beadf39351d0bcdb0efff0890c3723a67aad
+    blobstore_id: 7e62b352-50de-4e5c-66f4-cf223b78befd
+    sha1: sha256:8ec51482f19107e038350a4488e02975f7e9a355ba876aa7d2df52fbaaab9f03
+  fe5117ab40ba6d673737299c5f28c16425991d67ebb1c375099da7b9b6ceea17:
+    version: fe5117ab40ba6d673737299c5f28c16425991d67ebb1c375099da7b9b6ceea17
+    blobstore_id: e5ac57fa-0c16-4785-6ca1-658422e3b212
+    sha1: sha256:e070762b81da8733d9df2ea01990ee7c86aaa9c1c67611af99de136607bdbe65
+format-version: "2"

.final_builds/packages/kubectl-k3s/index.yml

@@ -0,0 +1,6 @@
+builds:
+  41fe074fc5fe879b1909bdc82b3050c42b94e2d4aa945de2ce4e5f5737feb8dc:
+    version: 41fe074fc5fe879b1909bdc82b3050c42b94e2d4aa945de2ce4e5f5737feb8dc
+    blobstore_id: 7c6d7770-354c-44b0-6150-82614409dfb2
+    sha1: sha256:79ec01ab7c166b7121052ee44ce0109bb0318bb8e14e5e23c7b1d9dc5f619910
+format-version: "2"

README.md

@@ -1,6 +1,41 @@
 # BOSH release for k3s
 
-This BOSH release and deployment manifest deploy a cluster of k3s.
+This BOSH release and deployment manifest deploy a cluster of k3s
+
+Lightweight Kubernetes. 5 less than k8s. https://k3s.io.
+
+The aim of the bosh release:
+- offer a very thin layer on top of k3 binary
+- reduced memory footprint wrt existing k8s bosh release
+
+- enrich k3 experience with bosh goodies
+  - reproductible / stable OS provided as stemcell
+  - easy cluster deployment (multi-vm, multi-master, correcly wired master / agent)
+  - multi-iaas capability (vpshere/openstack/gcp/azure/ ... )
+  - credhub secrets generation (k3s-token)
+- day 2 operations (persistend disk resize, stemcell rotation)  
+- ease of dev / operations
+  - k9s and kubectl are packaged and preconfigured inside the bosh instances
+  - easy automation with complementary bosh mechanismes
+    - bosh errand mechanism
+    - helm-kubectl bosh release
+    - terraform bosh release
+    - database bosh releases for backend (posgres / mysql /etcd)
+    - generic scripting release (if low level scripting is required)
+
+## design overview
+
+Provide a lightweight bosh packaging of Rancher k3s kubernetes distribution
+
+Includes
+- Rancher k3s binary
+- k9S binary
+
+
+The bosh release offers 2 jobs to build a full k3s bosh deployment:
+- k3s-server job.
+- k3s-agent job.
+
 
 ## Usage
 

addblob.sh

@@ -1,13 +1,18 @@
 #!/bin/bash
-wget https://github.com/rancher/k3s/releases/download/v1.17.2-alpha3%2Bk3s1/k3s -O src/k3s
+wget https://github.com/rancher/k3s/releases/download/v1.19.3%2Bk3s2/k3s -O src/k3s
 bosh add-blob src/k3s k3s/k3s
 
-wget https://github.com/rancher/k3s/releases/download/v1.17.2-alpha3%2Bk3s1/k3s-airgap-images-amd64.tar -O src/k3s-airgap-images-amd64.tar
+wget https://github.com/rancher/k3s/releases/download/v1.19.3%2Bk3s2/k3s-airgap-images-amd64.tar -O src/k3s-airgap-images-amd64.tar
 bosh add-blob src/k3s-airgap-images-amd64.tar k3s-images/k3s-airgap-images-amd64.tar
 
-wget https://github.com/derailed/k9s/releases/download/v0.13.6/k9s_0.13.6_Linux_x86_64.tar.gz -O src/k9s_0.13.6_Linux_x86_64.tar.gz
+wget https://github.com/derailed/k9s/releases/download/v0.23.10/k9s_Linux_x86_64.tar.gz -O src/k9s_Linux_x86_64.tar.gz
 cd src
-tar xfv k9s_0.13.6_Linux_x86_64.tar.gz
+tar xfv ./k9s_Linux_x86_64.tar.gz
 cd ..
 bosh add-blob src/k9s k9s/k9s
-# rm src/k9s_0.13.6_Linux_x86_64.tar.gz
+
+curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl -O src/kubectl
+
+bosh add-blob src/kubectl kubectl/kubectl
+
+

config/blobs.yml

@@ -1,9 +1,16 @@
 k3s-images/k3s-airgap-images-amd64.tar:
-  size: 349377024
-  sha: sha256:e1d140575deb3043cb0102d1ae510ab3273875a275662e1ca84a89c955471189
+  size: 369606656
+  object_id: bbf3379f-5241-48a2-50fd-3707324ef42c
+  sha: sha256:74bbc7ee9799c191f0c59b50c9fc290ba6399c7008d852a68ab2cf7892ead3e1
 k3s/k3s:
-  size: 52273152
-  sha: sha256:b283533e78a2bdec3b6b20b92729b3618221e1e89321dbce73ac1fb466ea4f1a
+  size: 53481472
+  object_id: 2ac8a792-4d4a-43df-42ef-1a63d2e84173
+  sha: sha256:6bc33818baa64a4821695d39445720b98061a8058ff89bd468bc31ab5bdd20f4
 k9s/k9s:
-  size: 42287104
-  sha: sha256:241b52c6373d46bb4fc076c5f656618fd0cbb728823030880e0d89d840278050
+  size: 49360896
+  object_id: 076f426a-dd4e-4549-5b17-728c0f59e8c3
+  sha: sha256:1625460a7940f00cf81382ba55eb2df3a1bc47c90d24119d6aff482261aa0e39
+kubectl/kubectl:
+  size: 43003904
+  object_id: bb931f61-9bd1-4cca-71a0-bf1139913a72
+  sha: sha256:79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f

jobs/k3s-agent/spec

@@ -3,14 +3,25 @@ name: k3s-agent
 
 packages:
 - k3s
+- kubectl-k3s
 - k3s-images
 templates:
   config/bpm.yml: config/bpm.yml
   bin/pre-start.erb: bin/pre-start
   bin/post-start.erb: bin/post-start  
   bin/pre-stop.erb: bin/pre-stop
+  bin/post-stop.erb: bin/post-stop
   bin/post-deploy.erb: bin/post-deploy
+  bin/drain.erb: bin/drain
   bin/ctl.erb: bin/ctl
+  bin/k3s-killall.sh: bin/k3s-killall.sh
+
+
+
+  config/registries.yaml.erb: config/registries.yaml
+  config/registry.ca.erb: config/registry.ca
+  config/registry.cert.erb: config/registry.cert
+  config/registry.key.erb: config/registry.key 
 
 
 # Documentation https://bosh.io/docs/links.html
@@ -26,13 +37,27 @@ provides:
   type: k3s-agent
 
 properties:
+
+  containerd_registry:
+    description: containerd registry configuration
+
+  registry.mirrors.tls.cert:
+    description: private registry certificate
+
+  registry.mirrors.tls.key:
+    description: private registry private key
+
+  registry.mirrors.tls.ca:
+    description: private registry ca
+
+
   k3s.v:
     description: "(logging) Number for the log level verbosity (default: 0)"
     default: 0
   k3s.bind-address value:
     description: "(listener) k3s bind address (default: 0.0.0.0)"
     default: 0.0.0.0
-  k3s.flannel-backend:                    
+  k3s.flannel-backend:
     description: (networking) One of 'none', 'vxlan', 'ipsec', or 'wireguard'
     default: vxlan
   k3s.token:
@@ -43,3 +68,35 @@ properties:
   k3s.node-taint: 
     description: (agent/node) Registering kubelet with set of taints
     default: ""
+
+  k3s.drain.delete-local-data:
+    description:  continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).
+    default: false
+
+  k3s.drain.disable-eviction: 
+    description: force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution
+    default: false
+
+  k3s.drain.grace-period:
+    description: period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used.
+    default: -1
+
+  k3s.drain.ignore-daemonsets:
+    description: Ignore DaemonSet-managed pods.
+    default: true 
+
+
+  k3s.drain.skip-wait-for-delete-timeout:
+    description: If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.
+    default: 0
+
+  k3s.drain.timeout:
+    description: The length of time to wait before giving up, zero means infinite
+    default: 0
+
+  k3s.drain.pod-selector: #Not implemented
+    description:  Label selector to filter pods on the node
+
+  k3s.drain.selector:  #Not implemented
+    description: Selector (label query) to filter on
+

jobs/k3s-agent/templates/bin/ctl.erb

@@ -14,12 +14,22 @@ case $1 in
 
 	export servers="<% masters = link('k3s-server') %><% masters.instances.each do |instance| %> --server=https://<%= instance.address %>:6443 <% end %>"
 
-
+
+        export K3S_NODE_NAME=<%= spec.name %>-<%= spec.index %>
+
     exec  /var/vcap/packages/k3s/k3s agent \
     -v <%= p('k3s.v') %> \
     --token=<%= p('k3s.token') %> \
     --data-dir=/var/vcap/store/k3s-agent \
     --private-registry=/var/vcap/jobs/k3s-agent/config/registries.yaml \
+    --resolv-conf=/etc/resolv.conf \
+    --node-external-ip=<%= spec.ip %> \
+    --node-label bosh.io/az=<%= spec.az %> \
+    --node-label bosh.io/name=<%= spec.name %>  \
+    --node-label bosh.io/bootstrap=<%= spec.bootstrap %>  \
+    --node-label bosh.io/index=<%= spec.index %>  \
+    --node-label bosh.io/address=<%= spec.address %>  \
+    --node-label topology.kubernetes.io/zone=<%= spec.az %> \
       $servers \
       >>  $LOG_DIR/k3s-agent.stdout.log \
       2>> $LOG_DIR/k3s-agent.stderr.log

jobs/k3s-agent/templates/bin/drain.erb

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+#FIXME: check how a node can drain itself (which kubeconfig ?)
+echo 0; exit 0
+
+
+LOG_DIR=/var/vcap/sys/log/k3s-agent
+
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-agent/kubeconfig.yml get pods --all-namespaces
+
+#drain
+/var/vcap/packages/kubectl-k3s/kubectl --kubeconfig=/var/vcap/store/k3s-agent/kubeconfig.yml  drain <%= spec.ip %> \
+--delete-local-data=<%= p('k3s.drain.delete-local-data') %> \
+--disable-eviction=<%= p('k3s.drain.disable-eviction') %> \
+--grace-period=<%= p('k3s.drain.grace-period') %> \
+--ignore-daemonsets=<%= p('k3s.drain.ignore-daemonsets') %> \
+--skip-wait-for-delete-timeout=<%= p('k3s.drain.skip-wait-for-delete-timeout') %> \
+--timeout=<%= p('k3s.drain.timeout') %>  \
+>> $JOB_DIR/drain.log \
+2>> $JOB_DIR/drain-stderr.log
+
+echo 0; exit 0
+
+