Merge pull request #71 from poblin-orange/develop

k3s 1.21.11

config/blobs.yml

@@ -1,11 +1,11 @@
 k3s-images/k3s-airgap-images-amd64.tar:
-  size: 498181120
-  object_id: 3f192c02-495c-47ac-5c74-5849d3c86f47
-  sha: sha256:a10f1f58851c7f5eeae0699ca761cad30d8ec297ca2d39ad692aecc87c01f74f
+  size: 500852736
+  object_id: fe6c595a-14b1-441d-643d-caa140b83ad0
+  sha: sha256:74e5a861e0c977d736ebb21b1a1a9f30f7236c00ebc590ac90a476972e93c9a0
 k3s/k3s:
-  size: 49082368
-  object_id: a84494e2-29b3-43e9-7f1c-f498e758e75b
-  sha: sha256:7a9cf9a0f2318b09f8b449f96440fdd3d877fcfd5e25b275a0e705400641ce3d
+  size: 49090560
+  object_id: 42a5dfd9-5a02-4bed-473d-07e6e9885393
+  sha: sha256:6450ece4cf5d7ada78c0ef9e27b45bdf5aa1049fccc8e5fc55720ae1087bf0a8
 k9s/k9s:
   size: 55410688
   object_id: 0b3b0840-1db4-4ac5-53b1-0f62a80862cb

jobs/k3s-agent/spec

@@ -24,6 +24,8 @@ templates:
   config/registry.cert.erb: config/registry.cert
   config/registry.key.erb: config/registry.key 
   config/token.csv.erb: config/token.csv
+
+  config/kubelet-config.yaml.erb: config/kubelet-config.yaml
 
 # Documentation https://bosh.io/docs/links.html
 # Tutorial      https://gist.github.com/Amit-PivotalLabs/c39528248b8cdc4ba8e347f8aa68abb6
@@ -100,7 +102,7 @@ properties:
 
   k3s.drain.timeout:
     description: The length of time to wait before giving up, zero means infinite
-    default: 0
+    default: 90s
 
   k3s.drain.pod-selector: #Not implemented
     description:  Label selector to filter pods on the node
@@ -126,3 +128,14 @@ properties:
   k3s.master_vip_api:
     description: externaly defined vip ip for HA k3s (enables multi master instance groups). This is used for public api access, tls-san, and agents to server communication
 
+#kubelet config
+  k3s.kubelet-config-file:
+    description: content of kubelet config file, to enable eg GraceFull Node Shutdown
+    default: |
+      apiVersion: kubelet.config.k8s.io/v1beta1
+      kind: KubeletConfiguration
+
+      shutdownGracePeriod: 30s
+      shutdownGracePeriodCriticalPods: 10s
+
+

jobs/k3s-agent/templates/bin/ctl.erb

@@ -75,6 +75,11 @@ export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.fi
 export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.last.ip %>"
 <% end %>
 
+<% if_p('k3s.kubelet-config-file') do |value| %>
+export FLAGS="$FLAGS --kubelet-arg=config=/var/vcap/jobs/k3s-agent/config/kubelet-config.yaml"
+<% end %>
+
+
 
     ulimit -n 1048576    # open files
     ulimit -u unlimited  # num processes

jobs/k3s-agent/templates/config/kubelet-config.yaml.erb

@@ -0,0 +1,3 @@
+<% if_p('k3s.kubelet-config-file') do |value| %>
+<%= p('k3s.kubelet-config-file') %>
+<% end %>

jobs/k3s-server/spec

@@ -27,6 +27,8 @@ templates:
   config/datastore-keyfile.erb: config/datastore-keyfile
 
   config/additional-manifest.yaml.erb: config/additional-manifest.yaml
+  config/audit-policy.yaml.erb: config/audit-policy.yaml
+  config/kubelet-config.yaml.erb: config/kubelet-config.yaml
 
 
 # Documentation https://bosh.io/docs/links.html
@@ -116,7 +118,7 @@ properties:
 
   k3s.drain.timeout:
     description: The length of time to wait before giving up, zero means infinite
-    default: 0
+    default: 90s 
 
   k3s.drain.pod-selector: #Not implemented
     description:  Label selector to filter pods on the node
@@ -161,7 +163,80 @@ properties:
   k3s.additional_tls_sans:
     description: array of additionnal tls-san for k8s api (added to default, ip, bosh-dns alias, or master vip if set)
     default: []
-
+
+#kubelet config
+  k3s.kubelet-config-file:
+    description: content of kubelet config file, to enable eg GraceFull Node Shutdown
+    default: |
+      apiVersion: kubelet.config.k8s.io/v1beta1
+      kind: KubeletConfiguration
+
+      shutdownGracePeriod: 30s
+      shutdownGracePeriodCriticalPods: 10s
+
+#audit policy file
+  k3s.audit-policy-file:
+    description: if set, defines and activate audit policy for k8s server
+    default: |
+      apiVersion: audit.k8s.io/v1 # This is required.
+      kind: Policy
+
+      # Prevent requests in the RequestReceived stage from generating audit events.
+      omitStages:
+        - "RequestReceived"
+
+      rules:
+        # Log "pods/log", "pods/status" at Metadata level
+        - level: Metadata
+          resources:
+          - group: ""
+            resources: ["pods/log", "pods/status"]
+        # Exclude logging requests to a configmap called "controller-config"
+        - level: None
+          resources:
+          - group: ""
+            resources: ["configmaps"]
+            resourceNames: ["controller-config"]
+        # Don't log watch requests by the "system:kube-proxy" on endpoints or services
+        - level: None
+          users: ["system:kube-proxy"]
+          verbs: ["watch"]
+          resources:
+          - group: "" # core API group
+            resources: ["endpoints", "services"]
+        # Log deployment changes at RequestResponse level
+        - level: Metadata
+          resources:
+          - group: ""
+            resources: ["deployments"]
+        # Log service changes at metadata level
+        - level: Metadata
+          resources:
+          - group: ""
+            resources: ["services"]
+        # Log the request body of configmap changes in the kube-system namespace.
+        - level: Request
+          resources:
+          - group: "" # core API group
+            resources: ["configmaps"]
+          # You can use an empty string [""] to select resources not associated with a namespace.
+          namespaces: ["kube-system"]
+        # Log configmap and secret changes in all other namespaces at the Metadata level.
+        - level: Metadata
+          resources:
+          - group: "" # core API group
+            resources: ["secrets", "configmaps"]
+        # Log all other resources in core and extensions at the Request level.
+        - level: Request
+          resources:
+          - group: "" # core API group
+          - group: "extensions" # Version of group should NOT be included.
+        # A wild-card rule to log all other requests at the Metadata level.
+        - level: Metadata
+          # Long-running requests like watches that fall under this rule will not
+          # generate an audit event in RequestReceived.
+          omitStages:
+            - "RequestReceived"
 
 #done    
 #   -v value                                   (logging) Number for the log level verbosity (default: 0)

jobs/k3s-server/templates/bin/ctl.erb

@@ -109,6 +109,21 @@ export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.fi
 export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.last.ip %>"
 <% end %>
 
+<% if_p('k3s.audit-policy-file') do |value| %>
+export FLAGS="$FLAGS --kube-apiserver-arg=audit-log-path=/var/vcap/sys/log/k3s-server/audit.log"
+export FLAGS="$FLAGS --kube-apiserver-arg=audit-policy-file=/var/vcap/jobs/k3s-server/config/audit-policy.yaml"
+
+export FLAGS="$FLAGS --kube-apiserver-arg=audit-log-maxage=15"
+export FLAGS="$FLAGS --kube-apiserver-arg=audit-log-maxbackup=5"
+export FLAGS="$FLAGS --kube-apiserver-arg=audit-log-maxsize=10"
+
+
+<% end %>
+
+<% if_p('k3s.kubelet-config-file') do |value| %>
+export FLAGS="$FLAGS --kubelet-arg=config=/var/vcap/jobs/k3s-server/config/kubelet-config.yaml"
+<% end %>
+
 
     echo $$ > $PIDFILE
 
@@ -138,10 +153,6 @@ export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.la
       $FLAGS \
       >>  $LOG_DIR/k3s-server.stdout.log \
       2>> $LOG_DIR/k3s-server.stderr.log
-
-
-    #TODO --kubelet-arg=" --hostname-override=<%= spec.ip %>" \
-
     ;;
 
   stop)

jobs/k3s-server/templates/config/audit-policy.yaml.erb

@@ -0,0 +1,3 @@
+<% if_p('k3s.audit-policy-file') do |value| %>
+<%= p('k3s.audit-policy-file') %>
+<% end %>

jobs/k3s-server/templates/config/kubelet-config.yaml.erb

@@ -0,0 +1,3 @@
+<% if_p('k3s.kubelet-config-file') do |value| %>
+<%= p('k3s.kubelet-config-file') %>
+<% end %>

vendir.lock

@@ -2,7 +2,7 @@ apiVersion: vendir.k14s.io/v1alpha1
 directories:
 - contents:
   - githubRelease:
-      url: https://api.github.com/repos/k3s-io/k3s/releases/60429322
+      url: https://api.github.com/repos/k3s-io/k3s/releases/63214645
     path: github.com/k3s-io/k3s
   - githubRelease:
       url: https://api.github.com/repos/derailed/k9s/releases/56095809

vendir.yml

@@ -11,7 +11,7 @@ directories:
   - path: github.com/k3s-io/k3s
     githubRelease:
       slug: k3s-io/k3s
-      tag: v1.21.10+k3s1 
+      tag: v1.21.11+k3s1 
       disableAutoChecksumValidation: true 
     includePaths:
     - "k3s"