diff --git a/config/blobs.yml b/config/blobs.yml index 7da5dfe2..5a495e5f 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -1,11 +1,11 @@ k3s-images/k3s-airgap-images-amd64.tar: - size: 498181120 - object_id: 3f192c02-495c-47ac-5c74-5849d3c86f47 - sha: sha256:a10f1f58851c7f5eeae0699ca761cad30d8ec297ca2d39ad692aecc87c01f74f + size: 500852736 + object_id: fe6c595a-14b1-441d-643d-caa140b83ad0 + sha: sha256:74e5a861e0c977d736ebb21b1a1a9f30f7236c00ebc590ac90a476972e93c9a0 k3s/k3s: - size: 49082368 - object_id: a84494e2-29b3-43e9-7f1c-f498e758e75b - sha: sha256:7a9cf9a0f2318b09f8b449f96440fdd3d877fcfd5e25b275a0e705400641ce3d + size: 49090560 + object_id: 42a5dfd9-5a02-4bed-473d-07e6e9885393 + sha: sha256:6450ece4cf5d7ada78c0ef9e27b45bdf5aa1049fccc8e5fc55720ae1087bf0a8 k9s/k9s: size: 55410688 object_id: 0b3b0840-1db4-4ac5-53b1-0f62a80862cb diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec index 15b2c07c..2246aa50 100644 --- a/jobs/k3s-agent/spec +++ b/jobs/k3s-agent/spec @@ -24,6 +24,8 @@ templates: config/registry.cert.erb: config/registry.cert config/registry.key.erb: config/registry.key config/token.csv.erb: config/token.csv + + config/kubelet-config.yaml.erb: config/kubelet-config.yaml # Documentation https://bosh.io/docs/links.html # Tutorial https://gist.github.com/Amit-PivotalLabs/c39528248b8cdc4ba8e347f8aa68abb6 @@ -100,7 +102,7 @@ properties: k3s.drain.timeout: description: The length of time to wait before giving up, zero means infinite - default: 0 + default: 90s k3s.drain.pod-selector: #Not implemented description: Label selector to filter pods on the node @@ -126,3 +128,14 @@ properties: k3s.master_vip_api: description: externaly defined vip ip for HA k3s (enables multi master instance groups). This is used for public api access, tls-san, and agents to server communication +#kubelet config + k3s.kubelet-config-file: + description: content of kubelet config file, to enable eg GraceFull Node Shutdown + default: | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + + diff --git a/jobs/k3s-agent/templates/bin/ctl.erb b/jobs/k3s-agent/templates/bin/ctl.erb index 2959f04e..b0d2e804 100755 --- a/jobs/k3s-agent/templates/bin/ctl.erb +++ b/jobs/k3s-agent/templates/bin/ctl.erb @@ -75,6 +75,11 @@ export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.fi export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.last.ip %>" <% end %> +<% if_p('k3s.kubelet-config-file') do |value| %> +export FLAGS="$FLAGS --kubelet-arg=config=/var/vcap/jobs/k3s-agent/config/kubelet-config.yaml" +<% end %> + + ulimit -n 1048576 # open files ulimit -u unlimited # num processes diff --git a/jobs/k3s-agent/templates/config/kubelet-config.yaml.erb b/jobs/k3s-agent/templates/config/kubelet-config.yaml.erb new file mode 100644 index 00000000..5b0747bd --- /dev/null +++ b/jobs/k3s-agent/templates/config/kubelet-config.yaml.erb @@ -0,0 +1,3 @@ +<% if_p('k3s.kubelet-config-file') do |value| %> +<%= p('k3s.kubelet-config-file') %> +<% end %> \ No newline at end of file diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec index d3a23a58..0744a58c 100644 --- a/jobs/k3s-server/spec +++ b/jobs/k3s-server/spec @@ -27,6 +27,8 @@ templates: config/datastore-keyfile.erb: config/datastore-keyfile config/additional-manifest.yaml.erb: config/additional-manifest.yaml + config/audit-policy.yaml.erb: config/audit-policy.yaml + config/kubelet-config.yaml.erb: config/kubelet-config.yaml # Documentation https://bosh.io/docs/links.html @@ -116,7 +118,7 @@ properties: k3s.drain.timeout: description: The length of time to wait before giving up, zero means infinite - default: 0 + default: 90s k3s.drain.pod-selector: #Not implemented description: Label selector to filter pods on the node @@ -161,7 +163,80 @@ properties: k3s.additional_tls_sans: description: array of additionnal tls-san for k8s api (added to default, ip, bosh-dns alias, or master vip if set) default: [] - + +#kubelet config + k3s.kubelet-config-file: + description: content of kubelet config file, to enable eg GraceFull Node Shutdown + default: | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + +#audit policy file + k3s.audit-policy-file: + description: if set, defines and activate audit policy for k8s server + default: | + apiVersion: audit.k8s.io/v1 # This is required. + kind: Policy + + # Prevent requests in the RequestReceived stage from generating audit events. + omitStages: + - "RequestReceived" + + rules: + # Log "pods/log", "pods/status" at Metadata level + - level: Metadata + resources: + - group: "" + resources: ["pods/log", "pods/status"] + # Exclude logging requests to a configmap called "controller-config" + - level: None + resources: + - group: "" + resources: ["configmaps"] + resourceNames: ["controller-config"] + # Don't log watch requests by the "system:kube-proxy" on endpoints or services + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core API group + resources: ["endpoints", "services"] + # Log deployment changes at RequestResponse level + - level: Metadata + resources: + - group: "" + resources: ["deployments"] + # Log service changes at metadata level + - level: Metadata + resources: + - group: "" + resources: ["services"] + # Log the request body of configmap changes in the kube-system namespace. + - level: Request + resources: + - group: "" # core API group + resources: ["configmaps"] + # You can use an empty string [""] to select resources not associated with a namespace. + namespaces: ["kube-system"] + # Log configmap and secret changes in all other namespaces at the Metadata level. + - level: Metadata + resources: + - group: "" # core API group + resources: ["secrets", "configmaps"] + # Log all other resources in core and extensions at the Request level. + - level: Request + resources: + - group: "" # core API group + - group: "extensions" # Version of group should NOT be included. + # A wild-card rule to log all other requests at the Metadata level. + - level: Metadata + # Long-running requests like watches that fall under this rule will not + # generate an audit event in RequestReceived. + omitStages: + - "RequestReceived" #done # -v value (logging) Number for the log level verbosity (default: 0) diff --git a/jobs/k3s-server/templates/bin/ctl.erb b/jobs/k3s-server/templates/bin/ctl.erb index ddc06bad..6003a408 100755 --- a/jobs/k3s-server/templates/bin/ctl.erb +++ b/jobs/k3s-server/templates/bin/ctl.erb @@ -109,6 +109,21 @@ export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.fi export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.last.ip %>" <% end %> +<% if_p('k3s.audit-policy-file') do |value| %> +export FLAGS="$FLAGS --kube-apiserver-arg=audit-log-path=/var/vcap/sys/log/k3s-server/audit.log" +export FLAGS="$FLAGS --kube-apiserver-arg=audit-policy-file=/var/vcap/jobs/k3s-server/config/audit-policy.yaml" + +export FLAGS="$FLAGS --kube-apiserver-arg=audit-log-maxage=15" +export FLAGS="$FLAGS --kube-apiserver-arg=audit-log-maxbackup=5" +export FLAGS="$FLAGS --kube-apiserver-arg=audit-log-maxsize=10" + + +<% end %> + +<% if_p('k3s.kubelet-config-file') do |value| %> +export FLAGS="$FLAGS --kubelet-arg=config=/var/vcap/jobs/k3s-server/config/kubelet-config.yaml" +<% end %> + echo $$ > $PIDFILE @@ -138,10 +153,6 @@ export FLAGS="$FLAGS --node-external-ip=<%= spec.networks.marshal_dump.values.la $FLAGS \ >> $LOG_DIR/k3s-server.stdout.log \ 2>> $LOG_DIR/k3s-server.stderr.log - - - #TODO --kubelet-arg=" --hostname-override=<%= spec.ip %>" \ - ;; stop) diff --git a/jobs/k3s-server/templates/config/audit-policy.yaml.erb b/jobs/k3s-server/templates/config/audit-policy.yaml.erb new file mode 100644 index 00000000..26aba458 --- /dev/null +++ b/jobs/k3s-server/templates/config/audit-policy.yaml.erb @@ -0,0 +1,3 @@ +<% if_p('k3s.audit-policy-file') do |value| %> +<%= p('k3s.audit-policy-file') %> +<% end %> \ No newline at end of file diff --git a/jobs/k3s-server/templates/config/kubelet-config.yaml.erb b/jobs/k3s-server/templates/config/kubelet-config.yaml.erb new file mode 100644 index 00000000..5b0747bd --- /dev/null +++ b/jobs/k3s-server/templates/config/kubelet-config.yaml.erb @@ -0,0 +1,3 @@ +<% if_p('k3s.kubelet-config-file') do |value| %> +<%= p('k3s.kubelet-config-file') %> +<% end %> \ No newline at end of file diff --git a/vendir.lock b/vendir.lock index 757cb50e..5bf94f6d 100755 --- a/vendir.lock +++ b/vendir.lock @@ -2,7 +2,7 @@ apiVersion: vendir.k14s.io/v1alpha1 directories: - contents: - githubRelease: - url: https://api.github.com/repos/k3s-io/k3s/releases/60429322 + url: https://api.github.com/repos/k3s-io/k3s/releases/63214645 path: github.com/k3s-io/k3s - githubRelease: url: https://api.github.com/repos/derailed/k9s/releases/56095809 diff --git a/vendir.yml b/vendir.yml index 8eaa61cd..af41604d 100644 --- a/vendir.yml +++ b/vendir.yml @@ -11,7 +11,7 @@ directories: - path: github.com/k3s-io/k3s githubRelease: slug: k3s-io/k3s - tag: v1.21.10+k3s1 + tag: v1.21.11+k3s1 disableAutoChecksumValidation: true includePaths: - "k3s"