diff --git a/jobs/k3s-agent/spec b/jobs/k3s-agent/spec index 8af395f0..d9431e01 100644 --- a/jobs/k3s-agent/spec +++ b/jobs/k3s-agent/spec @@ -19,13 +19,12 @@ templates: bin/setup-user-env.erb: bin/setup-user-env bin/k3s-killall.sh: bin/k3s-killall.sh - config/registries.yaml.erb: config/registries.yaml config/registry.ca.erb: config/registry.ca config/registry.cert.erb: config/registry.cert config/registry.key.erb: config/registry.key config/token.csv.erb: config/token.csv - + config/kubelet-config.yaml.erb: config/kubelet-config.yaml # Documentation https://bosh.io/docs/links.html @@ -35,13 +34,11 @@ consumes: type: k3s-server optional: false - provides: - name: k3s-agent type: k3s-agent properties: - containerd_registry: description: containerd registry configuration @@ -60,14 +57,24 @@ properties: k3s.v: description: "(logging) Number for the log level verbosity (default: 0)" default: 0 + k3s.bind-address value: description: "(listener) k3s bind address (default: 0.0.0.0)" default: 0.0.0.0 k3s.token: description: (cluster) Shared secret used to join a server or agent to a cluster [$K3S_TOKEN] - -## additional label and taints (added to k3s and bosh default labels) + +# Disable harware options on VxLAN interface + k3s.disable-vxlan-hardware-options: + description: Disable VxLAN harware options on private interface + default: + - tx-udp_tnl-segmentation + - tx-udp_tnl-csum-segmentation +# - tx-checksum-ip-generic + + +# additional label and taints (added to k3s and bosh default labels) k3s.node-labels: description: (agent/node) Registering and starting kubelet with set of labels format is key=value default: [] @@ -78,17 +85,16 @@ properties: k3s.set-provider-id-prefix: description: If set, the default provider id (k3s://- will be set as kubelet arg as ://- - -## drain options +# drain options k3s.drain.kubeconfig: description: bosh drain for agents need a k3s kubeconfig. If set, the drain will occur and use this kubeconfig k3s.drain.delete-emptydir-data: - description: continue even if there are pods using emptyDir (local data that will be deleted when the node is drained). + description: continue even if there are pods using emptyDir (local data that will be deleted when the node is drained). default: true k3s.drain.disable-eviction: - description: force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution + description: Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution default: false k3s.drain.grace-period: @@ -99,7 +105,6 @@ properties: description: Ignore DaemonSet-managed pods. default: true - k3s.drain.skip-wait-for-delete-timeout: description: If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip. default: 0 @@ -121,18 +126,15 @@ properties: description: if set, the bosh post-stop script wont leverage k3s-killall.sh script default: false - -#token-file +# token-file k3s.token-file-content: description: token-file content. see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file -## args - +# args k3s.kubelet-args: description: (agent/flags) Customized flag for kubelet process default: [] - k3s.kube-proxy-arg: description: (agent/flags) Customized flag for kube-proxy process default: [] @@ -140,18 +142,15 @@ properties: k3s.master_vip_api: description: externaly defined vip ip for HA k3s (enables multi master instance groups). This is used for public api access, tls-san, and agents to server communication -#kubelet config +# kubelet config k3s.kubelet-config-file: description: content of kubelet config file, to enable eg GraceFull Node Shutdown default: | apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration - shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s k3s.containerd_additional_env_vars: description: additional env vars to set for containerd (the key will be prefixed with CONTAINERD_, and set in k3s launch context default: [] - - diff --git a/jobs/k3s-agent/templates/bin/pre-start.erb b/jobs/k3s-agent/templates/bin/pre-start.erb index 5d0c2b39..28681d36 100644 --- a/jobs/k3s-agent/templates/bin/pre-start.erb +++ b/jobs/k3s-agent/templates/bin/pre-start.erb @@ -1,14 +1,54 @@ -#!/bin/sh -export JOB_DIR=/var/vcap/jobs/k3s-agent +#!/bin/bash + + +export JOB_DIR="/var/vcap/jobs/k3s-agent" /var/vcap/packages/k3s/k3s check-config # Setup ssh env vars ${JOB_DIR}/bin/setup-user-env - -#prepare a persistent directory so /etc/rancher/node paswword file is kept on bosh recreate +# Prepare a persistent directory so /etc/rancher/node paswword file is kept on bosh recreate mkdir -p /etc mkdir -p /var/vcap/store/k3s-node/etc/rancher -ln -sf /var/vcap/store/k3s-node/etc/rancher /etc/rancher +ln -sf /var/vcap/store/k3s-node/etc/rancher /etc/rancher + +set -e +# Set overlay IP +OVERLAY_IP="<%= spec.ip %>" +<% if spec.ip != spec.networks.marshal_dump.values.first.ip %> + OVERLAY_IP="<%= spec.networks.marshal_dump.values.last.ip %>" +<% end %> +<% if spec.ip != spec.networks.marshal_dump.values.last.ip %> + OVERLAY_IP="<%= spec.networks.marshal_dump.values.first.ip %>" +<% end %> + +# Disable VxLAN harware options on private interface +# (see: https://github.com/orange-cloudfoundry/paas-templates/issues/2062) +INTERFACE="$(ip --brief address show | grep "${OVERLAY_IP}" | awk '{print $1}')" + +#clean previous patch services +! rm -f /etc/systemd/system/ethtool-patch-*.service + +<% p('k3s.disable-vxlan-hardware-options').each do |option| %> + #--- Disable hardware option on private interface +OPTION="<%= option %>" +if [ "${OPTION}" != "" ] ; then +serviceFile="ethtool-patch-${INTERFACE}-${OPTION}.service" +cat > /etc/systemd/system/${serviceFile} << EOF +[Unit] +Description=Turn off ${OPTION} on ${INTERFACE} +After=sys-subsystem-net-devices-${INTERFACE}.device +[Install] +WantedBy=sys-subsystem-net-devices-${INTERFACE}.device +[Service] +Type=oneshot +ExecStart=/sbin/ethtool -K ${INTERFACE} ${OPTION} off +EOF + +#--- Start service +/usr/bin/systemctl enable ${serviceFile} +/usr/bin/systemctl start ${serviceFile} +fi +<% end %> -exit 0 +exit 0 \ No newline at end of file diff --git a/jobs/k3s-server/spec b/jobs/k3s-server/spec index 65477955..27bb894a 100644 --- a/jobs/k3s-server/spec +++ b/jobs/k3s-server/spec @@ -31,7 +31,6 @@ templates: config/audit-policy.yaml.erb: config/audit-policy.yaml config/kubelet-config.yaml.erb: config/kubelet-config.yaml - # Documentation https://bosh.io/docs/links.html # Tutorial https://gist.github.com/Amit-PivotalLabs/c39528248b8cdc4ba8e347f8aa68abb6 consumes: @@ -44,7 +43,6 @@ provides: type: k3s-server properties: - containerd_registry: description: containerd registry configuration @@ -60,9 +58,11 @@ properties: k3s.v: description: "(logging) Number for the log level verbosity (default: 0)" default: 0 + k3s.bind-address value: description: "(listener) k3s bind address (default: 0.0.0.0)" default: 0.0.0.0 + k3s.flannel-backend: description: (networking) One of 'none', 'vxlan', 'ipsec', or 'wireguard' default: vxlan @@ -76,7 +76,6 @@ properties: k3s.token: description: (cluster) Shared secret used to join a server or agent to a cluster [$K3S_TOKEN] - k3s.kubelet-args: description: (agent/flags) Customized flag for kubelet process default: [] @@ -84,7 +83,15 @@ properties: k3s.node_name_prefix: description: explicitly set k8s node name. If not set, - is set automatically. If set, name is - -#disable defaut k3s components +# Disable harware options on VxLAN interface + k3s.disable-vxlan-hardware-options: + description: Disable VxLAN harware options on private interface + default: + - tx-udp_tnl-segmentation + - tx-udp_tnl-csum-segmentation +# - tx-checksum-ip-generic + +# Disable defaut k3s components k3s.disable: description: (components) Do not deploy packaged components and delete any deployed components (valid itemms are coredns, servicelb, traefik, local-storage, metrics-server) default: [] @@ -92,13 +99,10 @@ properties: k3s.disable-cloud-controller: description: (components) If set, Disable k3s default cloud controller manager - k3s.set-provider-id-prefix: description: If set, the default provider id (k3s://- will be set as kubelet arg as ://- - - -## additional label and taints (added to k3s and bosh default labels) +# Additional label and taints (added to k3s and bosh default labels) k3s.node-labels: description: (agent/node) Registering and starting kubelet with set of labels default: [] @@ -106,14 +110,12 @@ properties: description: (agent/node) Registering kubelet with set of taints.format is key=value:Effect) default: [] -## additional manifests to apply +# Additional manifests to apply k3s.additional-manifests: default: [] description: array of additionnel yaml to apply at k3s start - ## drain options - k3s.drain.delete-emptydir-data: description: continue even if there are pods using emptyDir (local data that will be deleted when the node is drained). default: true @@ -152,22 +154,23 @@ properties: description: if set, the bosh post-stop script wont leverage k3s-killall.sh script default: false - - -## db options +# db options k3s.embedded-ha-etcd: description: if set, use embedded etcd in ha mode. requires an odd number of servers, overrides datastore-endpoints - + k3s.datastore-endpoint: description: Specify etcd, Mysql, Postgres, or Sqlite (default) data source name + k3s.datastore-cafile: description: TLS Certificate Authority file used to secure datastore backend communication + k3s.datastore-certfile: description: TLS certification file used to secure datastore backend communication + k3s.datastore-keyfile: description: TLS key file used to secure datastore backend communication -## args +# args k3s.kube-apiserver-arg: description: (flags) Customized flag for kube-apiserver process default: [] @@ -188,7 +191,7 @@ properties: description: (flags) Customized flag for kube-cloud-controller-manager process default: [] -#token-file +# token-file k3s.token-file-content: description: token-file content. see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file @@ -199,13 +202,12 @@ properties: description: array of additionnal tls-san for k8s api (added to default, ip, bosh-dns alias, or master vip if set) default: [] -#kubelet config +# kubelet config k3s.kubelet-config-file: description: content of kubelet config file, to enable eg GraceFull Node Shutdown default: | apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration - shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s @@ -213,7 +215,7 @@ properties: description: additional env vars (name/value map array) to set for containerd (the key will be prefixed with CONTAINERD_, and set in k3s launch context default: [] -#cidr config +# cidr config k3s.cluster-cidr: description: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)" k3s.service-cidr: @@ -221,9 +223,7 @@ properties: k3s.cluster-dns: description: "(networking) IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)" - - -#audit policy file +# audit policy file k3s.audit-policy-file: description: if set, defines and activate audit policy for k8s server default: | @@ -310,4 +310,3 @@ properties: # --docker (agent/runtime) Use docker instead of containerd # --node-ip value, -i value (agent/networking) IP address to advertise for node # --node-external-ip value (agent/networking) External IP address to advertise for node - diff --git a/jobs/k3s-server/templates/bin/pre-start.erb b/jobs/k3s-server/templates/bin/pre-start.erb index c655a535..440df4a1 100644 --- a/jobs/k3s-server/templates/bin/pre-start.erb +++ b/jobs/k3s-server/templates/bin/pre-start.erb @@ -1,22 +1,59 @@ -#!/bin/sh -export JOB_DIR=/var/vcap/jobs/k3s-server - +#!/bin/bash +export JOB_DIR="/var/vcap/jobs/k3s-server" /var/vcap/packages/k3s/k3s check-config # Setup ssh env vars ${JOB_DIR}/bin/setup-user-env -#prepare a persistent directory so /etc/rancher/node paswword file is kept on bosh recreate +# Prepare a persistent directory so /etc/rancher/node paswword file is kept on bosh recreate mkdir -p /etc mkdir -p /var/vcap/store/k3s-node/etc/rancher -ln -sf /var/vcap/store/k3s-node/etc/rancher /etc/rancher +ln -sf /var/vcap/store/k3s-node/etc/rancher /etc/rancher -#fix cert chmod +# Fix cert chmod chmod go-r /var/vcap/jobs/k3s-server/config/datastore-* -#copy additional manifest file - +# Copy additional manifest file mkdir -p /var/vcap/store/k3s-server/ -cp -f $JOB_DIR/config/additional-manifest.yaml /var/vcap/store/k3s-server/server/manifests/additional-manifest.yaml +cp -f ${JOB_DIR}/config/additional-manifest.yaml /var/vcap/store/k3s-server/server/manifests/additional-manifest.yaml + +set -e +# Set overlay IP +OVERLAY_IP="<%= spec.ip %>" +<% if spec.ip != spec.networks.marshal_dump.values.first.ip %> + OVERLAY_IP="<%= spec.networks.marshal_dump.values.last.ip %>" +<% end %> +<% if spec.ip != spec.networks.marshal_dump.values.last.ip %> + OVERLAY_IP="<%= spec.networks.marshal_dump.values.first.ip %>" +<% end %> + +# Disable VxLAN harware options on private interface +# (see: https://github.com/orange-cloudfoundry/paas-templates/issues/2062) +INTERFACE="$(ip --brief address show | grep "${OVERLAY_IP}" | awk '{print $1}')" + +#clean previous patch services +! rm -f /etc/systemd/system/ethtool-patch-*.service + +<% p('k3s.disable-vxlan-hardware-options').each do |option| %> + #--- Disable hardware option on private interface +OPTION="<%= option %>" +if [ "${OPTION}" != "" ] ; then +serviceFile="ethtool-patch-${INTERFACE}-${OPTION}.service" +cat > /etc/systemd/system/${serviceFile} << EOF +[Unit] +Description=Turn off ${OPTION} on ${INTERFACE} +After=sys-subsystem-net-devices-${INTERFACE}.device +[Install] +WantedBy=sys-subsystem-net-devices-${INTERFACE}.device +[Service] +Type=oneshot +ExecStart=/sbin/ethtool -K ${INTERFACE} ${OPTION} off +EOF + +#--- Start service +/usr/bin/systemctl enable ${serviceFile} +/usr/bin/systemctl start ${serviceFile} +fi +<% end %> -exit 0 +exit 0 \ No newline at end of file