Skip to content
This repository has been archived by the owner on Oct 16, 2024. It is now read-only.

Commit

Permalink
Merge pull request #29 from poblin-orange/feature-disable-tcp-checksum
Browse files Browse the repository at this point in the history 
feature-disable-tcp-checksum
  • Loading branch information
@poblin-orange
poblin-orange authored Sep 25, 2023
2 parents 99d0b2c + 31a70b1 commit 8bec696
Show file tree
Hide file tree
Showing 4 changed files with 135 additions and 60 deletions.
39 changes: 19 additions & 20 deletions jobs/k3s-agent/spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,12 @@ templates:
bin/setup-user-env.erb: bin/setup-user-env
bin/k3s-killall.sh: bin/k3s-killall.sh


config/registries.yaml.erb: config/registries.yaml
config/registry.ca.erb: config/registry.ca
config/registry.cert.erb: config/registry.cert
config/registry.key.erb: config/registry.key
config/token.csv.erb: config/token.csv

config/kubelet-config.yaml.erb: config/kubelet-config.yaml

# Documentation https://bosh.io/docs/links.html
Expand All @@ -35,13 +34,11 @@ consumes:
type: k3s-server
optional: false


provides:
- name: k3s-agent
type: k3s-agent

properties:

containerd_registry:
description: containerd registry configuration

Expand All @@ -60,14 +57,24 @@ properties:
k3s.v:
description: "(logging) Number for the log level verbosity (default: 0)"
default: 0

k3s.bind-address value:
description: "(listener) k3s bind address (default: 0.0.0.0)"
default: 0.0.0.0

k3s.token:
description: (cluster) Shared secret used to join a server or agent to a cluster [$K3S_TOKEN]

## additional label and taints (added to k3s and bosh default labels)

# Disable harware options on VxLAN interface
k3s.disable-vxlan-hardware-options:
description: Disable VxLAN harware options on private interface
default:
- tx-udp_tnl-segmentation
- tx-udp_tnl-csum-segmentation
# - tx-checksum-ip-generic


# additional label and taints (added to k3s and bosh default labels)
k3s.node-labels:
description: (agent/node) Registering and starting kubelet with set of labels format is key=value
default: []
Expand All @@ -78,17 +85,16 @@ properties:
k3s.set-provider-id-prefix:
description: If set, the default provider id (k3s://<instance>-<index> will be set as kubelet arg as <prefix>://<instance>-<index>


## drain options
# drain options
k3s.drain.kubeconfig:
description: bosh drain for agents need a k3s kubeconfig. If set, the drain will occur and use this kubeconfig

k3s.drain.delete-emptydir-data:
description: continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).
description: continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).
default: true

k3s.drain.disable-eviction:
description: force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution
description: Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution
default: false

k3s.drain.grace-period:
Expand All @@ -99,7 +105,6 @@ properties:
description: Ignore DaemonSet-managed pods.
default: true


k3s.drain.skip-wait-for-delete-timeout:
description: If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.
default: 0
Expand All @@ -121,37 +126,31 @@ properties:
description: if set, the bosh post-stop script wont leverage k3s-killall.sh script
default: false


#token-file
# token-file
k3s.token-file-content:
description: token-file content. see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file

## args

# args
k3s.kubelet-args:
description: (agent/flags) Customized flag for kubelet process
default: []


k3s.kube-proxy-arg:
description: (agent/flags) Customized flag for kube-proxy process
default: []

k3s.master_vip_api:
description: externaly defined vip ip for HA k3s (enables multi master instance groups). This is used for public api access, tls-san, and agents to server communication

#kubelet config
# kubelet config
k3s.kubelet-config-file:
description: content of kubelet config file, to enable eg GraceFull Node Shutdown
default: |
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration

shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s

k3s.containerd_additional_env_vars:
description: additional env vars to set for containerd (the key will be prefixed with CONTAINERD_, and set in k3s launch context
default: []


52 changes: 46 additions & 6 deletions jobs/k3s-agent/templates/bin/pre-start.erb
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,54 @@
#!/bin/sh
export JOB_DIR=/var/vcap/jobs/k3s-agent
#!/bin/bash


export JOB_DIR="/var/vcap/jobs/k3s-agent"
/var/vcap/packages/k3s/k3s check-config

# Setup ssh env vars
${JOB_DIR}/bin/setup-user-env


#prepare a persistent directory so /etc/rancher/node paswword file is kept on bosh recreate
# Prepare a persistent directory so /etc/rancher/node paswword file is kept on bosh recreate
mkdir -p /etc
mkdir -p /var/vcap/store/k3s-node/etc/rancher
ln -sf /var/vcap/store/k3s-node/etc/rancher /etc/rancher
ln -sf /var/vcap/store/k3s-node/etc/rancher /etc/rancher

set -e
# Set overlay IP
OVERLAY_IP="<%= spec.ip %>"
<% if spec.ip != spec.networks.marshal_dump.values.first.ip %>
OVERLAY_IP="<%= spec.networks.marshal_dump.values.last.ip %>"
<% end %>
<% if spec.ip != spec.networks.marshal_dump.values.last.ip %>
OVERLAY_IP="<%= spec.networks.marshal_dump.values.first.ip %>"
<% end %>

# Disable VxLAN harware options on private interface
# (see: https://github.com/orange-cloudfoundry/paas-templates/issues/2062)
INTERFACE="$(ip --brief address show | grep "${OVERLAY_IP}" | awk '{print $1}')"

#clean previous patch services
! rm -f /etc/systemd/system/ethtool-patch-*.service

<% p('k3s.disable-vxlan-hardware-options').each do |option| %>
#--- Disable hardware option on private interface
OPTION="<%= option %>"
if [ "${OPTION}" != "" ] ; then
serviceFile="ethtool-patch-${INTERFACE}-${OPTION}.service"
cat > /etc/systemd/system/${serviceFile} << EOF
[Unit]
Description=Turn off ${OPTION} on ${INTERFACE}
After=sys-subsystem-net-devices-${INTERFACE}.device
[Install]
WantedBy=sys-subsystem-net-devices-${INTERFACE}.device
[Service]
Type=oneshot
ExecStart=/sbin/ethtool -K ${INTERFACE} ${OPTION} off
EOF

#--- Start service
/usr/bin/systemctl enable ${serviceFile}
/usr/bin/systemctl start ${serviceFile}
fi
<% end %>

exit 0
exit 0
47 changes: 23 additions & 24 deletions jobs/k3s-server/spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ templates:
config/audit-policy.yaml.erb: config/audit-policy.yaml
config/kubelet-config.yaml.erb: config/kubelet-config.yaml


# Documentation https://bosh.io/docs/links.html
# Tutorial https://gist.github.com/Amit-PivotalLabs/c39528248b8cdc4ba8e347f8aa68abb6
consumes:
Expand All @@ -44,7 +43,6 @@ provides:
type: k3s-server

properties:

containerd_registry:
description: containerd registry configuration

Expand All @@ -60,9 +58,11 @@ properties:
k3s.v:
description: "(logging) Number for the log level verbosity (default: 0)"
default: 0

k3s.bind-address value:
description: "(listener) k3s bind address (default: 0.0.0.0)"
default: 0.0.0.0

k3s.flannel-backend:
description: (networking) One of 'none', 'vxlan', 'ipsec', or 'wireguard'
default: vxlan
Expand All @@ -76,44 +76,46 @@ properties:
k3s.token:
description: (cluster) Shared secret used to join a server or agent to a cluster [$K3S_TOKEN]


k3s.kubelet-args:
description: (agent/flags) Customized flag for kubelet process
default: []

k3s.node_name_prefix:
description: explicitly set k8s node name. If not set, <instance-group-name>-<index> is set automatically. If set, name is <node_name_prefix>-<index>

#disable defaut k3s components
# Disable harware options on VxLAN interface
k3s.disable-vxlan-hardware-options:
description: Disable VxLAN harware options on private interface
default:
- tx-udp_tnl-segmentation
- tx-udp_tnl-csum-segmentation
# - tx-checksum-ip-generic

# Disable defaut k3s components
k3s.disable:
description: (components) Do not deploy packaged components and delete any deployed components (valid itemms are coredns, servicelb, traefik, local-storage, metrics-server)
default: []

k3s.disable-cloud-controller:
description: (components) If set, Disable k3s default cloud controller manager


k3s.set-provider-id-prefix:
description: If set, the default provider id (k3s://<instance>-<index> will be set as kubelet arg as <prefix>://<instance>-<index>



## additional label and taints (added to k3s and bosh default labels)
# Additional label and taints (added to k3s and bosh default labels)
k3s.node-labels:
description: (agent/node) Registering and starting kubelet with set of labels
default: []
k3s.node-taints:
description: (agent/node) Registering kubelet with set of taints.format is key=value:Effect)
default: []

## additional manifests to apply
# Additional manifests to apply
k3s.additional-manifests:
default: []
description: array of additionnel yaml to apply at k3s start


## drain options

k3s.drain.delete-emptydir-data:
description: continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).
default: true
Expand Down Expand Up @@ -152,22 +154,23 @@ properties:
description: if set, the bosh post-stop script wont leverage k3s-killall.sh script
default: false



## db options
# db options
k3s.embedded-ha-etcd:
description: if set, use embedded etcd in ha mode. requires an odd number of servers, overrides datastore-endpoints

k3s.datastore-endpoint:
description: Specify etcd, Mysql, Postgres, or Sqlite (default) data source name

k3s.datastore-cafile:
description: TLS Certificate Authority file used to secure datastore backend communication

k3s.datastore-certfile:
description: TLS certification file used to secure datastore backend communication

k3s.datastore-keyfile:
description: TLS key file used to secure datastore backend communication

## args
# args
k3s.kube-apiserver-arg:
description: (flags) Customized flag for kube-apiserver process
default: []
Expand All @@ -188,7 +191,7 @@ properties:
description: (flags) Customized flag for kube-cloud-controller-manager process
default: []

#token-file
# token-file
k3s.token-file-content:
description: token-file content. see https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file

Expand All @@ -199,31 +202,28 @@ properties:
description: array of additionnal tls-san for k8s api (added to default, ip, bosh-dns alias, or master vip if set)
default: []

#kubelet config
# kubelet config
k3s.kubelet-config-file:
description: content of kubelet config file, to enable eg GraceFull Node Shutdown
default: |
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration

shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s

k3s.containerd_additional_env_vars:
description: additional env vars (name/value map array) to set for containerd (the key will be prefixed with CONTAINERD_, and set in k3s launch context
default: []

#cidr config
# cidr config
k3s.cluster-cidr:
description: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)"
k3s.service-cidr:
description: "(networking) IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16)"
k3s.cluster-dns:
description: "(networking) IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)"



#audit policy file
# audit policy file
k3s.audit-policy-file:
description: if set, defines and activate audit policy for k8s server
default: |
Expand Down Expand Up @@ -310,4 +310,3 @@ properties:
# --docker (agent/runtime) Use docker instead of containerd
# --node-ip value, -i value (agent/networking) IP address to advertise for node
# --node-external-ip value (agent/networking) External IP address to advertise for node

Loading

0 comments on commit 8bec696

Please sign in to comment.