Skip to content

Commit

Permalink
Init eks support
Browse files Browse the repository at this point in the history
Signed-off-by: Rodrigo Reis <[email protected]>
Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]>
  • Loading branch information
mikkeloscar committed Jun 24, 2024
1 parent 6f8ce27 commit 092af3c
Show file tree
Hide file tree
Showing 83 changed files with 1,729 additions and 232 deletions.
1,037 changes: 812 additions & 225 deletions cluster/cluster.yaml

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions cluster/config-defaults.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1101,3 +1101,10 @@ sysctl_settings: ""

# enables/disables the minDomains field for pod topology spread.
min_domains_in_pod_topology_spread_enabled: "true"

eks: "false"
eks_control_plane_logging: "false"
eks_ip_family: "ipv4"
# prefix delegation can only be configured for ipv4. For ipv6 it can only be
# true.
aws_vpc_cni_prefix_delegation: "true"
256 changes: 256 additions & 0 deletions cluster/manifests/01-aws-node/daemonset.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
{{- if eq .Cluster.ConfigItems.eks "true" }}
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
app.kubernetes.io/instance: aws-vpc-cni
app.kubernetes.io/name: aws-node
app.kubernetes.io/version: v1.15.1
k8s-app: aws-node
application: kubernetes
component: aws-node
name: aws-node
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: aws-node
template:
metadata:
labels:
app.kubernetes.io/instance: aws-vpc-cni
app.kubernetes.io/name: aws-node
k8s-app: aws-node
application: kubernetes
component: aws-node
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
- key: kubernetes.io/arch
operator: In
values:
- amd64
- arm64
- key: eks.amazonaws.com/compute-type
operator: NotIn
values:
- fargate
containers:
- env:
- name: ADDITIONAL_ENI_TAGS
value: '{}'
- name: ANNOTATE_POD_IP
value: "false"
- name: AWS_VPC_CNI_NODE_PORT_SUPPORT
value: "true"
- name: AWS_VPC_ENI_MTU
value: "9001"
- name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG
value: "false"
- name: AWS_VPC_K8S_CNI_EXTERNALSNAT
value: "false"
- name: AWS_VPC_K8S_CNI_LOGLEVEL
value: DEBUG
- name: AWS_VPC_K8S_CNI_LOG_FILE
value: /host/var/log/aws-routed-eni/ipamd.log
- name: AWS_VPC_K8S_CNI_RANDOMIZESNAT
value: prng
- name: AWS_VPC_K8S_CNI_VETHPREFIX
value: eni
- name: AWS_VPC_K8S_PLUGIN_LOG_FILE
value: /var/log/aws-routed-eni/plugin.log
- name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL
value: DEBUG
- name: CLUSTER_NAME
value: "{{ .Cluster.ID | eksID }}"
- name: DISABLE_INTROSPECTION
value: "false"
- name: DISABLE_METRICS
value: "false"
- name: DISABLE_NETWORK_RESOURCE_PROVISIONING
value: "false"
- name: ENABLE_IPv4
value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}true{{else}}false{{end}}"
- name: ENABLE_IPv6
value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}"
- name: ENABLE_POD_ENI
value: "false"
- name: ENABLE_PREFIX_DELEGATION
value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}{{.Cluster.ConfigItems.aws_vpc_cni_prefix_delegation}}{{else}}true{{end}}"
- name: NETWORK_POLICY_ENFORCING_MODE
value: standard
- name: VPC_CNI_VERSION
value: v1.17.1
- name: VPC_ID
value: "{{ .Cluster.ConfigItems.vpc_id }}"
- name: WARM_ENI_TARGET
value: "1"
- name: WARM_PREFIX_TARGET
value: "1"
- name: MY_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: MY_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon-k8s-cni:v1.17.1-eksbuild.1
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- /app/grpc-health-probe
- -addr=:50051
- -connect-timeout=5s
- -rpc-timeout=5s
failureThreshold: 3
initialDelaySeconds: 60
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
name: aws-node
ports:
- containerPort: 61678
hostPort: 61678
name: metrics
protocol: TCP
readinessProbe:
exec:
command:
- /app/grpc-health-probe
- -addr=:50051
- -connect-timeout=5s
- -rpc-timeout=5s
failureThreshold: 3
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
resources:
requests:
cpu: 25m
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
- mountPath: /host/var/log/aws-routed-eni
name: log-dir
- mountPath: /var/run/aws-node
name: run-dir
- mountPath: /run/xtables.lock
name: xtables-lock
- args:
- --enable-ipv6={{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}
- --enable-network-policy=false
- --enable-cloudwatch-logs=false
- --enable-policy-event-logs=false
- --metrics-bind-addr=:8162
- --health-probe-bind-addr=:8163
- --conntrack-cache-cleanup-period=300
env:
- name: MY_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon/aws-network-policy-agent:v1.1.0-eksbuild.1
imagePullPolicy: IfNotPresent
name: aws-eks-nodeagent
resources:
requests:
cpu: 25m
securityContext:
capabilities:
add:
- NET_ADMIN
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /sys/fs/bpf
name: bpf-pin-path
- mountPath: /var/log/aws-routed-eni
name: log-dir
- mountPath: /var/run/aws-node
name: run-dir
dnsPolicy: ClusterFirst
hostNetwork: true
initContainers:
- env:
- name: DISABLE_TCP_EARLY_DEMUX
value: "false"
- name: ENABLE_IPv6
value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}"
image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon-k8s-cni-init:v1.17.1-eksbuild.1
imagePullPolicy: IfNotPresent
name: aws-vpc-cni-init
resources:
requests:
cpu: 25m
securityContext:
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
priorityClassName: system-node-critical
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: aws-node
serviceAccountName: aws-node
terminationGracePeriodSeconds: 10
tolerations:
- operator: Exists
volumes:
- hostPath:
path: /sys/fs/bpf
type: ""
name: bpf-pin-path
- hostPath:
path: /opt/cni/bin
type: ""
name: cni-bin-dir
- hostPath:
path: /etc/kubernetes/cni/net.d #/etc/cni/net.d
type: ""
name: cni-net-dir
- hostPath:
path: /var/log/aws-routed-eni
type: DirectoryOrCreate
name: log-dir
- hostPath:
path: /var/run/aws-node
type: DirectoryOrCreate
name: run-dir
- hostPath:
path: /run/xtables.lock
type: ""
name: xtables-lock
updateStrategy:
rollingUpdate:
maxSurge: 0
maxUnavailable: 10%
type: RollingUpdate
{{- end }}
13 changes: 13 additions & 0 deletions cluster/manifests/01-aws-node/sa.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{{- if eq .Cluster.ConfigItems.eks "true"}}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: aws-node
namespace: kube-system
labels:
application: kubernetes
component: aws-node
annotations:
eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{.Cluster.LocalID}}-aws-node"
{{- end}}
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ data:
{{ else }}
forward . /etc/resolv.conf
{{ end }}
pprof 127.0.0.1:9156
pprof :9156
cache 30
reload
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,11 @@ spec:
spec:
initContainers:
- name: ensure-apiserver
{{- if eq .Cluster.ConfigItems.eks "true" }}
image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/ensure-apiserver:master-6
{{- else }}
image: container-registry.zalando.net/teapot/ensure-apiserver:master-6
{{- end }}
resources:
requests:
cpu: 1m
Expand Down Expand Up @@ -57,7 +61,11 @@ spec:
command:
- dig
- "+short"
{{- if and (eq .Cluster.ConfigItems.eks "true") (eq .Cluster.ConfigItems.eks_ip_family "ipv4") }}
- "@127.0.0.1"
{{- else }}
- "::1"
{{- end }}
- "kubernetes.default.svc.cluster.local"
initialDelaySeconds: 60
periodSeconds: 10
Expand All @@ -78,7 +86,8 @@ spec:
image: container-registry.zalando.net/teapot/unbound-telemetry:master-5
args:
- tcp
- --bind=0.0.0.0:9054
# TODO: ipv6
- --bind=:9054
ports:
- name: metrics
containerPort: 9054
Expand All @@ -92,7 +101,11 @@ spec:
{{ end }}
{{ if eq .Cluster.ConfigItems.dns_cache "dnsmasq" }}
- name: dnsmasq
{{- if eq .Cluster.ConfigItems.eks "true" }}
image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/k8s-dns-dnsmasq-nanny:1.17.4-master-15
{{- else }}
image: container-registry.zalando.net/teapot/k8s-dns-dnsmasq-nanny:1.17.4-master-15
{{- end }}
securityContext:
privileged: true
livenessProbe:
Expand All @@ -118,8 +131,8 @@ spec:
- --neg-ttl=60
# send requests to the last server first, only fallback to the previous ones if it's unreachable
- --strict-order
- --server=10.5.0.11#53
- --server=127.0.0.1#9254
- --server=10.5.0.11#53 # TODO: fix this for ipv6
- --server={{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}127.0.0.1{{else}}::1{{end}}#9254
ports:
- containerPort: 53
name: dns
Expand All @@ -134,7 +147,11 @@ spec:
cpu: {{.Cluster.ConfigItems.dns_dnsmasq_cpu}}
memory: {{.Cluster.ConfigItems.dns_dnsmasq_mem}}
- name: sidecar
{{- if eq .Cluster.ConfigItems.eks "true" }}
image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/k8s-dns-sidecar:1.17.4-master-15
{{- else }}
image: container-registry.zalando.net/teapot/k8s-dns-sidecar:1.17.4-master-15
{{- end }}
securityContext:
privileged: true
livenessProbe:
Expand All @@ -149,6 +166,7 @@ spec:
args:
- --v=2
- --logtostderr
# TODO: ipv6
- --probe=dnsmasq,127.0.0.1:9254,ec2.amazonaws.com,5,A
- --prometheus-port=9054
ports:
Expand All @@ -163,7 +181,11 @@ spec:
memory: {{.Cluster.ConfigItems.dns_dnsmasq_sidecar_mem}}
{{ end }}
- name: coredns
{{- if eq .Cluster.ConfigItems.eks "true" }}
image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/coredns:1.11.3-master-24
{{- else }}
image: container-registry.zalando.net/teapot/coredns:1.11.3-master-24
{{- end }}
args: [ "-conf", "/etc/coredns/Corefile" ]
env:
- name: ZONE
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@ metadata:
spec:
selector:
component: coredns
{{- if ne .Cluster.ConfigItems.eks "true"}}
# TODO: what to do with eks service range?
clusterIP: 10.5.0.11
{{- end}}
ports:
- name: dns
port: 53
Expand Down
Loading

0 comments on commit 092af3c

Please sign in to comment.