diff --git a/cluster/cluster.yaml b/cluster/cluster.yaml index 4d2517324e..3c5aa9e8ce 100644 --- a/cluster/cluster.yaml +++ b/cluster/cluster.yaml @@ -5,6 +5,267 @@ Metadata: InfrastructureComponent: "true" application: "kubernetes" Resources: +{{ if eq .Cluster.ConfigItems.eks "true" }} + EKSClusterRole: + Type: AWS::IAM::Role + Properties: + RoleName: "{{.Cluster.LocalID}}-eks-cluster-role" + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - eks.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonEKSClusterPolicy + - arn:aws:iam::aws:policy/AmazonEKSVPCResourceController + + EKSControlPlaneSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + # GroupDescription: "{{ .Cluster.Alias }}-eks-control-plane" + GroupDescription: "{{ .Cluster.Alias }}-control-plane" + # TODO: + # SecurityGroupIngress: + Tags: + - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}' + Value: owned + VpcId: "{{.Cluster.ConfigItems.vpc_id}}" + EKSWorkerSecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + # GroupDescription: "{{ .Cluster.ID }}-eks-worker-sg" + GroupDescription: "{{ .Cluster.ID }}-worker-sg" + SecurityGroupIngress: + - CidrIp: 0.0.0.0/0 + FromPort: -1 + IpProtocol: icmp + ToPort: -1 + - CidrIp: {{ if eq .Cluster.ConfigItems.ssh_vpc_only "true" }}"{{.Values.vpc_ipv4_cidr}}"{{ else }}"0.0.0.0/0"{{ end }} + FromPort: 22 + IpProtocol: tcp + ToPort: 22 +{{- if index .Cluster.ConfigItems "open_sg_ingress_ranges" }} +{{- range $index, $element := sgIngressRanges .Cluster.ConfigItems.open_sg_ingress_ranges }} + - CidrIp: {{ $element.CIDR }} + FromPort: {{ $element.FromPort }} + IpProtocol: {{ $element.Protocol }} + ToPort: {{ $element.ToPort }} +{{- end }} +{{- end }} + - CidrIp: {{ if eq .Cluster.ConfigItems.kube_aws_ingress_controller_nlb_enabled "true" }}"0.0.0.0/0"{{else}}"{{.Values.vpc_ipv4_cidr}}"{{end}} + FromPort: 9998 + IpProtocol: tcp + ToPort: 9999 +{{- if ne .Cluster.ConfigItems.skipper_redis_replicas "0"}} + - CidrIp: "{{.Values.vpc_ipv4_cidr}}" + FromPort: 9990 + IpProtocol: tcp + ToPort: 9990 + - CidrIp: "{{.Values.vpc_ipv4_cidr}}" + FromPort: 9990 + IpProtocol: udp + ToPort: 9990 +{{- end }} + - CidrIp: "{{.Values.vpc_ipv4_cidr}}" + FromPort: 10248 + IpProtocol: tcp + ToPort: 10248 + - CidrIp: "{{.Values.vpc_ipv4_cidr}}" + FromPort: 9054 + IpProtocol: tcp + ToPort: 9054 + - CidrIp: "{{.Values.vpc_ipv4_cidr}}" + FromPort: 9153 + IpProtocol: tcp + ToPort: 9153 + # Allow all traffic from Cluster Security Group + - SourceSecurityGroupId: !GetAtt EKSCluster.ClusterSecurityGroupId + IpProtocol : "-1" + - CidrIp: "{{.Values.vpc_ipv4_cidr}}" + FromPort: 30000 + IpProtocol: tcp + ToPort: 32767 + - CidrIp: "{{.Values.vpc_ipv4_cidr}}" + FromPort: 53 + IpProtocol: tcp + ToPort: 53 + - CidrIp: "{{.Values.vpc_ipv4_cidr}}" + FromPort: 53 + IpProtocol: udp + ToPort: 53 + Tags: + - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}' + Value: owned + - Key: 'karpenter.sh/discovery' + Value: '{{ .Cluster.ID }}/WorkerNodeSecurityGroup' + VpcId: "{{.Cluster.ConfigItems.vpc_id}}" + EKSSelfWorkerSecurityIngress: + Type: AWS::EC2::SecurityGroupIngress + Properties: + IpProtocol: "-1" + GroupId: !Ref EKSWorkerSecurityGroup + SourceSecurityGroupId: !Ref EKSWorkerSecurityGroup + # allow access from Worker Security Group to the Control Plane SG + EKSGroupControlPlaneWorkerSecurityIngress: + Type: AWS::EC2::SecurityGroupIngress + Properties: + FromPort: 443 + GroupId: !GetAtt EKSCluster.ClusterSecurityGroupId + IpProtocol: tcp + SourceSecurityGroupId: !Ref EKSWorkerSecurityGroup + ToPort: 443 + EKSCluster: + Type: AWS::EKS::Cluster + Properties: + Name: "{{.Cluster.ID | eksID}}" + Version: "1.30" + RoleArn: !GetAtt EKSClusterRole.Arn + KubernetesNetworkConfig: + IpFamily: "{{.Cluster.ConfigItems.eks_ip_family}}" + AccessConfig: + AuthenticationMode: API + #BootstrapClusterCreatorAdminPermissions: false + EncryptionConfig: + - Provider: + KeyArn: !GetAtt EtcdEncryptionKey.Arn # TODO: maybe use another key for EKS? + Resources: + - secrets + ResourcesVpcConfig: + SecurityGroupIds: + - !Ref EKSControlPlaneSecurityGroup + SubnetIds: + {{ with $values := .Values }} + {{ range $az := $values.availability_zones }} + - "{{ index $values.subnets $az }}" + {{ end }} + {{ end }} + EndpointPublicAccess: true + EndpointPrivateAccess: true + # PublicAccessCidrs: [ "1.1.1.2/32" ] +{{- if eq .Cluster.ConfigItems.eks_control_plane_logging "true" }} + Logging: + ClusterLogging: + EnabledTypes: + - Type: api + - Type: audit + - Type: authenticator + - Type: controllerManager + - Type: scheduler +{{- end }} + # Tags: + # - Key: "application" + # Value: "kubernetes" + # TODO: vanity domain name? (certficate issue) + # + # + EKSIdentityProvider: + Type: AWS::EKS::IdentityProviderConfig + Properties: + ClusterName: !Ref EKSCluster + IdentityProviderConfigName: "okta" + Oidc: + IssuerUrl: "{{.Cluster.ConfigItems.okta_auth_issuer_url}}" + ClientId: "{{.Cluster.ConfigItems.okta_auth_client_id}}" + UsernameClaim: "email" + UsernamePrefix: "okta:" + GroupsClaim: "groups" + GroupsPrefix: "okta:" + Type: "oidc" + EKSAccessEntryNodeAuth: + Type: "AWS::EKS::AccessEntry" + Properties: + ClusterName: !Ref EKSCluster + PrincipalArn: !Sub "arn:aws:iam::${AWS::AccountId}:role/{{.Cluster.LocalID}}-worker" + Type: "EC2_LINUX" + EKSAccessEntryZalandoIAMAuth: + Type: "AWS::EKS::AccessEntry" + Properties: + ClusterName: !Ref EKSCluster + PrincipalArn: !Sub "arn:aws:iam::${AWS::AccountId}:role/zalando-iam" + Username: "zalando-iam:zalando:service:{{`{{SessionName}}`}}" + Type: "STANDARD" + # TODO: IAM POLICY + EKSCNIIPv6Policy: + Type: AWS::IAM::ManagedPolicy + Properties: + # PolicyName: "{{.Cluster.LocalID}}-eks-cni-ipv6" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "ec2:AssignIpv6Addresses" + - "ec2:DescribeInstances" + - "ec2:DescribeTags" + - "ec2:DescribeNetworkInterfaces" + - "ec2:DescribeInstanceTypes" + Resource: "*" + - Effect: Allow + Action: + - "ec2:CreateTags" + Resource: + - "arn:aws:ec2:*:*:network-interface/*" + EKSAWSNodeIAMRole: + Type: AWS::IAM::Role + Properties: + RoleName: "{{.Cluster.LocalID}}-aws-node" + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": [ + "ec2.amazonaws.com" + ] + }, + "Action": [ + "sts:AssumeRole" + ] + }, + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:aws-node" + } + } + } + ] + } + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] + Path: / + ManagedPolicyArns: +{{- if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }} + - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy +{{- else }} + - !Ref EKSCNIIPv6Policy +{{- end }} + EKSOIDCProvider: + Type: AWS::IAM::OIDCProvider + Properties: + ClientIdList: + - "sts.amazonaws.com" + Url: !GetAtt EKSCluster.OpenIdConnectIssuerUrl + ThumbprintList: + # SHA-1 sum of the root certificate in the trust chain for the certificate + # use to serve the open id discovery document. + - "9e99a48a9960b14926bb7f3b02e22da2b0ab7280" +{{ else }} {{ if ne .Cluster.ConfigItems.delete_vpc_resources "true" }} EtcdClusterSecurityGroupIngressFromMaster: Type: 'AWS::EC2::SecurityGroupIngress' @@ -22,25 +283,6 @@ Resources: GroupId: !ImportValue '{{.Cluster.ConfigItems.etcd_stack_name}}:etcd-security-group-id' IpProtocol: tcp SourceSecurityGroupId: !Ref MasterSecurityGroup - IngressLoadBalancerSecurityGroup: - Properties: - GroupDescription: !Ref 'AWS::StackName' - SecurityGroupIngress: - - CidrIp: 0.0.0.0/0 - FromPort: 80 - IpProtocol: tcp - ToPort: 80 - - CidrIp: 0.0.0.0/0 - FromPort: 443 - IpProtocol: tcp - ToPort: 443 - Tags: - - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}' - Value: owned - - Key: 'kubernetes:application' - Value: kube-ingress-aws-controller - VpcId: "{{.Cluster.ConfigItems.vpc_id}}" - Type: 'AWS::EC2::SecurityGroup' MasterKubeletToMasterKubeletSecurityGroup: Properties: FromPort: 10250 @@ -422,6 +664,29 @@ Resources: # SHA-1 sum of the root certificate in the trust chain for the certificate # use to serve the open id discovery document. - "9e99a48a9960b14926bb7f3b02e22da2b0ab7280" +# end of non eks resources +{{ end }} +{{- if ne .Cluster.ConfigItems.delete_vpc_resources "true" }} + IngressLoadBalancerSecurityGroup: + Properties: + GroupDescription: !Ref 'AWS::StackName' + SecurityGroupIngress: + - CidrIp: 0.0.0.0/0 + FromPort: 80 + IpProtocol: tcp + ToPort: 80 + - CidrIp: 0.0.0.0/0 + FromPort: 443 + IpProtocol: tcp + ToPort: 443 + Tags: + - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}' + Value: owned + - Key: 'kubernetes:application' + Value: kube-ingress-aws-controller + VpcId: "{{.Cluster.ConfigItems.vpc_id}}" + Type: 'AWS::EC2::SecurityGroup' +{{- end }} WorkerIAMRole: # role used by worker nodes, including karpenter ones Properties: AssumeRolePolicyDocument: @@ -442,6 +707,11 @@ Resources: - !Ref MasterIAMRole Version: 2012-10-17 Path: / +{{- if eq .Cluster.ConfigItems.eks "true"}} + ManagedPolicyArns: + - "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" + - "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" +{{- end }} Policies: - PolicyDocument: Statement: @@ -508,17 +778,34 @@ Resources: Type: 'AWS::IAM::Role' AutoscalerIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:cluster-autoscaler" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:cluster-autoscaler" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -561,18 +848,35 @@ Resources: {{- if eq .Cluster.ConfigItems.karpenter_pools_enabled "true"}} KarpenterIAMRole: # role for the karpenter controller Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: "arn:aws:iam::{{ .Cluster.InfrastructureAccount | getAWSAccountID }}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:aud": "sts.amazonaws.com" - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:karpenter" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:aud": "sts.amazonaws.com", + "${OIDC}:sub": "system:serviceaccount:kube-system:karpenter" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyName: "KarpenterControllerPolicy-{{ .Cluster.ID | awsValidID }}" @@ -967,17 +1271,34 @@ Resources: Type: AWS::IAM::Role Properties: RoleName: "{{.Cluster.LocalID}}-deployment-service-controller" - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:deployment-service-controller" + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:deployment-service-controller" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Policies: - PolicyName: ControllerPolicy PolicyDocument: @@ -1052,17 +1373,34 @@ Resources: Type: AWS::IAM::Role Properties: RoleName: "{{.Cluster.LocalID}}-deployment-service-status-service" - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:deployment-service-status-service" + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:deployment-service-status-service" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Policies: - PolicyName: StatusServicePolicy PolicyDocument: @@ -1093,17 +1431,34 @@ Resources: Resource: '*' ExternalDNSIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:external-dns" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:external-dns" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -1117,17 +1472,34 @@ Resources: Type: 'AWS::IAM::Role' IngressControllerIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{ .Values.hosted_zone }}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:kube-ingress-aws-controller" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:kube-ingress-aws-controller" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -1261,17 +1633,34 @@ Resources: {{ end }} ClusterLifecycleControllerIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:cluster-lifecycle-controller" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:cluster-lifecycle-controller" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -1315,25 +1704,49 @@ Resources: Type: 'AWS::IAM::Role' KubeReadyIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:kube-node-ready" - - Action: - - 'sts:AssumeRole' - Effect: Allow - Principal: - AWS: !Join - - '' - - - !Sub 'arn:aws:iam::${AWS::AccountId}:role/' - - !Ref MasterIAMRole - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ +{{- if ne .Cluster.ConfigItems.eks "true" }} + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "AWS": "${MasterIAMRole}" + } + }, +{{- end }} + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:kube-node-ready" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" + MasterIAMRole: !Join + - '' + - - !Sub 'arn:aws:iam::${AWS::AccountId}:role/' + - !Ref MasterIAMRole +{{- end }} Path: / Policies: - PolicyDocument: @@ -1420,6 +1833,7 @@ Resources: Version: 2012-10-17 PolicyName: root Type: 'AWS::IAM::Role' +{{- if ne .Cluster.ConfigItems.eks "true" }} CloudControllerManagerIAMRole: Properties: AssumeRolePolicyDocument: @@ -1545,20 +1959,38 @@ Resources: Version: 2012-10-17 PolicyName: root RoleName: "{{.Cluster.LocalID}}-etcd-backup" +{{- end }} Type: 'AWS::IAM::Role' StaticEgressControllerIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:kube-static-egress-controller" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:kube-static-egress-controller" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -1783,37 +2215,65 @@ Resources: Properties: RoleName: "{{.Cluster.LocalID}}-app-logging-agent" Path: / - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Action: - - 'sts:AssumeRole' - Effect: Allow - Principal: - Service: - - ec2.amazonaws.com - - Action: ["sts:AssumeRole"] - Effect: Allow - Principal: - AWS: !Join - - '' - - - !Sub 'arn:aws:iam::${AWS::AccountId}:role/' - - !Ref WorkerIAMRole - - Action: ["sts:AssumeRole"] - Effect: Allow - Principal: - AWS: !Join - - '' - - - !Sub 'arn:aws:iam::${AWS::AccountId}:role/' - - !Ref MasterIAMRole - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:visibility:logging-agent" + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sts:AssumeRole" + ], + "Principal": { + "Service": [ + "ec2.amazonaws.com" + ] + } + }, + { + "Effect": "Allow", + "Action": [ + "sts:AssumeRole" + ], + "Principal": { + "AWS": "${WorkerIAMRole}" + } + }, + { + "Effect": "Allow", + "Action": [ + "sts:AssumeRole" + ], + "Principal": { + "AWS": "${MasterIAMRole}" + } + }, + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:visibility:logging-agent" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} + WorkerIAMRole: !Sub 'arn:aws:iam::${AWS::AccountId}:role/${WorkerIAMRole}' + MasterIAMRole: !Sub 'arn:aws:iam::${AWS::AccountId}:role/${MasterIAMRole}' Policies: - PolicyName: AllowS3BucketAccess PolicyDocument: @@ -1843,17 +2303,34 @@ Resources: KubeMetricsIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:custom-metrics-apiserver" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:custom-metrics-apiserver" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -1877,17 +2354,34 @@ Resources: EBSCSIControllerIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:ebs-csi-controller-sa" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:ebs-csi-controller-sa" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -2112,17 +2606,34 @@ Resources: Type: 'AWS::IAM::Role' Properties: RoleName: "{{.Cluster.LocalID}}-aws-node-decommissioner" - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:aws-node-decommissioner" + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:aws-node-decommissioner" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -2136,17 +2647,34 @@ Resources: PolicyName: root EmergencyAccessServiceIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:emergency-access-service" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:emergency-access-service" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -2170,17 +2698,34 @@ Resources: Type: 'AWS::IAM::Role' AudittrailAdapterIAMRole: Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" - Action: - - 'sts:AssumeRoleWithWebIdentity' - Condition: - StringEquals: - "{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:audittrail-adapter" - Version: 2012-10-17 + AssumeRolePolicyDocument: !Sub + - | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": [ + "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}" + ] + }, + "Action": [ + "sts:AssumeRoleWithWebIdentity" + ], + "Condition": { + "StringEquals": { + "${OIDC}:sub": "system:serviceaccount:kube-system:audittrail-adapter" + } + } + } + ] + } +{{- if eq .Cluster.ConfigItems.eks "true" }} + - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]] +{{- else }} + - OIDC: "{{.Cluster.LocalID}}.{{.Values.hosted_zone}}" +{{- end }} Path: / Policies: - PolicyDocument: @@ -2267,6 +2812,17 @@ Resources: - "kms:*" - "tag:TagResources" Resource: "*" +{{- if eq .Cluster.ConfigItems.eks "true" }} + - Sid: "Allow EKSClusterRole access to describe the key" + Effect: "Allow" + Principal: + AWS: + - !GetAtt EKSClusterRole.Arn + Action: + - "kms:DescribeKey" + - "kms:CreateGrant" + Resource: "*" +{{- else }} - Sid: "Enable master nodes to encrypt and decrypt secrets in etcd" Effect: "Allow" Principal: @@ -2278,6 +2834,8 @@ Resources: Action: - "kms:Encrypt" - "kms:Decrypt" +{{- end }} +{{- if ne .Cluster.ConfigItems.eks "true" }} MasterFilesEncryptionKey: Type: "AWS::KMS::Key" Properties: @@ -2315,6 +2873,7 @@ Resources: Resource: "*" Action: - "kms:Decrypt" +{{- end}} WorkerFilesEncryptionKey: Type: "AWS::KMS::Key" Properties: @@ -2353,22 +2912,37 @@ Resources: Action: - "kms:Decrypt" Outputs: - MasterIAMRole: +{{- if eq .Cluster.ConfigItems.eks "true" }} + EKSControlPlaneEndpoint: Export: - Name: '{{.Cluster.ID}}:master-iam-role' - Value: !Ref MasterIAMRole - WorkerIAMRole: + Name: "{{.Cluster.ID}}:eks-control-plane-endpoint" + Value: !GetAtt EKSCluster.Endpoint + # EKSServiceIpv6Cidr: + # Export: + # Name: "{{.Cluster.ID}}:eks-service-cidr" + # Value: !GetAtt EKSCluster.KubernetesNetworkConfig.ServiceIpv6Cidr + EKSOpenIdConnectIssuerUrl: Export: - Name: '{{.Cluster.ID}}:worker-iam-role' - Value: !Ref WorkerIAMRole - RemoteFilesEncryptionKey: + Name: "{{.Cluster.ID}}:eks-oidc-url" + Value: !GetAtt EKSCluster.OpenIdConnectIssuerUrl + EKSClusterSecurityGroupId: Export: - Name: '{{ .Cluster.ID}}:remote-files-encryption-key' - Value: !Ref RemoteFilesEncryptionKey - WorkerFilesEncryptionKey: + Name: "{{.Cluster.ID}}:eks-security-group-id" + Value: !GetAtt EKSCluster.ClusterSecurityGroupId + EKSWorkerSecurityGroup: Export: - Name: '{{ .Cluster.ID}}:worker-files-encryption-key' - Value: !Ref WorkerFilesEncryptionKey + Name: "{{.Cluster.ID}}:worker-security-group" + Value: !Ref EKSWorkerSecurityGroup + # Too big for Cloudformation output? + # EKSCertificateAuthorityData: + # Export: + # Name: "{{.Cluster.ID}}:eks-certificate-authority-data" + # Value: !GetAtt EKSCluster.CertificateAuthorityData +{{- else }} + MasterIAMRole: + Export: + Name: '{{.Cluster.ID}}:master-iam-role' + Value: !Ref MasterIAMRole MasterFilesEncryptionKey: Export: Name: '{{ .Cluster.ID}}:master-files-encryption-key' @@ -2391,3 +2965,16 @@ Outputs: Export: Name: '{{ .Cluster.ID}}:etcd-encryption-key' Value: !Ref EtcdEncryptionKey +{{- end}} + WorkerIAMRole: + Export: + Name: '{{.Cluster.ID}}:worker-iam-role' + Value: !Ref WorkerIAMRole + RemoteFilesEncryptionKey: + Export: + Name: '{{ .Cluster.ID}}:remote-files-encryption-key' + Value: !Ref RemoteFilesEncryptionKey + WorkerFilesEncryptionKey: + Export: + Name: '{{ .Cluster.ID}}:worker-files-encryption-key' + Value: !Ref WorkerFilesEncryptionKey diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 32804cc30d..51883d6023 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -1101,3 +1101,10 @@ sysctl_settings: "" # enables/disables the minDomains field for pod topology spread. min_domains_in_pod_topology_spread_enabled: "true" + +eks: "false" +eks_control_plane_logging: "false" +eks_ip_family: "ipv4" +# prefix delegation can only be configured for ipv4. For ipv6 it can only be +# true. +aws_vpc_cni_prefix_delegation: "true" diff --git a/cluster/manifests/01-aws-node/daemonset.yaml b/cluster/manifests/01-aws-node/daemonset.yaml new file mode 100644 index 0000000000..2247cfb9db --- /dev/null +++ b/cluster/manifests/01-aws-node/daemonset.yaml @@ -0,0 +1,256 @@ +{{- if eq .Cluster.ConfigItems.eks "true" }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/instance: aws-vpc-cni + app.kubernetes.io/name: aws-node + app.kubernetes.io/version: v1.15.1 + k8s-app: aws-node + application: kubernetes + component: aws-node + name: aws-node + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: aws-node + template: + metadata: + labels: + app.kubernetes.io/instance: aws-vpc-cni + app.kubernetes.io/name: aws-node + k8s-app: aws-node + application: kubernetes + component: aws-node + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - env: + - name: ADDITIONAL_ENI_TAGS + value: '{}' + - name: ANNOTATE_POD_IP + value: "false" + - name: AWS_VPC_CNI_NODE_PORT_SUPPORT + value: "true" + - name: AWS_VPC_ENI_MTU + value: "9001" + - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG + value: "false" + - name: AWS_VPC_K8S_CNI_EXTERNALSNAT + value: "false" + - name: AWS_VPC_K8S_CNI_LOGLEVEL + value: DEBUG + - name: AWS_VPC_K8S_CNI_LOG_FILE + value: /host/var/log/aws-routed-eni/ipamd.log + - name: AWS_VPC_K8S_CNI_RANDOMIZESNAT + value: prng + - name: AWS_VPC_K8S_CNI_VETHPREFIX + value: eni + - name: AWS_VPC_K8S_PLUGIN_LOG_FILE + value: /var/log/aws-routed-eni/plugin.log + - name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL + value: DEBUG + - name: CLUSTER_NAME + value: "{{ .Cluster.ID | eksID }}" + - name: DISABLE_INTROSPECTION + value: "false" + - name: DISABLE_METRICS + value: "false" + - name: DISABLE_NETWORK_RESOURCE_PROVISIONING + value: "false" + - name: ENABLE_IPv4 + value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}true{{else}}false{{end}}" + - name: ENABLE_IPv6 + value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}" + - name: ENABLE_POD_ENI + value: "false" + - name: ENABLE_PREFIX_DELEGATION + value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}{{.Cluster.ConfigItems.aws_vpc_cni_prefix_delegation}}{{else}}true{{end}}" + - name: NETWORK_POLICY_ENFORCING_MODE + value: standard + - name: VPC_CNI_VERSION + value: v1.17.1 + - name: VPC_ID + value: "{{ .Cluster.ConfigItems.vpc_id }}" + - name: WARM_ENI_TARGET + value: "1" + - name: WARM_PREFIX_TARGET + value: "1" + - name: MY_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: MY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon-k8s-cni:v1.17.1-eksbuild.1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /app/grpc-health-probe + - -addr=:50051 + - -connect-timeout=5s + - -rpc-timeout=5s + failureThreshold: 3 + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 10 + name: aws-node + ports: + - containerPort: 61678 + hostPort: 61678 + name: metrics + protocol: TCP + readinessProbe: + exec: + command: + - /app/grpc-health-probe + - -addr=:50051 + - -connect-timeout=5s + - -rpc-timeout=5s + failureThreshold: 3 + initialDelaySeconds: 1 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 10 + resources: + requests: + cpu: 25m + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + - mountPath: /host/var/log/aws-routed-eni + name: log-dir + - mountPath: /var/run/aws-node + name: run-dir + - mountPath: /run/xtables.lock + name: xtables-lock + - args: + - --enable-ipv6={{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}} + - --enable-network-policy=false + - --enable-cloudwatch-logs=false + - --enable-policy-event-logs=false + - --metrics-bind-addr=:8162 + - --health-probe-bind-addr=:8163 + - --conntrack-cache-cleanup-period=300 + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon/aws-network-policy-agent:v1.1.0-eksbuild.1 + imagePullPolicy: IfNotPresent + name: aws-eks-nodeagent + resources: + requests: + cpu: 25m + securityContext: + capabilities: + add: + - NET_ADMIN + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /sys/fs/bpf + name: bpf-pin-path + - mountPath: /var/log/aws-routed-eni + name: log-dir + - mountPath: /var/run/aws-node + name: run-dir + dnsPolicy: ClusterFirst + hostNetwork: true + initContainers: + - env: + - name: DISABLE_TCP_EARLY_DEMUX + value: "false" + - name: ENABLE_IPv6 + value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}" + image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon-k8s-cni-init:v1.17.1-eksbuild.1 + imagePullPolicy: IfNotPresent + name: aws-vpc-cni-init + resources: + requests: + cpu: 25m + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + priorityClassName: system-node-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: aws-node + serviceAccountName: aws-node + terminationGracePeriodSeconds: 10 + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /sys/fs/bpf + type: "" + name: bpf-pin-path + - hostPath: + path: /opt/cni/bin + type: "" + name: cni-bin-dir + - hostPath: + path: /etc/kubernetes/cni/net.d #/etc/cni/net.d + type: "" + name: cni-net-dir + - hostPath: + path: /var/log/aws-routed-eni + type: DirectoryOrCreate + name: log-dir + - hostPath: + path: /var/run/aws-node + type: DirectoryOrCreate + name: run-dir + - hostPath: + path: /run/xtables.lock + type: "" + name: xtables-lock + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 10% + type: RollingUpdate +{{- end }} diff --git a/cluster/manifests/01-aws-node/sa.yaml b/cluster/manifests/01-aws-node/sa.yaml new file mode 100644 index 0000000000..e287112f9e --- /dev/null +++ b/cluster/manifests/01-aws-node/sa.yaml @@ -0,0 +1,13 @@ +{{- if eq .Cluster.ConfigItems.eks "true"}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aws-node + namespace: kube-system + labels: + application: kubernetes + component: aws-node + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{.Cluster.LocalID}}-aws-node" +{{- end}} diff --git a/cluster/manifests/coredns-local/configmap-local.yaml b/cluster/manifests/01-coredns-local/configmap-local.yaml similarity index 99% rename from cluster/manifests/coredns-local/configmap-local.yaml rename to cluster/manifests/01-coredns-local/configmap-local.yaml index 7a95abcf24..2b031b30c6 100644 --- a/cluster/manifests/coredns-local/configmap-local.yaml +++ b/cluster/manifests/01-coredns-local/configmap-local.yaml @@ -120,7 +120,7 @@ data: {{ else }} forward . /etc/resolv.conf {{ end }} - pprof 127.0.0.1:9156 + pprof :9156 cache 30 reload } diff --git a/cluster/manifests/coredns-local/daemonset-coredns.yaml b/cluster/manifests/01-coredns-local/daemonset-coredns.yaml similarity index 85% rename from cluster/manifests/coredns-local/daemonset-coredns.yaml rename to cluster/manifests/01-coredns-local/daemonset-coredns.yaml index f49241c065..9f65b24e49 100644 --- a/cluster/manifests/coredns-local/daemonset-coredns.yaml +++ b/cluster/manifests/01-coredns-local/daemonset-coredns.yaml @@ -27,7 +27,11 @@ spec: spec: initContainers: - name: ensure-apiserver +{{- if eq .Cluster.ConfigItems.eks "true" }} + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/ensure-apiserver:master-6 +{{- else }} image: container-registry.zalando.net/teapot/ensure-apiserver:master-6 +{{- end }} resources: requests: cpu: 1m @@ -57,7 +61,11 @@ spec: command: - dig - "+short" +{{- if and (eq .Cluster.ConfigItems.eks "true") (eq .Cluster.ConfigItems.eks_ip_family "ipv4") }} - "@127.0.0.1" +{{- else }} + - "::1" +{{- end }} - "kubernetes.default.svc.cluster.local" initialDelaySeconds: 60 periodSeconds: 10 @@ -78,7 +86,8 @@ spec: image: container-registry.zalando.net/teapot/unbound-telemetry:master-5 args: - tcp - - --bind=0.0.0.0:9054 + # TODO: ipv6 + - --bind=:9054 ports: - name: metrics containerPort: 9054 @@ -92,7 +101,11 @@ spec: {{ end }} {{ if eq .Cluster.ConfigItems.dns_cache "dnsmasq" }} - name: dnsmasq + {{- if eq .Cluster.ConfigItems.eks "true" }} + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/k8s-dns-dnsmasq-nanny:1.17.4-master-15 + {{- else }} image: container-registry.zalando.net/teapot/k8s-dns-dnsmasq-nanny:1.17.4-master-15 + {{- end }} securityContext: privileged: true livenessProbe: @@ -118,8 +131,8 @@ spec: - --neg-ttl=60 # send requests to the last server first, only fallback to the previous ones if it's unreachable - --strict-order - - --server=10.5.0.11#53 - - --server=127.0.0.1#9254 + - --server=10.5.0.11#53 # TODO: fix this for ipv6 + - --server={{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}127.0.0.1{{else}}::1{{end}}#9254 ports: - containerPort: 53 name: dns @@ -134,7 +147,11 @@ spec: cpu: {{.Cluster.ConfigItems.dns_dnsmasq_cpu}} memory: {{.Cluster.ConfigItems.dns_dnsmasq_mem}} - name: sidecar + {{- if eq .Cluster.ConfigItems.eks "true" }} + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/k8s-dns-sidecar:1.17.4-master-15 + {{- else }} image: container-registry.zalando.net/teapot/k8s-dns-sidecar:1.17.4-master-15 + {{- end }} securityContext: privileged: true livenessProbe: @@ -149,6 +166,7 @@ spec: args: - --v=2 - --logtostderr + # TODO: ipv6 - --probe=dnsmasq,127.0.0.1:9254,ec2.amazonaws.com,5,A - --prometheus-port=9054 ports: @@ -163,7 +181,11 @@ spec: memory: {{.Cluster.ConfigItems.dns_dnsmasq_sidecar_mem}} {{ end }} - name: coredns + {{- if eq .Cluster.ConfigItems.eks "true" }} + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/coredns:1.11.3-master-24 + {{- else }} image: container-registry.zalando.net/teapot/coredns:1.11.3-master-24 + {{- end }} args: [ "-conf", "/etc/coredns/Corefile" ] env: - name: ZONE diff --git a/cluster/manifests/coredns-local/rbac.yaml b/cluster/manifests/01-coredns-local/rbac.yaml similarity index 100% rename from cluster/manifests/coredns-local/rbac.yaml rename to cluster/manifests/01-coredns-local/rbac.yaml diff --git a/cluster/manifests/coredns-local/service-coredns.yaml b/cluster/manifests/01-coredns-local/service-coredns.yaml similarity index 76% rename from cluster/manifests/coredns-local/service-coredns.yaml rename to cluster/manifests/01-coredns-local/service-coredns.yaml index 1c80a67dd6..ae017bc0f4 100644 --- a/cluster/manifests/coredns-local/service-coredns.yaml +++ b/cluster/manifests/01-coredns-local/service-coredns.yaml @@ -9,7 +9,10 @@ metadata: spec: selector: component: coredns +{{- if ne .Cluster.ConfigItems.eks "true"}} +# TODO: what to do with eks service range? clusterIP: 10.5.0.11 +{{- end}} ports: - name: dns port: 53 diff --git a/cluster/manifests/01-admission-control/config.yaml b/cluster/manifests/02-admission-control/config.yaml similarity index 100% rename from cluster/manifests/01-admission-control/config.yaml rename to cluster/manifests/02-admission-control/config.yaml diff --git a/cluster/manifests/02-admission-control/deployment.yaml b/cluster/manifests/02-admission-control/deployment.yaml new file mode 100644 index 0000000000..1bc3258e72 --- /dev/null +++ b/cluster/manifests/02-admission-control/deployment.yaml @@ -0,0 +1,73 @@ +{{- if eq .Cluster.ConfigItems.eks "true"}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: teapot-admission-controller + namespace: kube-system + labels: + application: kubernetes + component: admission-controller +spec: + replicas: 2 + selector: + matchLabels: + deployment: teapot-admission-controller + template: + metadata: + labels: + deployment: teapot-admission-controller + application: kubernetes + component: admission-controller + annotations: + logging/destination: "{{.Cluster.ConfigItems.log_destination_infra}}" + spec: + tolerations: + - key: dedicated + value: cluster-seed + effect: NoSchedule + dnsConfig: + options: + - name: ndots + value: "1" + serviceAccountName: admission-controller + priorityClassName: system-cluster-critical + containers: + - name: admission-controller + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-206 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "sleep 60"] + readinessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: 8085 + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + requests: + cpu: 50m + memory: 100Mi + # TODO: limits + args: + - --address=:8085 + - --tls-cert-file=/etc/tls-certs/admission-controller.pem + - --tls-key-file=/etc/tls-certs/admission-controller-key.pem +{{- if index .Cluster.ConfigItems "application_registry_url" }} + - --application-registry-url={{.Cluster.ConfigItems.application_registry_url}} +{{- end }} +{{- if index .Cluster.ConfigItems "docker_meta_url" }} + - --docker-meta-url={{.Cluster.ConfigItems.docker_meta_url}} +{{- end }} + ports: + - containerPort: 8085 + volumeMounts: + - mountPath: /etc/tls-certs + name: tls-certs + readOnly: true + volumes: + - name: tls-certs + secret: + secretName: admission-controller-tls-certs +{{- end }} diff --git a/cluster/manifests/02-admission-control/rbac.yaml b/cluster/manifests/02-admission-control/rbac.yaml new file mode 100644 index 0000000000..9bda2ef26e --- /dev/null +++ b/cluster/manifests/02-admission-control/rbac.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admission-controller + namespace: kube-system + labels: + application: kubernetes + component: admission-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: admission-controller + labels: + application: kubernetes + component: admission-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin # TODO: consider limiting this. +subjects: + - kind: ServiceAccount + name: admission-controller + namespace: kube-system diff --git a/cluster/manifests/02-admission-control/secret.yaml b/cluster/manifests/02-admission-control/secret.yaml new file mode 100644 index 0000000000..db33e14713 --- /dev/null +++ b/cluster/manifests/02-admission-control/secret.yaml @@ -0,0 +1,14 @@ +{{- if eq .Cluster.ConfigItems.eks "true"}} +apiVersion: v1 +kind: Secret +metadata: + name: admission-controller-tls-certs + namespace: kube-system + labels: + application: kubernetes + component: admission-controller +type: Opaque +data: + admission-controller.pem: "{{ .Cluster.ConfigItems.admission_controller_cert }}" + admission-controller-key.pem: "{{ .Cluster.ConfigItems.admission_controller_key }}" +{{- end }} diff --git a/cluster/manifests/02-admission-control/service.yaml b/cluster/manifests/02-admission-control/service.yaml new file mode 100644 index 0000000000..84dc171758 --- /dev/null +++ b/cluster/manifests/02-admission-control/service.yaml @@ -0,0 +1,17 @@ +{{- if eq .Cluster.ConfigItems.eks "true"}} +apiVersion: v1 +kind: Service +metadata: + name: admission-controller + namespace: kube-system + labels: + application: kubernetes + component: admission-controller +spec: + selector: + deployment: teapot-admission-controller + ports: + - port: 443 + targetPort: 8085 + protocol: TCP +{{- end }} diff --git a/cluster/manifests/01-admission-control/teapot.yaml b/cluster/manifests/02-admission-control/teapot.yaml similarity index 59% rename from cluster/manifests/01-admission-control/teapot.yaml rename to cluster/manifests/02-admission-control/teapot.yaml index 7ba1ca80dc..5e85a7b2a0 100644 --- a/cluster/manifests/01-admission-control/teapot.yaml +++ b/cluster/manifests/02-admission-control/teapot.yaml @@ -7,8 +7,26 @@ metadata: component: teapot-admission-controller webhooks: - name: pod-admitter.teapot.zalan.do +{{- if eq .Cluster.ConfigItems.eks "true"}} + # avoid admission-control applying to the admission-controller components (🐔🥚) + objectSelector: + matchExpressions: + - key: component + operator: NotIn + values: ["admission-controller", "coredns", "aws-node"] + - key: k8s-app + operator: NotIn + values: ["kube-proxy"] +{{- end }} clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/pod" + {{- else }} url: "https://localhost:8085/pod" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -21,8 +39,26 @@ webhooks: resources: ["pods", "pods/ephemeralcontainers"] {{- if eq .Cluster.ConfigItems.teapot_admission_controller_inject_environment_variables "true" }} - name: pod-binding-admitter.teapot.zalan.do +{{- if eq .Cluster.ConfigItems.eks "true"}} + # avoid admission-control applying to the admission-controller components (🐔🥚) + objectSelector: + matchExpressions: + - key: component + operator: NotIn + values: ["admission-controller", "coredns", "aws-node"] + - key: k8s-app + operator: NotIn + values: ["kube-proxy"] +{{- end }} clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/pod" + {{- else }} url: "https://localhost:8085/pod" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Ignore @@ -35,7 +71,14 @@ webhooks: {{- end }} - name: storageclass-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/storageclass" + {{- else }} url: "https://localhost:8085/storageclass" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -46,8 +89,23 @@ webhooks: apiVersions: ["v1", "v1beta1"] resources: ["storageclasses"] - name: node-admitter.teapot.zalan.do +{{- if eq .Cluster.ConfigItems.eks "true"}} + # avoid admission-control applying to seed node where the admission-controller components run (🐔🥚) + objectSelector: + matchExpressions: + - key: dedicated + operator: NotIn + values: ["cluster-seed"] +{{- end }} clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/node" + {{- else }} url: "https://localhost:8085/node" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -60,7 +118,14 @@ webhooks: resources: ["nodes"] - name: configmap-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/configmap" + {{- else }} url: "https://localhost:8085/configmap" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -73,7 +138,14 @@ webhooks: resources: ["configmaps"] - name: cronjob-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/cronjob" + {{- else }} url: "https://localhost:8085/cronjob" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -85,7 +157,14 @@ webhooks: resources: ["cronjobs"] - name: job-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/job" + {{- else }} url: "https://localhost:8085/job" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -97,8 +176,23 @@ webhooks: apiVersions: ["v1"] resources: ["jobs"] - name: deployment-admitter.teapot.zalan.do +{{- if eq .Cluster.ConfigItems.eks "true"}} + # avoid admission-control applying to the admission-controller components (🐔🥚) + objectSelector: + matchExpressions: + - key: component + operator: NotIn + values: ["admission-controller"] +{{- end }} clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/deployment" + {{- else }} url: "https://localhost:8085/deployment" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -111,7 +205,14 @@ webhooks: resources: ["deployments"] - name: statefulset-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/statefulset" + {{- else }} url: "https://localhost:8085/statefulset" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -124,7 +225,14 @@ webhooks: resources: ["statefulsets"] - name: crd-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/crd" + {{- else }} url: "https://localhost:8085/crd" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -137,7 +245,14 @@ webhooks: resources: ["customresourcedefinitions"] - name: stack-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/stack" + {{- else }} url: "https://localhost:8085/stack" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -149,7 +264,14 @@ webhooks: resources: ["stacks"] - name: stackset-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/stackset" + {{- else }} url: "https://localhost:8085/stackset" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -161,7 +283,14 @@ webhooks: resources: ["stacksets"] - name: hpa-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/hpa" + {{- else }} url: "https://localhost:8085/hpa" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -173,8 +302,23 @@ webhooks: apiVersions: ["v2"] resources: ["horizontalpodautoscalers"] - name: serviceaccount-admitter.teapot.zalan.do +{{- if eq .Cluster.ConfigItems.eks "true"}} + # avoid admission-control applying to the admission-controller components (🐔🥚) + objectSelector: + matchExpressions: + - key: component + operator: NotIn + values: ["admission-controller", "aws-node", "coredns"] +{{- end }} clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/serviceaccount" + {{- else }} url: "https://localhost:8085/serviceaccount" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -187,8 +331,26 @@ webhooks: resources: ["serviceaccounts"] {{- if eq .Cluster.ConfigItems.teapot_admission_controller_check_daemonset_resources "true" }} - name: daemonset-admitter.teapot.zalan.do +{{- if eq .Cluster.ConfigItems.eks "true"}} + # avoid admission-control applying to the admission-controller components (🐔🥚) + objectSelector: + matchExpressions: + - key: component + operator: NotIn + values: ["aws-node", "coredns"] + - key: k8s-app + operator: NotIn + values: ["kube-proxy"] +{{- end }} clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/daemonset" + {{- else }} url: "https://localhost:8085/daemonset" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -202,7 +364,14 @@ webhooks: {{- end }} - name: priorityclass-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/priorityclass" + {{- else }} url: "https://localhost:8085/priorityclass" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -215,7 +384,14 @@ webhooks: resources: ["priorityclasses"] - name: postgresql-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/postgresql" + {{- else }} url: "https://localhost:8085/postgresql" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -228,7 +404,14 @@ webhooks: resources: ["postgresqls"] - name: namespace-admitter.teapot.zalan.do clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/namespace" + {{- else }} url: "https://localhost:8085/namespace" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail @@ -240,8 +423,23 @@ webhooks: apiVersions: ["v1"] resources: ["namespaces"] - name: service-admitter.teapot.zalan.do +{{- if eq .Cluster.ConfigItems.eks "true"}} + # avoid admission-control applying to the admission-controller components (🐔🥚) + objectSelector: + matchExpressions: + - key: component + operator: NotIn + values: ["admission-controller"] +{{- end }} clientConfig: + {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "admission-controller" + namespace: "kube-system" + path: "/service" + {{- else }} url: "https://localhost:8085/service" + {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1beta1"] failurePolicy: Fail diff --git a/cluster/manifests/01-platformcredentialsset/customresourcedefinition.yaml b/cluster/manifests/02-platformcredentialsset/customresourcedefinition.yaml similarity index 100% rename from cluster/manifests/01-platformcredentialsset/customresourcedefinition.yaml rename to cluster/manifests/02-platformcredentialsset/customresourcedefinition.yaml diff --git a/cluster/manifests/01-priorityclasses/cluster-critical-nonpreempting.yaml b/cluster/manifests/02-priorityclasses/cluster-critical-nonpreempting.yaml similarity index 100% rename from cluster/manifests/01-priorityclasses/cluster-critical-nonpreempting.yaml rename to cluster/manifests/02-priorityclasses/cluster-critical-nonpreempting.yaml diff --git a/cluster/manifests/02-skipper-validation-webhook/deployment.yaml b/cluster/manifests/02-skipper-validation-webhook/deployment.yaml new file mode 100644 index 0000000000..9142d86902 --- /dev/null +++ b/cluster/manifests/02-skipper-validation-webhook/deployment.yaml @@ -0,0 +1,66 @@ +{{- if eq .Cluster.ConfigItems.eks "true"}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: skipper-validation-webhook + namespace: kube-system + labels: + application: skipper-ingress + component: webhook +spec: + replicas: 2 + selector: + matchLabels: + deployment: skipper-validation-webhook + template: + metadata: + labels: + deployment: skipper-validation-webhook + application: skipper-ingress + component: webhook + annotations: + logging/destination: "{{.Cluster.ConfigItems.log_destination_infra}}" + spec: + tolerations: + - key: dedicated + value: cluster-seed + effect: NoSchedule + dnsConfig: + options: + - name: ndots + value: "1" + priorityClassName: system-cluster-critical + containers: + - name: skipper-admission-webhook + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.17.1 + args: + - webhook + - --address=:9085 + - --tls-cert-file=/etc/tls-certs/skipper-validation-webhook.pem + - --tls-key-file=/etc/tls-certs/skipper-validation-webhook-key.pem + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", " sleep 60"] + readinessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: 9085 + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + requests: + cpu: 30m + memory: 100Mi + ports: + - containerPort: 9085 + volumeMounts: + - mountPath: /etc/tls-certs + name: tls-certs + readOnly: true + volumes: + - name: tls-certs + secret: + secretName: skipper-validation-webhook-tls-certs +{{- end }} diff --git a/cluster/manifests/02-skipper-validation-webhook/secret.yaml b/cluster/manifests/02-skipper-validation-webhook/secret.yaml new file mode 100644 index 0000000000..7137c0d9d5 --- /dev/null +++ b/cluster/manifests/02-skipper-validation-webhook/secret.yaml @@ -0,0 +1,14 @@ +{{- if eq .Cluster.ConfigItems.eks "true"}} +apiVersion: v1 +kind: Secret +metadata: + name: skipper-validation-webhook-tls-certs + namespace: kube-system + labels: + deployment: skipper-validation-webhook + application: skipper-ingress +type: Opaque +data: + skipper-validation-webhook.pem: "{{ .Cluster.ConfigItems.skipper_validation_webhook_cert }}" + skipper-validation-webhook-key.pem: "{{ .Cluster.ConfigItems.skipper_validation_webhook_key }}" +{{- end }} diff --git a/cluster/manifests/02-skipper-validation-webhook/service.yaml b/cluster/manifests/02-skipper-validation-webhook/service.yaml new file mode 100644 index 0000000000..d72dce2c28 --- /dev/null +++ b/cluster/manifests/02-skipper-validation-webhook/service.yaml @@ -0,0 +1,17 @@ +{{- if eq .Cluster.ConfigItems.eks "true"}} +apiVersion: v1 +kind: Service +metadata: + name: skipper-validation-webhook + namespace: kube-system + labels: + application: skipper-ingress + component: webhook +spec: + selector: + deployment: skipper-validation-webhook + ports: + - port: 443 + targetPort: 9085 + protocol: TCP +{{- end }} diff --git a/cluster/manifests/01-admission-control/skipper-webhook.yaml b/cluster/manifests/02-skipper-validation-webhook/skipper-webhook.yaml similarity index 73% rename from cluster/manifests/01-admission-control/skipper-webhook.yaml rename to cluster/manifests/02-skipper-validation-webhook/skipper-webhook.yaml index b791d4623c..5adb2a0c56 100644 --- a/cluster/manifests/01-admission-control/skipper-webhook.yaml +++ b/cluster/manifests/02-skipper-validation-webhook/skipper-webhook.yaml @@ -14,7 +14,14 @@ webhooks: apiVersions: ["v1"] resources: ["routegroups"] clientConfig: + # {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "skipper-validation-webhook" + namespace: "kube-system" + path: "/routegroups" + # {{- else }} url: "https://localhost:9085/routegroups" + # {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1"] sideEffects: None @@ -28,7 +35,14 @@ webhooks: apiVersions: ["v1"] resources: ["ingresses"] clientConfig: + # {{- if eq .Cluster.ConfigItems.eks "true"}} + service: + name: "skipper-validation-webhook" + namespace: "kube-system" + path: "/ingresses" + # {{- else }} url: "https://localhost:9085/ingresses" + # {{- end }} caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}" admissionReviewVersions: ["v1"] sideEffects: None diff --git a/cluster/manifests/01-vertical-pod-autoscaler/01-crd.yaml b/cluster/manifests/02-vertical-pod-autoscaler/01-crd.yaml similarity index 100% rename from cluster/manifests/01-vertical-pod-autoscaler/01-crd.yaml rename to cluster/manifests/02-vertical-pod-autoscaler/01-crd.yaml diff --git a/cluster/manifests/01-vertical-pod-autoscaler/02-secret.yaml b/cluster/manifests/02-vertical-pod-autoscaler/02-secret.yaml similarity index 100% rename from cluster/manifests/01-vertical-pod-autoscaler/02-secret.yaml rename to cluster/manifests/02-vertical-pod-autoscaler/02-secret.yaml diff --git a/cluster/manifests/01-vertical-pod-autoscaler/admission-controller-deployment.yaml b/cluster/manifests/02-vertical-pod-autoscaler/admission-controller-deployment.yaml similarity index 100% rename from cluster/manifests/01-vertical-pod-autoscaler/admission-controller-deployment.yaml rename to cluster/manifests/02-vertical-pod-autoscaler/admission-controller-deployment.yaml diff --git a/cluster/manifests/01-vertical-pod-autoscaler/rbac.yaml b/cluster/manifests/02-vertical-pod-autoscaler/rbac.yaml similarity index 100% rename from cluster/manifests/01-vertical-pod-autoscaler/rbac.yaml rename to cluster/manifests/02-vertical-pod-autoscaler/rbac.yaml diff --git a/cluster/manifests/01-vertical-pod-autoscaler/recommender-deployment.yaml b/cluster/manifests/02-vertical-pod-autoscaler/recommender-deployment.yaml similarity index 100% rename from cluster/manifests/01-vertical-pod-autoscaler/recommender-deployment.yaml rename to cluster/manifests/02-vertical-pod-autoscaler/recommender-deployment.yaml diff --git a/cluster/manifests/01-vertical-pod-autoscaler/service.yaml b/cluster/manifests/02-vertical-pod-autoscaler/service.yaml similarity index 100% rename from cluster/manifests/01-vertical-pod-autoscaler/service.yaml rename to cluster/manifests/02-vertical-pod-autoscaler/service.yaml diff --git a/cluster/manifests/01-vertical-pod-autoscaler/updater-deployment.yaml b/cluster/manifests/02-vertical-pod-autoscaler/updater-deployment.yaml similarity index 100% rename from cluster/manifests/01-vertical-pod-autoscaler/updater-deployment.yaml rename to cluster/manifests/02-vertical-pod-autoscaler/updater-deployment.yaml diff --git a/cluster/manifests/01-visibility/01-namespace.yaml b/cluster/manifests/02-visibility/01-namespace.yaml similarity index 100% rename from cluster/manifests/01-visibility/01-namespace.yaml rename to cluster/manifests/02-visibility/01-namespace.yaml diff --git a/cluster/manifests/02-kube-aws-iam-controller/crd.yaml b/cluster/manifests/03-kube-aws-iam-controller/crd.yaml similarity index 100% rename from cluster/manifests/02-kube-aws-iam-controller/crd.yaml rename to cluster/manifests/03-kube-aws-iam-controller/crd.yaml diff --git a/cluster/manifests/02-kube-aws-iam-controller/deployment.yaml b/cluster/manifests/03-kube-aws-iam-controller/deployment.yaml similarity index 96% rename from cluster/manifests/02-kube-aws-iam-controller/deployment.yaml rename to cluster/manifests/03-kube-aws-iam-controller/deployment.yaml index 629ad83e7f..38605bcc80 100644 --- a/cluster/manifests/02-kube-aws-iam-controller/deployment.yaml +++ b/cluster/manifests/03-kube-aws-iam-controller/deployment.yaml @@ -45,5 +45,7 @@ spec: - key: node.kubernetes.io/role value: master effect: NoSchedule +{{- if ne .Cluster.ConfigItems.eks "true" }} nodeSelector: node.kubernetes.io/role: master +{{- end }} diff --git a/cluster/manifests/02-kube-aws-iam-controller/rbac.yaml b/cluster/manifests/03-kube-aws-iam-controller/rbac.yaml similarity index 100% rename from cluster/manifests/02-kube-aws-iam-controller/rbac.yaml rename to cluster/manifests/03-kube-aws-iam-controller/rbac.yaml diff --git a/cluster/manifests/02-kube-aws-iam-controller/vpa.yaml b/cluster/manifests/03-kube-aws-iam-controller/vpa.yaml similarity index 100% rename from cluster/manifests/02-kube-aws-iam-controller/vpa.yaml rename to cluster/manifests/03-kube-aws-iam-controller/vpa.yaml diff --git a/cluster/manifests/03-ebs-csi/clusterrole-attacher.yaml b/cluster/manifests/04-ebs-csi/clusterrole-attacher.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/clusterrole-attacher.yaml rename to cluster/manifests/04-ebs-csi/clusterrole-attacher.yaml diff --git a/cluster/manifests/03-ebs-csi/clusterrole-csi-node.yaml b/cluster/manifests/04-ebs-csi/clusterrole-csi-node.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/clusterrole-csi-node.yaml rename to cluster/manifests/04-ebs-csi/clusterrole-csi-node.yaml diff --git a/cluster/manifests/03-ebs-csi/clusterrole-provisioner.yaml b/cluster/manifests/04-ebs-csi/clusterrole-provisioner.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/clusterrole-provisioner.yaml rename to cluster/manifests/04-ebs-csi/clusterrole-provisioner.yaml diff --git a/cluster/manifests/03-ebs-csi/clusterrole-resizer.yaml b/cluster/manifests/04-ebs-csi/clusterrole-resizer.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/clusterrole-resizer.yaml rename to cluster/manifests/04-ebs-csi/clusterrole-resizer.yaml diff --git a/cluster/manifests/03-ebs-csi/clusterrolebinding-attacher.yaml b/cluster/manifests/04-ebs-csi/clusterrolebinding-attacher.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/clusterrolebinding-attacher.yaml rename to cluster/manifests/04-ebs-csi/clusterrolebinding-attacher.yaml diff --git a/cluster/manifests/03-ebs-csi/clusterrolebinding-csi-node.yaml b/cluster/manifests/04-ebs-csi/clusterrolebinding-csi-node.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/clusterrolebinding-csi-node.yaml rename to cluster/manifests/04-ebs-csi/clusterrolebinding-csi-node.yaml diff --git a/cluster/manifests/03-ebs-csi/clusterrolebinding-provisioner.yaml b/cluster/manifests/04-ebs-csi/clusterrolebinding-provisioner.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/clusterrolebinding-provisioner.yaml rename to cluster/manifests/04-ebs-csi/clusterrolebinding-provisioner.yaml diff --git a/cluster/manifests/03-ebs-csi/clusterrolebinding-resizer.yaml b/cluster/manifests/04-ebs-csi/clusterrolebinding-resizer.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/clusterrolebinding-resizer.yaml rename to cluster/manifests/04-ebs-csi/clusterrolebinding-resizer.yaml diff --git a/cluster/manifests/03-ebs-csi/controller.yaml b/cluster/manifests/04-ebs-csi/controller.yaml similarity index 99% rename from cluster/manifests/03-ebs-csi/controller.yaml rename to cluster/manifests/04-ebs-csi/controller.yaml index de32176699..583c8a8730 100644 --- a/cluster/manifests/03-ebs-csi/controller.yaml +++ b/cluster/manifests/04-ebs-csi/controller.yaml @@ -22,8 +22,10 @@ spec: spec: serviceAccountName: ebs-csi-controller-sa priorityClassName: system-cluster-critical +{{- if ne .Cluster.ConfigItems.eks "true" }} nodeSelector: node.kubernetes.io/role: master +{{- end }} tolerations: - key: node.kubernetes.io/role value: master diff --git a/cluster/manifests/03-ebs-csi/csidriver.yaml b/cluster/manifests/04-ebs-csi/csidriver.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/csidriver.yaml rename to cluster/manifests/04-ebs-csi/csidriver.yaml diff --git a/cluster/manifests/03-ebs-csi/node.yaml b/cluster/manifests/04-ebs-csi/node.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/node.yaml rename to cluster/manifests/04-ebs-csi/node.yaml diff --git a/cluster/manifests/03-ebs-csi/poddisruptionbudget-controller.yaml b/cluster/manifests/04-ebs-csi/poddisruptionbudget-controller.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/poddisruptionbudget-controller.yaml rename to cluster/manifests/04-ebs-csi/poddisruptionbudget-controller.yaml diff --git a/cluster/manifests/03-ebs-csi/role-leases.yaml b/cluster/manifests/04-ebs-csi/role-leases.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/role-leases.yaml rename to cluster/manifests/04-ebs-csi/role-leases.yaml diff --git a/cluster/manifests/03-ebs-csi/rolebinding-csi-node.yaml b/cluster/manifests/04-ebs-csi/rolebinding-csi-node.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/rolebinding-csi-node.yaml rename to cluster/manifests/04-ebs-csi/rolebinding-csi-node.yaml diff --git a/cluster/manifests/03-ebs-csi/rolebinding-leases.yaml b/cluster/manifests/04-ebs-csi/rolebinding-leases.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/rolebinding-leases.yaml rename to cluster/manifests/04-ebs-csi/rolebinding-leases.yaml diff --git a/cluster/manifests/03-ebs-csi/serviceaccount-csi-controller.yaml b/cluster/manifests/04-ebs-csi/serviceaccount-csi-controller.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/serviceaccount-csi-controller.yaml rename to cluster/manifests/04-ebs-csi/serviceaccount-csi-controller.yaml diff --git a/cluster/manifests/03-ebs-csi/serviceaccount-csi-node.yaml b/cluster/manifests/04-ebs-csi/serviceaccount-csi-node.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/serviceaccount-csi-node.yaml rename to cluster/manifests/04-ebs-csi/serviceaccount-csi-node.yaml diff --git a/cluster/manifests/03-ebs-csi/storageclass.yaml b/cluster/manifests/04-ebs-csi/storageclass.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/storageclass.yaml rename to cluster/manifests/04-ebs-csi/storageclass.yaml diff --git a/cluster/manifests/03-ebs-csi/vpa.yaml b/cluster/manifests/04-ebs-csi/vpa.yaml similarity index 100% rename from cluster/manifests/03-ebs-csi/vpa.yaml rename to cluster/manifests/04-ebs-csi/vpa.yaml diff --git a/cluster/manifests/aws-cloud-controller-manager/daemonset.yaml b/cluster/manifests/aws-cloud-controller-manager/daemonset.yaml index 87d17318ac..36182835e1 100644 --- a/cluster/manifests/aws-cloud-controller-manager/daemonset.yaml +++ b/cluster/manifests/aws-cloud-controller-manager/daemonset.yaml @@ -1,3 +1,4 @@ +{{- if ne .Cluster.ConfigItems.eks "true"}} {{- if eq .Cluster.ConfigItems.aws_cloud_controller_manager_enabled "true" }} apiVersion: apps/v1 kind: DaemonSet @@ -52,3 +53,4 @@ spec: updateStrategy: type: RollingUpdate {{- end }} +{{- end }} diff --git a/cluster/manifests/aws-cloud-controller-manager/rbac.yaml b/cluster/manifests/aws-cloud-controller-manager/rbac.yaml index ffe4ef69d8..917a62f82d 100644 --- a/cluster/manifests/aws-cloud-controller-manager/rbac.yaml +++ b/cluster/manifests/aws-cloud-controller-manager/rbac.yaml @@ -1,3 +1,4 @@ +{{- if ne .Cluster.ConfigItems.eks "true"}} {{- if eq .Cluster.ConfigItems.aws_cloud_controller_manager_enabled "true" }} apiVersion: v1 kind: ServiceAccount @@ -136,3 +137,4 @@ subjects: name: cloud-controller-manager namespace: kube-system {{- end }} +{{- end }} diff --git a/cluster/manifests/aws-node-decommissioner/01-rbac.yaml b/cluster/manifests/aws-node-decommissioner/01-rbac.yaml index e7597c09cf..527986abfb 100644 --- a/cluster/manifests/aws-node-decommissioner/01-rbac.yaml +++ b/cluster/manifests/aws-node-decommissioner/01-rbac.yaml @@ -3,10 +3,15 @@ kind: ServiceAccount metadata: name: "aws-node-decommissioner" namespace: "kube-system" - annotations: + labels: application: kubernetes component: aws-node-decommissioner + annotations: +{{- if eq .Cluster.ConfigItems.eks "true"}} + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{ .Cluster.LocalID }}-aws-node-decommissioner" +{{- else}} iam.amazonaws.com/role: "{{ .Cluster.LocalID }}-aws-node-decommissioner" +{{- end}} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/cluster/manifests/cluster-lifecycle-controller/deployment.yaml b/cluster/manifests/cluster-lifecycle-controller/deployment.yaml index cbb6ab7ad5..6be31c1c13 100644 --- a/cluster/manifests/cluster-lifecycle-controller/deployment.yaml +++ b/cluster/manifests/cluster-lifecycle-controller/deployment.yaml @@ -33,6 +33,10 @@ spec: effect: NoSchedule - key: node.kubernetes.io/not-ready operator: Exists +{{- if eq .Cluster.ConfigItems.eks "true"}} + - operator: Exists + effect: NoSchedule +{{- end}} containers: - name: cluster-lifecycle-controller image: container-registry.zalando.net/teapot/cluster-lifecycle-controller:master-41 @@ -55,5 +59,7 @@ spec: env: - name: AWS_REGION value: "{{ .Cluster.Region }}" +{{- if ne .Cluster.ConfigItems.eks "true" }} nodeSelector: node.kubernetes.io/role: master +{{- end }} diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index 066e845733..83e249cebb 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -3,6 +3,14 @@ pre_apply: [] # everything defined under here will be deleted after applying the manifests post_apply: +{{- if eq .Cluster.ConfigItems.eks "true"}} +- name: coredns + kind: Deployment + namespace: kube-system +- name: coredns + kind: ConfigMap + namespace: kube-system +{{- end }} {{ if eq .Cluster.ConfigItems.teapot_admission_controller_process_resources "true" }} - name: limits namespace: default diff --git a/cluster/manifests/deployment-service/controller-rbac.yaml b/cluster/manifests/deployment-service/controller-rbac.yaml index 18fd74cdeb..1e36b34862 100644 --- a/cluster/manifests/deployment-service/controller-rbac.yaml +++ b/cluster/manifests/deployment-service/controller-rbac.yaml @@ -7,7 +7,11 @@ metadata: application: "deployment-service" component: "controller" annotations: + {{- if eq .Cluster.ConfigItems.eks "true" }} + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{.Cluster.LocalID}}-deployment-service-controller" + {{- else }} iam.amazonaws.com/role: "{{.Cluster.LocalID}}-deployment-service-controller" + {{- end }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 diff --git a/cluster/manifests/deployment-service/status-service-rbac.yaml b/cluster/manifests/deployment-service/status-service-rbac.yaml index 7f087aa0bf..fe25acb91e 100644 --- a/cluster/manifests/deployment-service/status-service-rbac.yaml +++ b/cluster/manifests/deployment-service/status-service-rbac.yaml @@ -7,7 +7,11 @@ metadata: application: "deployment-service" component: "status-service" annotations: + {{- if eq .Cluster.ConfigItems.eks "true" }} + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{.Cluster.LocalID}}-deployment-service-status-service" + {{- else }} iam.amazonaws.com/role: "{{.Cluster.LocalID}}-deployment-service-status-service" + {{- end }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 diff --git a/cluster/manifests/etcd-backup/01-rbac.yaml b/cluster/manifests/etcd-backup/01-rbac.yaml index 6f9d5bd51e..ee412b156a 100644 --- a/cluster/manifests/etcd-backup/01-rbac.yaml +++ b/cluster/manifests/etcd-backup/01-rbac.yaml @@ -1,3 +1,4 @@ +{{- if ne .Cluster.ConfigItems.eks "true" }} apiVersion: v1 kind: ServiceAccount metadata: @@ -19,3 +20,4 @@ subjects: - kind: ServiceAccount name: etcd-backup namespace: kube-system +{{- end }} diff --git a/cluster/manifests/etcd-backup/cronjob.yaml b/cluster/manifests/etcd-backup/cronjob.yaml index f04d3edc04..8095839128 100644 --- a/cluster/manifests/etcd-backup/cronjob.yaml +++ b/cluster/manifests/etcd-backup/cronjob.yaml @@ -1,3 +1,4 @@ +{{- if ne .Cluster.ConfigItems.eks "true" }} apiVersion: batch/v1 kind: CronJob metadata: @@ -99,3 +100,4 @@ spec: path: /etc/kubernetes/ssl/etcd-key.pem type: File {{ end }} +{{- end }} diff --git a/cluster/manifests/external-dns/01-rbac.yaml b/cluster/manifests/external-dns/01-rbac.yaml index 6232889e65..2f31af00cb 100644 --- a/cluster/manifests/external-dns/01-rbac.yaml +++ b/cluster/manifests/external-dns/01-rbac.yaml @@ -8,7 +8,11 @@ metadata: application: kubernetes component: external-dns annotations: +{{- if eq .Cluster.ConfigItems.eks "true"}} + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{ .Cluster.LocalID }}-app-external-dns" +{{- else}} iam.amazonaws.com/role: "{{ .Cluster.LocalID }}-app-external-dns" +{{- end}} --- # allows to list services and ingresses apiVersion: rbac.authorization.k8s.io/v1 diff --git a/cluster/manifests/external-dns/deployment.yaml b/cluster/manifests/external-dns/deployment.yaml index 63eeca1da4..8d0020ebfc 100644 --- a/cluster/manifests/external-dns/deployment.yaml +++ b/cluster/manifests/external-dns/deployment.yaml @@ -33,10 +33,12 @@ spec: serviceAccountName: external-dns containers: - name: external-dns - image: container-registry.zalando.net/teapot/external-dns:v0.13.6-master-38 + # image: container-registry.zalando.net/teapot/external-dns:v0.13.6-master-38 + image: mikkeloscar/external-dns:1 args: - --source=service - --source=ingress + # TODO: support routegroup with eks ipv6 - --source=skipper-routegroup {{- range split .Cluster.ConfigItems.external_dns_domain_filter "," }} - --domain-filter={{ . }} @@ -52,6 +54,7 @@ spec: - --aws-zones-cache-duration={{ .Cluster.ConfigItems.external_dns_zones_cache_duration }} - --annotation-filter=external-dns.alpha.kubernetes.io/exclude notin (true) - --policy={{ .Cluster.ConfigItems.external_dns_policy }} + - --log-level=debug resources: requests: cpu: 50m diff --git a/cluster/manifests/flannel/daemonset.yaml b/cluster/manifests/flannel/daemonset.yaml index 852f36c122..ccffbbacda 100644 --- a/cluster/manifests/flannel/daemonset.yaml +++ b/cluster/manifests/flannel/daemonset.yaml @@ -1,3 +1,5 @@ +# TODO: what to do with kube-proxy for eks? +{{- if ne .Cluster.ConfigItems.eks "true"}} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -135,3 +137,4 @@ spec: path: /lib/tc type: "" name: lib-tc +{{- end}} diff --git a/cluster/manifests/flannel/rbac.yaml b/cluster/manifests/flannel/rbac.yaml index f3afba8e7d..8251de0b8d 100644 --- a/cluster/manifests/flannel/rbac.yaml +++ b/cluster/manifests/flannel/rbac.yaml @@ -1,3 +1,5 @@ +# TODO: what to do with kube-proxy for eks? +{{- if ne .Cluster.ConfigItems.eks "true"}} apiVersion: v1 kind: ServiceAccount metadata: @@ -42,3 +44,4 @@ subjects: - kind: ServiceAccount name: flannel namespace: kube-system +{{- end}} diff --git a/cluster/manifests/ingress-controller/01-rbac.yaml b/cluster/manifests/ingress-controller/01-rbac.yaml index 8d659a6f69..f20c784c84 100644 --- a/cluster/manifests/ingress-controller/01-rbac.yaml +++ b/cluster/manifests/ingress-controller/01-rbac.yaml @@ -4,7 +4,11 @@ metadata: name: kube-ingress-aws-controller namespace: kube-system annotations: +{{- if eq .Cluster.ConfigItems.eks "true"}} + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{ .Cluster.LocalID }}-app-ingr-ctrl" +{{- else}} iam.amazonaws.com/role: "{{ .Cluster.LocalID }}-app-ingr-ctrl" +{{- end}} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/cluster/manifests/kube-cluster-autoscaler/01-rbac.yaml b/cluster/manifests/kube-cluster-autoscaler/01-rbac.yaml index 96382371a4..50700c09e8 100644 --- a/cluster/manifests/kube-cluster-autoscaler/01-rbac.yaml +++ b/cluster/manifests/kube-cluster-autoscaler/01-rbac.yaml @@ -7,7 +7,11 @@ metadata: application: kubernetes component: kube-cluster-autoscaler annotations: +{{- if eq .Cluster.ConfigItems.eks "true"}} + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{ .Cluster.LocalID }}-app-autoscaler" +{{- else}} iam.amazonaws.com/role: "{{ .Cluster.LocalID }}-app-autoscaler" +{{- end}} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml b/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml index ae6be74392..5c22f4a965 100644 --- a/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml +++ b/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml @@ -1,5 +1,9 @@ apiVersion: apps/v1 +{{- if eq .Cluster.ConfigItems.eks "true"}} +kind: Deployment +{{- else}} kind: DaemonSet +{{- end}} metadata: name: kube-cluster-autoscaler namespace: kube-system @@ -10,8 +14,10 @@ spec: selector: matchLabels: daemonset: kube-cluster-autoscaler +{{- if ne .Cluster.ConfigItems.eks "true"}} updateStrategy: type: RollingUpdate +{{- end}} template: metadata: labels: @@ -31,8 +37,13 @@ spec: serviceAccountName: cluster-autoscaler dnsPolicy: Default tolerations: +{{- if eq .Cluster.ConfigItems.eks "true"}} + - key: dedicated + value: cluster-seed +{{- else}} - key: node.kubernetes.io/role value: master +{{- end}} effect: NoSchedule containers: - name: cluster-autoscaler @@ -72,5 +83,7 @@ spec: env: - name: AWS_REGION value: "{{ .Cluster.Region }}" +{{- if ne .Cluster.ConfigItems.eks "true"}} nodeSelector: node.kubernetes.io/role: master +{{- end}} diff --git a/cluster/manifests/kube-metrics-adapter/01-rbac.yaml b/cluster/manifests/kube-metrics-adapter/01-rbac.yaml index 39877bcf59..c56db461e8 100644 --- a/cluster/manifests/kube-metrics-adapter/01-rbac.yaml +++ b/cluster/manifests/kube-metrics-adapter/01-rbac.yaml @@ -4,7 +4,11 @@ metadata: name: custom-metrics-apiserver namespace: kube-system annotations: + {{- if eq .Cluster.ConfigItems.eks "true" }} + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{ .Cluster.LocalID }}-kube-metrics-adapter" + {{- else }} iam.amazonaws.com/role: "{{ .Cluster.LocalID }}-kube-metrics-adapter" + {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/cluster/manifests/kube-node-ready-controller/daemonset.yaml b/cluster/manifests/kube-node-ready-controller/daemonset.yaml index eb9871519e..736b819c5c 100644 --- a/cluster/manifests/kube-node-ready-controller/daemonset.yaml +++ b/cluster/manifests/kube-node-ready-controller/daemonset.yaml @@ -1,5 +1,9 @@ apiVersion: apps/v1 +{{- if eq .Cluster.ConfigItems.eks "true"}} +kind: Deployment +{{- else}} kind: DaemonSet +{{- end}} metadata: name: kube-node-ready-controller namespace: kube-system @@ -10,8 +14,10 @@ spec: selector: matchLabels: daemonset: kube-node-ready-controller +{{- if ne .Cluster.ConfigItems.eks "true"}} updateStrategy: type: RollingUpdate +{{- end}} template: metadata: labels: @@ -29,11 +35,17 @@ spec: - name: ndots value: "1" serviceAccountName: kube-node-ready-controller + priorityClassName: system-cluster-critical dnsPolicy: Default tolerations: - key: node.kubernetes.io/role value: master effect: NoSchedule +{{- if eq .Cluster.ConfigItems.eks "true"}} + - key: dedicated + value: cluster-seed + effect: NoSchedule +{{- end}} containers: - name: controller image: container-registry.zalando.net/teapot/kube-node-ready-controller:master-26 @@ -41,5 +53,7 @@ spec: requests: cpu: {{.Cluster.ConfigItems.kube_node_ready_controller_cpu}} memory: {{.Cluster.ConfigItems.kube_node_ready_controller_memory}} +{{- if ne .Cluster.ConfigItems.eks "true"}} nodeSelector: node.kubernetes.io/role: master +{{- end}} diff --git a/cluster/manifests/kube-node-ready/01-rbac.yaml b/cluster/manifests/kube-node-ready/01-rbac.yaml index e38976084d..0746c12f82 100644 --- a/cluster/manifests/kube-node-ready/01-rbac.yaml +++ b/cluster/manifests/kube-node-ready/01-rbac.yaml @@ -4,4 +4,8 @@ metadata: name: kube-node-ready namespace: kube-system annotations: + {{- if eq .Cluster.ConfigItems.eks "true" }} + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{ .Cluster.LocalID }}-kube-node-ready" + {{- else }} iam.amazonaws.com/role: "{{ .Cluster.LocalID }}-kube-node-ready" + {{- end }} diff --git a/cluster/manifests/kube-proxy/configmap.yaml b/cluster/manifests/kube-proxy/configmap.yaml index a2c2a0eea0..71e790de55 100644 --- a/cluster/manifests/kube-proxy/configmap.yaml +++ b/cluster/manifests/kube-proxy/configmap.yaml @@ -1,3 +1,4 @@ +{{- if ne .Cluster.ConfigItems.eks "true"}} apiVersion: v1 kind: ConfigMap metadata: @@ -42,3 +43,4 @@ data: portRange: "" resourceContainer: /kube-proxy udpIdleTimeout: 250ms +{{- end}} diff --git a/cluster/manifests/kube-proxy/daemonset.yaml b/cluster/manifests/kube-proxy/daemonset.yaml index 79b78c33aa..03f88721b8 100644 --- a/cluster/manifests/kube-proxy/daemonset.yaml +++ b/cluster/manifests/kube-proxy/daemonset.yaml @@ -1,3 +1,5 @@ +# TODO: what to do with kube-proxy for eks? +{{- if ne .Cluster.ConfigItems.eks "true"}} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -82,3 +84,4 @@ spec: - name: kube-proxy-config configMap: name: kube-proxy-config +{{- end}} diff --git a/cluster/manifests/kube-proxy/rbac.yaml b/cluster/manifests/kube-proxy/rbac.yaml index a7c3787ad5..b054cf1a11 100644 --- a/cluster/manifests/kube-proxy/rbac.yaml +++ b/cluster/manifests/kube-proxy/rbac.yaml @@ -1,3 +1,5 @@ +# TODO: what to do with kube-proxy for eks? +{{- if ne .Cluster.ConfigItems.eks "true"}} apiVersion: v1 kind: ServiceAccount metadata: @@ -31,3 +33,4 @@ roleRef: kind: ClusterRole name: system:node-proxier apiGroup: rbac.authorization.k8s.io +{{- end}} diff --git a/cluster/manifests/prometheus/configmap.yaml b/cluster/manifests/prometheus/configmap.yaml index 863452ca33..db0f7eba2d 100644 --- a/cluster/manifests/prometheus/configmap.yaml +++ b/cluster/manifests/prometheus/configmap.yaml @@ -183,6 +183,7 @@ data: target_label: node_name - action: labeldrop regex: "^(pod|node|container)$" +{{- if ne .Cluster.ConfigItems.eks "true" }} - job_name: 'etcd-servers' scheme: http dns_sd_configs: @@ -205,6 +206,7 @@ data: - source_labels: [ __name__ ] regex: 'node_textfile.*' action: drop +{{- end }} {{ if eq .Cluster.ConfigItems.network_monitoring_enabled "true" }} {{if eq .Cluster.ConfigItems.network_monitoring_separate_prometheus "false"}} - job_name: "kubenurse" diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index 8ba59711bf..4be95567bf 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -532,6 +532,10 @@ spec: - "-enable-kubernetes-endpointslices={{ .Cluster.ConfigItems.skipper_endpointslices_enabled }}" - "-address=:9990" - "-wait-for-healthcheck-interval={{ .Cluster.ConfigItems.skipper_wait_for_healthcheck_interval }}" +{{- if and (eq .Cluster.ConfigItems.eks "true") (eq .Cluster.ConfigItems.eks_ip_family "ipv6")}} + # TODO: dynamically get the subnet range (or figure out if these are known internal addresses?) + - "-whitelisted-healthcheck-cidr=2a05:d014:09c0:bf00:0:0:0:0/64,2a05:d014:09c0:bf01:0:0:0:0/64,2a05:d014:09c0:bf02:0:0:0:0/64" +{{- end }} {{ if eq .Cluster.ConfigItems.enable_skipper_eastwest "true"}} - "-enable-kubernetes-east-west" - "-kubernetes-east-west-domain=.ingress.cluster.local" diff --git a/cluster/manifests/skipper/service-internal.yaml b/cluster/manifests/skipper/service-internal.yaml index e748de846e..167693f0a7 100644 --- a/cluster/manifests/skipper/service-internal.yaml +++ b/cluster/manifests/skipper/service-internal.yaml @@ -14,7 +14,12 @@ metadata: component: ingress spec: type: ClusterIP +{{- if ne .Cluster.ConfigItems.eks "true"}} +# TODO: how to do internal-ingress? +# function to derive IP from range? +# Can be hardcoded for ipv4, must be dynamic for ipv6 clusterIP: 10.5.99.99 +{{- end}} ports: - port: 80 targetPort: 9999 diff --git a/cluster/manifests/z-karpenter/01-serviceaccount.yaml b/cluster/manifests/z-karpenter/01-serviceaccount.yaml index 72810faf85..7b5f0fc5c4 100644 --- a/cluster/manifests/z-karpenter/01-serviceaccount.yaml +++ b/cluster/manifests/z-karpenter/01-serviceaccount.yaml @@ -10,5 +10,9 @@ metadata: application: kubernetes component: karpenter annotations: +{{- if eq .Cluster.ConfigItems.eks "true"}} + eks.amazonaws.com/role-arn: "arn:aws:iam::{{.Cluster.InfrastructureAccount | getAWSAccountID}}:role/{{ .Cluster.LocalID }}-app-karpenter" +{{- else}} iam.amazonaws.com/role: '{{ .Cluster.LocalID }}-app-karpenter' +{{- end}} {{end}} diff --git a/cluster/manifests/z-karpenter/deployment.yaml b/cluster/manifests/z-karpenter/deployment.yaml index 86dbd273fe..935e9c99d2 100644 --- a/cluster/manifests/z-karpenter/deployment.yaml +++ b/cluster/manifests/z-karpenter/deployment.yaml @@ -164,4 +164,9 @@ spec: - key: node.kubernetes.io/role value: master effect: NoSchedule +{{- if eq .Cluster.ConfigItems.eks "true"}} + - key: dedicated + value: cluster-seed + effect: NoSchedule +{{- end}} {{ end }} diff --git a/cluster/node-pools/worker-combined/stack.yaml b/cluster/node-pools/worker-combined/stack.yaml index 652d531b6f..57eba5890c 100644 --- a/cluster/node-pools/worker-combined/stack.yaml +++ b/cluster/node-pools/worker-combined/stack.yaml @@ -151,6 +151,9 @@ Resources: NetworkInterfaces: - DeviceIndex: 0 AssociatePublicIpAddress: true + {{- if and (eq .Cluster.ConfigItems.eks "true") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }} + Ipv6AddressCount: 1 + {{- end}} Groups: - !ImportValue '{{ .Cluster.ID }}:worker-security-group' EbsOptimized: false diff --git a/cluster/node-pools/worker-karpenter/provisioners.yaml b/cluster/node-pools/worker-karpenter/provisioners.yaml index 3a7208ae62..2538700e4a 100644 --- a/cluster/node-pools/worker-karpenter/provisioners.yaml +++ b/cluster/node-pools/worker-karpenter/provisioners.yaml @@ -22,6 +22,7 @@ spec: - tags: karpenter.sh/discovery: "{{ .Cluster.ID }}/WorkerNodeSecurityGroup" associatePublicIPAddress: true + # TODO: eks: ipv6 instanceProfile: "{{ .Cluster.ID | awsValidID }}-WorkerKarpenter-InstanceProfile" blockDeviceMappings: - deviceName: /dev/sda1 @@ -112,6 +113,12 @@ spec: # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators requirements: +#{{ if and (eq .Cluster.ConfigItems.eks "true") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }} + - key: karpenter.k8s.aws/instance-hypervisor + operator: In + values: + - nitro +#{{ end }} #{{ if and (eq (len .NodePool.InstanceTypes) 1) (eq (index .NodePool.InstanceTypes 0) "default-for-karpenter") }} - key: "karpenter.k8s.aws/instance-family" operator: In @@ -183,11 +190,17 @@ spec: kubelet: clusterDNS: [ "10.0.1.100" ] cpuCFSQuota: false +#{{ if eq .Cluster.ConfigItems.eks "true" }} + # TODO: Uses lower limit for simplicity need to support dynamic value based + # on instance type. + maxPods: 10 +#{{ else }} # {{ if ne .Cluster.ConfigItems.karpenter_max_pods_per_node "" }} maxPods: {{ .Cluster.ConfigItems.karpenter_max_pods_per_node }} # {{ else }} maxPods: {{ nodeCIDRMaxPods (parseInt64 .Cluster.ConfigItems.node_cidr_mask_size) (parseInt64 .Cluster.ConfigItems.node_max_pods_extra_capacity) }} # {{ end }} +#{{ end }} systemReserved: cpu: "{{ .Cluster.ConfigItems.kubelet_system_reserved_cpu }}" memory: "{{ .Cluster.ConfigItems.kubelet_system_reserved_memory }}" diff --git a/cluster/node-pools/worker-splitaz/files.yaml b/cluster/node-pools/worker-splitaz/files.yaml index a4dbbfcf64..df18020d52 100644 --- a/cluster/node-pools/worker-splitaz/files.yaml +++ b/cluster/node-pools/worker-splitaz/files.yaml @@ -2,6 +2,12 @@ files: - path: /etc/kubernetes/.local-id data: "{{ .Cluster.LocalID | base64 }}" permissions: 0400 +{{- if eq .Cluster.ConfigItems.eks "true" }} + - path: /etc/kubernetes/ssl/ca.pem + data: {{.Cluster.ConfigItems.eks_certficate_authority_data}} + permissions: 0400 + encrypted: false +{{- else }} - path: /etc/kubernetes/ssl/worker.pem data: {{ .Cluster.ConfigItems.worker_cert }} permissions: 0400 @@ -14,3 +20,4 @@ files: data: {{ .Cluster.ConfigItems.ca_cert_decompressed }} permissions: 0400 encrypted: false +{{- end }} diff --git a/cluster/node-pools/worker-splitaz/stack.yaml b/cluster/node-pools/worker-splitaz/stack.yaml index ad6afc45db..39d0310ab0 100644 --- a/cluster/node-pools/worker-splitaz/stack.yaml +++ b/cluster/node-pools/worker-splitaz/stack.yaml @@ -164,6 +164,9 @@ Resources: NetworkInterfaces: - DeviceIndex: 0 AssociatePublicIpAddress: true + {{- if and (eq .Cluster.ConfigItems.eks "true") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }} + Ipv6AddressCount: 1 + {{- end}} Groups: - !ImportValue '{{ .Cluster.ID }}:worker-security-group' EbsOptimized: false diff --git a/cluster/node-pools/worker-splitaz/userdata.yaml b/cluster/node-pools/worker-splitaz/userdata.yaml index 6321d041db..67a5c34729 100644 --- a/cluster/node-pools/worker-splitaz/userdata.yaml +++ b/cluster/node-pools/worker-splitaz/userdata.yaml @@ -34,7 +34,12 @@ write_files: clusters: - name: local cluster: + {{- if eq .Cluster.ConfigItems.eks "true" }} + server: {{.Cluster.ConfigItems.eks_endpoint}} + certificate-authority-data: "{{.Cluster.ConfigItems.eks_certficate_authority_data}}" + {{- else }} server: {{ .Cluster.APIServerURL }} + {{- end}} users: - name: kubelet user: @@ -45,7 +50,7 @@ write_files: - eks - get-token - --cluster-name - - "{{.Cluster.ID}}" + - '{{- if eq .Cluster.ConfigItems.eks "true" }}{{.Cluster.ID | eksID}}{{else}}{{.Cluster.ID}}{{end}}' contexts: - context: cluster: local @@ -66,21 +71,37 @@ write_files: imageGCLowThresholdPercent: {{.Cluster.ConfigItems.kubelet_image_gc_low_threshold}} clusterDomain: cluster.local cpuCFSQuota: false +{{- if eq .Cluster.ConfigItems.eks "true"}} + hairpinMode: hairpin-veth +{{- end }} featureGates: SizeMemoryBackedVolumes: {{ .Cluster.ConfigItems.enable_size_memory_backed_volumes }} {{- if eq .NodePool.ConfigItems.exec_probe_timeout_enabled "false" }} ExecProbeTimeout: false +{{- end }} +{{- if eq .Cluster.ConfigItems.eks "true"}} + RotateKubeletServerCertificate: true {{- end }} podPidsLimit: {{ .NodePool.ConfigItems.pod_max_pids }} cpuManagerPolicy: {{ .NodePool.ConfigItems.cpu_manager_policy }} +{{- if eq .Cluster.ConfigItems.eks "true" }} + # TODO: Uses lower limit for simplicity need to support dynamic value + # based on instance type. + maxPods: 14 +{{- else }} maxPods: {{ nodeCIDRMaxPods (parseInt64 .Cluster.ConfigItems.node_cidr_mask_size) (parseInt64 .Cluster.ConfigItems.node_max_pods_extra_capacity) }} +{{- end }} {{- if ne .Cluster.ConfigItems.serialize_image_pulls "true" }} serializeImagePulls: false {{- end }} healthzPort: 10248 healthzBindAddress: "0.0.0.0" +{{- if eq .Cluster.ConfigItems.eks "true"}} + serverTLSBootstrap: true +{{- else }} tlsCertFile: "/etc/kubernetes/ssl/worker.pem" tlsPrivateKeyFile: "/etc/kubernetes/ssl/worker-key.pem" +{{- end }} eventRecordQPS: 50 eventBurst: 50 kubeAPIQPS: 50 @@ -130,6 +151,7 @@ write_files: {{- end}} {{- end}} +{{- if ne .Cluster.ConfigItems.eks "true" }} - owner: root:root path: /etc/kubernetes/cni/net.d/10-flannel.conflist content: | @@ -146,3 +168,4 @@ write_files: } ] } +{{- end}}