From 70dc772361974393307e59dc3bd5b21fbfa1f5c3 Mon Sep 17 00:00:00 2001 From: noahjax Date: Tue, 16 Apr 2024 16:59:43 -0700 Subject: [PATCH] Adds appProtocol values of tcp on services * flyteadmin http port * flyteadmin grpc port * flyteconsole grpc port This is necessary because the ingress may be configured in a way that it sends TLS traffic to internal Flyte services. Istio will use port names to determine traffic - and may therefore assume the appProtocol of http, even though traffic from ingress -> flyteadmin is actually https. This misconfiguration prevents any traffic from flowing through the ingress to the service. Flyteadmin http and grcp ports *are* accessible using `http` and `grpc` values for appProtocol respectively within the cluster, but as soon as traffic travels between the ingress and the service those settings will not work. The most "compatible" setting is `tcp` which works for any network stream. - Adds the nginx.ingress.kubernetes.io/service-upstream: "true" Nginx Controller using endpoints instead of Services kubernetes/ingress-nginx#257 kubernetes/ingress-nginx@main/docs/user-guide/nginx-configuration/annotations.md#service-upstream Signed-off-by: noahjax Signed-off-by: ddl-ebrown --- charts/flyte-core/README.md | 4 ++-- charts/flyte-core/templates/admin/service.yaml | 4 ++++ charts/flyte-core/templates/console/service.yaml | 1 + charts/flyte-core/values.yaml | 1 + charts/flyteagent/templates/agent/service.yaml | 1 + deployment/agent/flyte_agent_helm_generated.yaml | 1 + deployment/eks/flyte_aws_scheduler_helm_generated.yaml | 7 +++++++ deployment/eks/flyte_helm_controlplane_generated.yaml | 7 +++++++ deployment/eks/flyte_helm_dataplane_generated.yaml | 2 ++ deployment/eks/flyte_helm_generated.yaml | 7 +++++++ deployment/gcp/flyte_helm_controlplane_generated.yaml | 7 +++++++ deployment/gcp/flyte_helm_dataplane_generated.yaml | 2 ++ deployment/gcp/flyte_helm_generated.yaml | 7 +++++++ deployment/sandbox/flyte_helm_generated.yaml | 6 ++++++ docker/sandbox-bundled/manifests/complete-agent.yaml | 7 ++++--- docker/sandbox-bundled/manifests/complete.yaml | 4 ++-- docker/sandbox-bundled/manifests/dev.yaml | 4 ++-- 17 files changed, 63 insertions(+), 9 deletions(-) diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 99d92c0d412..5ffddcf4c1b 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -74,12 +74,12 @@ helm install gateway bitnami/contour -n flyte | cluster_resource_manager.service_account_name | string | `"flyteadmin"` | Service account name to run with | | cluster_resource_manager.templates | list | `[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]` | Resource templates that should be applied | | cluster_resource_manager.templates[0] | object | `{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"}` | Template for namespaces resources | -| common | object | `{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"ingressClassName":null,"separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":false}}` | ---------------------------------------------- COMMON SETTINGS | +| common | object | `{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console","nginx.ingress.kubernetes.io/service-upstream":"true"},"enabled":true,"ingressClassName":null,"separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":false}}` | ---------------------------------------------- COMMON SETTINGS | | common.databaseSecret.name | string | `""` | Specify name of K8s Secret which contains Database password. Leave it empty if you don't need this Secret | | common.databaseSecret.secretManifest | object | `{}` | Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets | | common.flyteNamespaceTemplate.enabled | bool | `false` | - Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. | | common.ingress.albSSLRedirect | bool | `false` | - albSSLRedirect adds a special route for ssl redirect. Only useful in combination with the AWS LoadBalancer Controller. | -| common.ingress.annotations | object | `{"nginx.ingress.kubernetes.io/app-root":"/console"}` | - Ingress annotations applied to both HTTP and GRPC ingresses. | +| common.ingress.annotations | object | `{"nginx.ingress.kubernetes.io/app-root":"/console","nginx.ingress.kubernetes.io/service-upstream":"true"}` | - Ingress annotations applied to both HTTP and GRPC ingresses. | | common.ingress.enabled | bool | `true` | - Enable or disable creating Ingress for Flyte. Relevant to disable when using e.g. Istio as ingress controller. | | common.ingress.ingressClassName | string | `nil` | - Sets the ingressClassName | | common.ingress.separateGrpcIngress | bool | `false` | - separateGrpcIngress puts GRPC routes into a separate ingress if true. Required for certain ingress controllers like nginx. | diff --git a/charts/flyte-core/templates/admin/service.yaml b/charts/flyte-core/templates/admin/service.yaml index b23cbc24a2a..5991a902e4f 100644 --- a/charts/flyte-core/templates/admin/service.yaml +++ b/charts/flyte-core/templates/admin/service.yaml @@ -20,17 +20,21 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8088 - name: grpc port: 81 protocol: TCP + appProtocol: grpc targetPort: 8089 - name: redoc protocol: TCP + appProtocol: TCP port: 87 targetPort: 8087 - name: http-metrics protocol: TCP + appProtocol: TCP port: 10254 {{- with .Values.flyteadmin.service.additionalPorts -}} {{ tpl (toYaml .) $ | nindent 4 }} diff --git a/charts/flyte-core/templates/console/service.yaml b/charts/flyte-core/templates/console/service.yaml index 1845294cef5..7760cb6fcc8 100644 --- a/charts/flyte-core/templates/console/service.yaml +++ b/charts/flyte-core/templates/console/service.yaml @@ -16,6 +16,7 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8080 {{- if .Values.flyteconsole.serviceMonitor.enabled }} - name: http-metrics diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 6fe64a614c3..1822ae6e469 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -570,6 +570,7 @@ common: # --- Ingress annotations applied to both HTTP and GRPC ingresses. annotations: nginx.ingress.kubernetes.io/app-root: /console + nginx.ingress.kubernetes.io/service-upstream: "true" # --- albSSLRedirect adds a special route for ssl redirect. Only useful in combination with the AWS LoadBalancer Controller. albSSLRedirect: false # --- Ingress hostname diff --git a/charts/flyteagent/templates/agent/service.yaml b/charts/flyteagent/templates/agent/service.yaml index a6e9908b313..862b01aa022 100644 --- a/charts/flyteagent/templates/agent/service.yaml +++ b/charts/flyteagent/templates/agent/service.yaml @@ -15,5 +15,6 @@ spec: - name: {{ .Values.ports.name }} port: {{ .Values.ports.containerPort }} protocol: TCP + appProtocol: TCP targetPort: {{ .Values.ports.name }} selector: {{ include "flyteagent.selectorLabels" . | nindent 4 }} diff --git a/deployment/agent/flyte_agent_helm_generated.yaml b/deployment/agent/flyte_agent_helm_generated.yaml index 49b8f492c24..c57836cc29e 100644 --- a/deployment/agent/flyte_agent_helm_generated.yaml +++ b/deployment/agent/flyte_agent_helm_generated.yaml @@ -40,6 +40,7 @@ spec: - name: agent-grpc port: 8000 protocol: TCP + appProtocol: TCP targetPort: agent-grpc selector: app.kubernetes.io/name: flyteagent diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml index 2bf9fa2cfe3..0adc1e3598f 100644 --- a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml +++ b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml @@ -745,17 +745,21 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8088 - name: grpc port: 81 protocol: TCP + appProtocol: grpc targetPort: 8089 - name: redoc protocol: TCP + appProtocol: TCP port: 87 targetPort: 8087 - name: http-metrics protocol: TCP + appProtocol: TCP port: 10254 selector: app.kubernetes.io/name: flyteadmin @@ -778,6 +782,7 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8080 selector: app.kubernetes.io/name: flyteconsole @@ -1456,6 +1461,7 @@ metadata: alb.ingress.kubernetes.io/target-type: ip kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/app-root: /console + nginx.ingress.kubernetes.io/service-upstream: "true" spec: ingressClassName: rules: @@ -1626,6 +1632,7 @@ metadata: kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/app-root: /console nginx.ingress.kubernetes.io/backend-protocol: GRPC + nginx.ingress.kubernetes.io/service-upstream: "true" spec: ingressClassName: rules: diff --git a/deployment/eks/flyte_helm_controlplane_generated.yaml b/deployment/eks/flyte_helm_controlplane_generated.yaml index 982ed24d037..efc768d0c8e 100644 --- a/deployment/eks/flyte_helm_controlplane_generated.yaml +++ b/deployment/eks/flyte_helm_controlplane_generated.yaml @@ -468,17 +468,21 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8088 - name: grpc port: 81 protocol: TCP + appProtocol: grpc targetPort: 8089 - name: redoc protocol: TCP + appProtocol: TCP port: 87 targetPort: 8087 - name: http-metrics protocol: TCP + appProtocol: TCP port: 10254 selector: app.kubernetes.io/name: flyteadmin @@ -501,6 +505,7 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8080 selector: app.kubernetes.io/name: flyteconsole @@ -1072,6 +1077,7 @@ metadata: alb.ingress.kubernetes.io/target-type: ip kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/app-root: /console + nginx.ingress.kubernetes.io/service-upstream: "true" spec: ingressClassName: rules: @@ -1242,6 +1248,7 @@ metadata: kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/app-root: /console nginx.ingress.kubernetes.io/backend-protocol: GRPC + nginx.ingress.kubernetes.io/service-upstream: "true" spec: ingressClassName: rules: diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml index 8d17d712796..f89080e4523 100644 --- a/deployment/eks/flyte_helm_dataplane_generated.yaml +++ b/deployment/eks/flyte_helm_dataplane_generated.yaml @@ -613,6 +613,7 @@ metadata: alb.ingress.kubernetes.io/target-type: ip kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/app-root: /console + nginx.ingress.kubernetes.io/service-upstream: "true" spec: ingressClassName: rules: @@ -783,6 +784,7 @@ metadata: kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/app-root: /console nginx.ingress.kubernetes.io/backend-protocol: GRPC + nginx.ingress.kubernetes.io/service-upstream: "true" spec: ingressClassName: rules: diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index 9b2ce0aea31..d41c86d8acc 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -776,17 +776,21 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8088 - name: grpc port: 81 protocol: TCP + appProtocol: grpc targetPort: 8089 - name: redoc protocol: TCP + appProtocol: TCP port: 87 targetPort: 8087 - name: http-metrics protocol: TCP + appProtocol: TCP port: 10254 selector: app.kubernetes.io/name: flyteadmin @@ -809,6 +813,7 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8080 selector: app.kubernetes.io/name: flyteconsole @@ -1586,6 +1591,7 @@ metadata: alb.ingress.kubernetes.io/target-type: ip kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/app-root: /console + nginx.ingress.kubernetes.io/service-upstream: "true" spec: ingressClassName: rules: @@ -1756,6 +1762,7 @@ metadata: kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/app-root: /console nginx.ingress.kubernetes.io/backend-protocol: GRPC + nginx.ingress.kubernetes.io/service-upstream: "true" spec: ingressClassName: rules: diff --git a/deployment/gcp/flyte_helm_controlplane_generated.yaml b/deployment/gcp/flyte_helm_controlplane_generated.yaml index 02c6ef5edf8..4353a23b798 100644 --- a/deployment/gcp/flyte_helm_controlplane_generated.yaml +++ b/deployment/gcp/flyte_helm_controlplane_generated.yaml @@ -482,17 +482,21 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8088 - name: grpc port: 81 protocol: TCP + appProtocol: grpc targetPort: 8089 - name: redoc protocol: TCP + appProtocol: TCP port: 87 targetPort: 8087 - name: http-metrics protocol: TCP + appProtocol: TCP port: 10254 selector: app.kubernetes.io/name: flyteadmin @@ -515,6 +519,7 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8080 selector: app.kubernetes.io/name: flyteconsole @@ -1080,6 +1085,7 @@ metadata: cert-manager.io/issuer: letsencrypt-production kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/app-root: /console + nginx.ingress.kubernetes.io/service-upstream: "true" nginx.ingress.kubernetes.io/ssl-redirect: "true" spec: ingressClassName: @@ -1241,6 +1247,7 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/app-root: /console nginx.ingress.kubernetes.io/backend-protocol: GRPC + nginx.ingress.kubernetes.io/service-upstream: "true" nginx.ingress.kubernetes.io/ssl-redirect: "true" spec: ingressClassName: diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml index e89bde15bf0..da79ba8e5be 100644 --- a/deployment/gcp/flyte_helm_dataplane_generated.yaml +++ b/deployment/gcp/flyte_helm_dataplane_generated.yaml @@ -613,6 +613,7 @@ metadata: cert-manager.io/issuer: letsencrypt-production kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/app-root: /console + nginx.ingress.kubernetes.io/service-upstream: "true" nginx.ingress.kubernetes.io/ssl-redirect: "true" spec: ingressClassName: @@ -774,6 +775,7 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/app-root: /console nginx.ingress.kubernetes.io/backend-protocol: GRPC + nginx.ingress.kubernetes.io/service-upstream: "true" nginx.ingress.kubernetes.io/ssl-redirect: "true" spec: ingressClassName: diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index 5fe9dcdaaa5..910dc69360d 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -798,17 +798,21 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8088 - name: grpc port: 81 protocol: TCP + appProtocol: grpc targetPort: 8089 - name: redoc protocol: TCP + appProtocol: TCP port: 87 targetPort: 8087 - name: http-metrics protocol: TCP + appProtocol: TCP port: 10254 selector: app.kubernetes.io/name: flyteadmin @@ -831,6 +835,7 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8080 selector: app.kubernetes.io/name: flyteconsole @@ -1601,6 +1606,7 @@ metadata: cert-manager.io/issuer: letsencrypt-production kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/app-root: /console + nginx.ingress.kubernetes.io/service-upstream: "true" nginx.ingress.kubernetes.io/ssl-redirect: "true" spec: ingressClassName: @@ -1762,6 +1768,7 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/app-root: /console nginx.ingress.kubernetes.io/backend-protocol: GRPC + nginx.ingress.kubernetes.io/service-upstream: "true" nginx.ingress.kubernetes.io/ssl-redirect: "true" spec: ingressClassName: diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index c984ddac6c0..c414832a466 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -6139,17 +6139,21 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8088 - name: grpc port: 81 protocol: TCP + appProtocol: grpc targetPort: 8089 - name: redoc protocol: TCP + appProtocol: TCP port: 87 targetPort: 8087 - name: http-metrics protocol: TCP + appProtocol: TCP port: 10254 selector: app.kubernetes.io/name: flyteadmin @@ -6172,6 +6176,7 @@ spec: - name: http port: 80 protocol: TCP + appProtocol: TCP targetPort: 8080 selector: app.kubernetes.io/name: flyteconsole @@ -7612,6 +7617,7 @@ metadata: namespace: flyte annotations: nginx.ingress.kubernetes.io/app-root: /console + nginx.ingress.kubernetes.io/service-upstream: "true" spec: ingressClassName: rules: diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index d56bc39f38d..437c531449c 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -816,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: S1dTb05oQlRYSmZ0MHI5aw== + haSharedSecret: eUJPYzR3OGs2cnBGSHVaTw== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1133,7 +1133,8 @@ metadata: namespace: flyte spec: ports: - - name: agent-grpc + - appProtocol: TCP + name: agent-grpc port: 8000 protocol: TCP targetPort: agent-grpc @@ -1412,7 +1413,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: ba31bacfae25dc1d02d3c9cb0c3a633ee568ad397ad170d61542c719c0f36f63 + checksum/secret: 6e5346a93d131d1b54f2989e5b7a285480254dc7dd52fbe1d202ad944d86c6c4 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index 6268d178c33..079d8e64e6a 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -796,7 +796,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: b2hFVnVDM3pGdjdxTEdFeQ== + haSharedSecret: SzBnRnppNDlyWkxYUlJnQQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1360,7 +1360,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: cd3db72802e13b4d01900ebce88a78b6ff20abe47153caeed0276b7db0653294 + checksum/secret: a847efce83ff31fd4c3c0066322cfc7e1cf94422e9c2b13540a188f28e9d0bfc labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index 48d26004ecc..279a60c3859 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: c3I2c3ZwMUVyUU1wUHZSUA== + haSharedSecret: MkN6SUpGT3VNcmIwdnVreg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 91fe7fa5ccabcb8deb88baf1d94d2284ed1f439f4ff2ddbe48457f6081cedd24 + checksum/secret: 5520b963248bf22bf6727cb4258f802ad846eeff4350b94a210f8908fc719ae4 labels: app: docker-registry release: flyte-sandbox