add apiserver tls support

* add openshift ingress cabundle to pa/apiserver/ui * add ui tls enabled kfp server client Signed-off-by: Humair Khan <[email protected]>
opendatahub-io · Jul 12, 2024 · 1095bb1 · 1095bb1
1 parent 8280dcd
commit 1095bb1
Show file tree

Hide file tree

Showing 23 changed files with 622 additions and 92 deletions.
diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go
@@ -44,6 +44,12 @@ type DSPASpec struct {
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default:="v1"
 	DSPVersion string `json:"dspVersion,omitempty"`
+
+	// PodToPodTLS Set to "true" or "false" tls communication between DSPA components. Default to enable Inter-Pod tls by setting to "true". Only supported in DSP V2 and OpenShift.
+	// +kubebuilder:default:=true
+	// +kubebuilder:validation:Optional
+	PodToPodTLS *bool `json:"podToPodTLS"`
+
 	// WorkflowController is an argo-specific component that manages a DSPA's Workflow objects and handles the orchestration of them with the central Argo server
 	// +kubebuilder:validation:Optional
 	*WorkflowController `json:"workflowController,omitempty"`

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml
@@ -748,6 +748,12 @@ spec:
                         type: object
                     type: object
                 type: object
+              podToPodTLS:
+                default: true
+                description: PodToPodTLS Set to "true" or "false" tls communication
+                  between DSPA components. Default to enable Inter-Pod tls by setting
+                  to "true". Only supported in DSP V2 and OpenShift.
+                type: boolean
               scheduledWorkflow:
                 default:
                   deploy: true

diff --git a/config/internal/apiserver/default/deployment.yaml.tmpl b/config/internal/apiserver/default/deployment.yaml.tmpl
@@ -179,32 +179,32 @@ spec:
             {{ if .APIServer.EnableSamplePipeline }}
             - --sampleconfig=/config/sample_config.json
             {{ end }}
+            {{ if .PodToPodTLS }}
+            - --tlsCertPath=/etc/tls/private/tls.crt
+            - --tlsCertKeyPath=/etc/tls/private/tls.key
+            {{ end }}
           ports:
             - containerPort: 8888
               name: http
             - containerPort: 8887
               name: grpc
           livenessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              {{ if .PodToPodTLS }}
+              scheme: HTTPS
+              {{ end }}
             initialDelaySeconds: 3
             periodSeconds: 5
             timeoutSeconds: 2
           readinessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              {{ if .PodToPodTLS }}
+              scheme: HTTPS
+              {{ end }}
             initialDelaySeconds: 3
             periodSeconds: 5
             timeoutSeconds: 2
@@ -231,6 +231,10 @@ spec:
             - name: server-config
               mountPath: /config/config.json
               subPath: {{ .APIServer.CustomServerConfig.Key }}
+          {{ if .PodToPodTLS }}
+            - mountPath: /etc/tls/private
+              name: proxy-tls
+          {{ end }}
           {{ if or .APIServer.EnableSamplePipeline .CustomCABundle }}
             {{ if .APIServer.EnableSamplePipeline }}
             - name: sample-config
@@ -250,7 +254,14 @@ spec:
             - --https-address=:8443
             - --provider=openshift
             - --openshift-service-account={{.APIServerDefaultResourceName}}
+            {{ if .PodToPodTLS }}
+            # because we use certs signed by openshift, these certs are not valid for
+            # localhost, thus we have to use the service name
+            - --upstream=https://{{.APIServerServiceDNSName}}:8888
+            - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+            {{ else }}
             - --upstream=http://localhost:8888
+            {{ end }}
             - --tls-cert=/etc/tls/private/tls.crt
             - --tls-key=/etc/tls/private/tls.key
             - --cookie-secret=SECRET

diff --git a/config/internal/mlpipelines-ui/deployment.yaml.tmpl b/config/internal/mlpipelines-ui/deployment.yaml.tmpl
@@ -45,9 +45,15 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-{{.Name}}
+              value: {{.APIServerServiceDNSName}}
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
+            {{ if .PodToPodTLS }}
+            - name: ML_PIPELINE_SERVICE_SCHEME
+              value: 'https'
+            - name: NODE_EXTRA_CA_CERTS
+              value: '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt'
+            {{ end }}
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST
               value: ds-pipeline-md-{{.Name}}
             - name: METADATA_ENVOY_SERVICE_SERVICE_PORT

diff --git a/config/internal/persistence-agent/deployment.yaml.tmpl b/config/internal/persistence-agent/deployment.yaml.tmpl
@@ -40,6 +40,10 @@ spec:
               {{ else }}
               value: PipelineRun
               {{ end }}
+            {{ if .PodToPodTLS }}
+            - name: SSL_CERT_DIR
+              value: "/etc/pki/tls/certs:/var/run/secrets/kubernetes.io/serviceaccount/"
+            {{ end }}
           image: "{{.PersistenceAgent.Image}}"
           imagePullPolicy: IfNotPresent
           name: ds-pipeline-persistenceagent
@@ -48,7 +52,10 @@ spec:
             - "--logtostderr=true"
             - "--ttlSecondsAfterWorkflowFinish=86400"
             - "--numWorker={{.PersistenceAgent.NumWorkers}}"
-            - "--mlPipelineAPIServerName={{.APIServerServiceName}}"
+            - "--mlPipelineAPIServerName={{.APIServerServiceDNSName}}"
+            {{ if .PodToPodTLS }}
+            - "--mlPipelineServiceTLSEnabled=true"
+            {{ end }}
             - "--namespace={{.Namespace}}"
             - "--mlPipelineServiceHttpPort=8888"
             - "--mlPipelineServiceGRPCPort=8887"

diff --git a/config/overlays/make-deploy/kustomization.yaml b/config/overlays/make-deploy/kustomization.yaml
@@ -9,4 +9,4 @@ patchesStrategicMerge:
 images:
 - name: controller
   newName: quay.io/opendatahub/data-science-pipelines-operator
-  newTag: main
+  newTag: pr-656
diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go
@@ -86,6 +86,10 @@ type DSPAParams struct {
 	// pipeline pods
 	CustomCABundle *dspa.CABundle
 	DSPONamespace  string
+	// Use to enable tls communication between component pods.
+	PodToPodTLS bool
+
+	APIServerServiceDNSName string
 }
 
 type DBConnection struct {
@@ -578,6 +582,7 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 	p.APIServer = dsp.Spec.APIServer.DeepCopy()
 	p.APIServerDefaultResourceName = apiServerDefaultResourceNamePrefix + dsp.Name
 	p.APIServerServiceName = fmt.Sprintf("%s-%s", config.DSPServicePrefix, p.Name)
+	p.APIServerServiceDNSName = fmt.Sprintf("%s.%s.svc.cluster.local", p.APIServerServiceName, p.Namespace)
 	p.ScheduledWorkflow = dsp.Spec.ScheduledWorkflow.DeepCopy()
 	p.ScheduledWorkflowDefaultResourceName = scheduledWorkflowDefaultResourceNamePrefix + dsp.Name
 	p.PersistenceAgent = dsp.Spec.PersistenceAgent.DeepCopy()
@@ -589,8 +594,19 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 	p.MLMD = dsp.Spec.MLMD.DeepCopy()
 	p.CustomCABundleRootMountPath = config.CustomCABundleRootMountPath
 	p.PiplinesCABundleMountPath = config.GetCABundleFileMountPath()
+	p.PodToPodTLS = false
 	dspTrustedCAConfigMapKey := config.CustomDSPTrustedCAConfigMapKey
 
+	// PodToPodTLS is only used in v2 dsp
+	if p.UsingV2Pipelines(dsp) {
+		// by default it's enabled when omitted
+		if dsp.Spec.PodToPodTLS == nil {
+			p.PodToPodTLS = true
+		} else {
+			p.PodToPodTLS = *dsp.Spec.PodToPodTLS
+		}
+	}
+
 	log := loggr.WithValues("namespace", p.Namespace).WithValues("dspa_name", p.Name)
 
 	if p.APIServer != nil {
@@ -683,6 +699,8 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 			}
 		}
 
+		// If ODH bundle is not present and
+
 		if p.APIServer.CABundleFileMountPath != "" {
 			p.CustomCABundleRootMountPath = p.APIServer.CABundleFileMountPath
 		}

diff --git a/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml
@@ -46,7 +46,7 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-testdsp2
+              value: ds-pipeline-testdsp2.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST

diff --git a/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml
@@ -46,7 +46,7 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-testdsp4
+              value: ds-pipeline-testdsp4.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST

diff --git a/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml
@@ -46,7 +46,7 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-testdsp5
+              value: ds-pipeline-testdsp5.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST

diff --git a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml
@@ -115,6 +115,8 @@ spec:
           args:
             - --config=/config
             - -logtostderr=true
+            - --tlsCertPath=/etc/tls/private/tls.crt
+            - --tlsCertKeyPath=/etc/tls/private/tls.key
           ports:
             - containerPort: 8888
               name: http
@@ -123,29 +125,15 @@ spec:
               name: grpc
               protocol: TCP
           livenessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
-            initialDelaySeconds: 3
-            periodSeconds: 5
-            timeoutSeconds: 2
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              scheme: HTTPS
           readinessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
-            initialDelaySeconds: 3
-            periodSeconds: 5
-            timeoutSeconds: 2
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              scheme: HTTPS
           resources:
             requests:
               cpu: 250m
@@ -157,14 +145,17 @@ spec:
             - name: server-config
               mountPath: /config/config.json
               subPath: config.json
+            - mountPath: /etc/tls/private
+              name: proxy-tls
             - name: ca-bundle
               mountPath: /dspa/custom-certs
         - name: oauth-proxy
           args:
             - --https-address=:8443
             - --provider=openshift
             - --openshift-service-account=ds-pipeline-testdsp6
-            - --upstream=http://localhost:8888
+            - --upstream=https://ds-pipeline-testdsp6.default.svc.cluster.local:8888
+            - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
             - --tls-cert=/etc/tls/private/tls.crt
             - --tls-key=/etc/tls/private/tls.key
             - --cookie-secret=SECRET

diff --git a/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml
@@ -108,6 +108,8 @@ spec:
             - --config=/config
             - -logtostderr=true
             - --sampleconfig=/config/sample_config.json
+            - --tlsCertPath=/etc/tls/private/tls.crt
+            - --tlsCertKeyPath=/etc/tls/private/tls.key
           ports:
             - containerPort: 8888
               name: http
@@ -116,29 +118,15 @@ spec:
               name: grpc
               protocol: TCP
           livenessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
-            initialDelaySeconds: 3
-            periodSeconds: 5
-            timeoutSeconds: 2
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              scheme: HTTPS
           readinessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
-            initialDelaySeconds: 3
-            periodSeconds: 5
-            timeoutSeconds: 2
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              scheme: HTTPS
           resources:
             requests:
               cpu: 1231m
@@ -150,6 +138,8 @@ spec:
             - name: server-config
               mountPath: /config/config.json
               subPath: config.json
+            - mountPath: /etc/tls/private
+              name: proxy-tls
             - mountPath: /config/sample_config.json
               name: sample-config
               subPath: sample_config.json
@@ -160,7 +150,8 @@ spec:
             - --https-address=:8443
             - --provider=openshift
             - --openshift-service-account=ds-pipeline-testdsp7
-            - --upstream=http://localhost:8888
+            - --upstream=https://ds-pipeline-testdsp7.default.svc.cluster.local:8888
+            - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
             - --tls-cert=/etc/tls/private/tls.crt
             - --tls-key=/etc/tls/private/tls.key
             - --cookie-secret=SECRET

diff --git a/controllers/testdata/declarative/case_7/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/mlpipelines-ui_deployment.yaml
@@ -46,9 +46,13 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-testdsp7
+              value: ds-pipeline-testdsp7.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
+            - name: ML_PIPELINE_SERVICE_SCHEME
+              value: 'https'
+            - name: NODE_EXTRA_CA_CERTS
+              value: '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt'
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST
               value: ds-pipeline-md-testdsp7
             - name: METADATA_ENVOY_SERVICE_SERVICE_PORT

diff --git a/controllers/testdata/declarative/case_7/expected/created/persistence-agent_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/persistence-agent_deployment.yaml
@@ -36,6 +36,8 @@ spec:
               value: ""
             - name: EXECUTIONTYPE
               value: Workflow
+            - name: SSL_CERT_DIR
+              value: "/etc/pki/tls/certs:/var/run/secrets/kubernetes.io/serviceaccount/"
           image: persistenceagent:test7
           imagePullPolicy: IfNotPresent
           name: ds-pipeline-persistenceagent