From f2b61ac0640909d65a6f2a38cc3be46435f2efe7 Mon Sep 17 00:00:00 2001 From: Emil Davtyan Date: Wed, 3 Apr 2024 13:13:45 +0200 Subject: [PATCH] feat: inject custom ca (#5269) (#5273) * feat: add runner custom ca variable * feat: add runner ca certificate to default job template * feat: add runner ca certificate to job template * feat: add runner ca certificate to job scraper template * feat: add runner ca certificate to slave pod template --- cmd/api-server/main.go | 1 + config/job-scraper-template.yml | 21 ++++++- config/job-template.yml | 28 +++++++++ config/slave-pod-template.yml | 17 +++++ internal/config/config.go | 62 +++++++++++-------- pkg/executor/client/common.go | 25 ++++---- pkg/executor/client/job.go | 2 + .../containerexecutor/containerexecutor.go | 2 + .../containerexecutor/templates/job.tmpl | 30 ++++++++- pkg/scheduler/service.go | 3 + pkg/scheduler/test_scheduler.go | 1 + pkg/triggers/executor_test.go | 1 + pkg/triggers/service_test.go | 1 + 13 files changed, 152 insertions(+), 42 deletions(-) diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go index 292e6596c10..de002b166e3 100644 --- a/cmd/api-server/main.go +++ b/cmd/api-server/main.go @@ -533,6 +533,7 @@ func main() { logsStream, cfg.TestkubeNamespace, cfg.TestkubeProTLSSecret, + cfg.TestkubeProRunnerCustomCASecret, ) if mode == common.ModeAgent { sched.WithSubscriptionChecker(subscriptionChecker) diff --git a/config/job-scraper-template.yml b/config/job-scraper-template.yml index 85d622630c2..d7ad7a1cbdd 100644 --- a/config/job-scraper-template.yml +++ b/config/job-scraper-template.yml @@ -46,16 +46,33 @@ spec: command: - "/bin/runner" - '{{ .Jsn }}' + {{- if .RunnerCustomCASecret }} + env: + - name: SSL_CERT_DIR + value: /etc/testkube/certs + {{- end }} + volumeMounts: + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + mountPath: /etc/testkube/certs/testkube-custom-ca.pem + readOnly: true + subPath: ca.crt + {{- end }} {{- if .ArtifactRequest }} {{- if .ArtifactRequest.VolumeMountPath }} - volumeMounts: - name: artifact-volume mountPath: {{ .ArtifactRequest.VolumeMountPath }} {{- end }} {{- end }} + volumes: + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + secret: + secretName: {{ .RunnerCustomCASecret }} + defaultMode: 420 + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - volumes: - name: artifact-volume persistentVolumeClaim: claimName: {{ .Name }}-pvc diff --git a/config/job-template.yml b/config/job-template.yml index 84385ea2175..7c3f70ad29d 100644 --- a/config/job-template.yml +++ b/config/job-template.yml @@ -20,6 +20,11 @@ spec: command: - "/bin/runner" - '{{ .Jsn }}' + {{- if .RunnerCustomCASecret }} + env: + - name: SSL_CERT_DIR + value: /etc/testkube/certs + {{- end }} volumeMounts: - name: data-volume mountPath: /data @@ -27,6 +32,12 @@ spec: - name: {{ .CertificateSecret }} mountPath: /etc/certs {{- end }} + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + mountPath: /etc/testkube/certs/testkube-custom-ca.pem + readOnly: true + subPath: ca.crt + {{- end }} {{- if .AgentAPITLSSecret }} - mountPath: /tmp/agent-cert readOnly: true @@ -86,6 +97,11 @@ spec: command: - "/bin/runner" - '{{ .Jsn }}' + {{- if .RunnerCustomCASecret }} + env: + - name: SSL_CERT_DIR + value: /etc/testkube/certs + {{- end }} volumeMounts: - name: data-volume mountPath: /data @@ -93,6 +109,12 @@ spec: - name: {{ .CertificateSecret }} mountPath: /etc/certs {{- end }} + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + mountPath: /etc/testkube/certs/testkube-custom-ca.pem + readOnly: true + subPath: ca.crt + {{- end }} {{- if .AgentAPITLSSecret }} - mountPath: /tmp/agent-cert readOnly: true @@ -124,6 +146,12 @@ spec: secret: secretName: {{ .CertificateSecret }} {{- end }} + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + secret: + secretName: {{ .RunnerCustomCASecret }} + defaultMode: 420 + {{- end }} {{- if .AgentAPITLSSecret }} - name: {{ .AgentAPITLSSecret }} secret: diff --git a/config/slave-pod-template.yml b/config/slave-pod-template.yml index 25cff84b8c1..a24e53d8e5e 100644 --- a/config/slave-pod-template.yml +++ b/config/slave-pod-template.yml @@ -25,6 +25,11 @@ spec: command: - "/bin/runner" - '{{ .Jsn }}' + {{- if .RunnerCustomCASecret }} + env: + - name: SSL_CERT_DIR + value: /etc/testkube/certs + {{- end }} volumeMounts: - name: data-volume mountPath: /data @@ -32,6 +37,12 @@ spec: - name: {{ .CertificateSecret }} mountPath: /etc/certs {{- end }} + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + mountPath: /etc/testkube/certs/testkube-custom-ca.pem + readOnly: true + subPath: ca.crt + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume @@ -159,6 +170,12 @@ spec: secret: secretName: {{ .CertificateSecret }} {{- end }} + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + secret: + secretName: {{ .RunnerCustomCASecret }} + defaultMode: 420 + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume diff --git a/internal/config/config.go b/internal/config/config.go index a375d8ca4e0..a534ff0a307 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -75,33 +75,34 @@ type Config struct { TestkubeProConnectionTimeout int `envconfig:"TESTKUBE_PRO_CONNECTION_TIMEOUT" default:"10"` TestkubeProCertFile string `envconfig:"TESTKUBE_PRO_CERT_FILE" default:""` TestkubeProKeyFile string `envconfig:"TESTKUBE_PRO_KEY_FILE" default:""` - TestkubeProCAFile string `envconfig:"TESTKUBE_PRO_CA_FILE" default:""` - TestkubeProTLSSecret string `envconfig:"TESTKUBE_PRO_TLS_SECRET" default:""` - TestkubeWatcherNamespaces string `envconfig:"TESTKUBE_WATCHER_NAMESPACES" default:""` - GraphqlPort string `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"` - TestkubeRegistry string `envconfig:"TESTKUBE_REGISTRY" default:""` - TestkubePodStartTimeout time.Duration `envconfig:"TESTKUBE_POD_START_TIMEOUT" default:"30m"` - CDEventsTarget string `envconfig:"CDEVENTS_TARGET" default:""` - TestkubeDashboardURI string `envconfig:"TESTKUBE_DASHBOARD_URI" default:""` - DisableReconciler bool `envconfig:"DISABLE_RECONCILER" default:"false"` - TestkubeClusterName string `envconfig:"TESTKUBE_CLUSTER_NAME" default:""` - CompressArtifacts bool `envconfig:"COMPRESSARTIFACTS" default:"false"` - TestkubeHelmchartVersion string `envconfig:"TESTKUBE_HELMCHART_VERSION" default:""` - DebugListenAddr string `envconfig:"DEBUG_LISTEN_ADDR" default:"0.0.0.0:1337"` - EnableDebugServer bool `envconfig:"ENABLE_DEBUG_SERVER" default:"false"` - EnableSecretsEndpoint bool `envconfig:"ENABLE_SECRETS_ENDPOINT" default:"false"` - DisableMongoMigrations bool `envconfig:"DISABLE_MONGO_MIGRATIONS" default:"false"` - Debug bool `envconfig:"DEBUG" default:"false"` - EnableImageDataPersistentCache bool `envconfig:"TESTKUBE_ENABLE_IMAGE_DATA_PERSISTENT_CACHE" default:"false"` - ImageDataPersistentCacheKey string `envconfig:"TESTKUBE_IMAGE_DATA_PERSISTENT_CACHE_KEY" default:"testkube-image-cache"` - LogServerGrpcAddress string `envconfig:"LOG_SERVER_GRPC_ADDRESS" default:":9090"` - LogServerSecure bool `envconfig:"LOG_SERVER_SECURE" default:"false"` - LogServerSkipVerify bool `envconfig:"LOG_SERVER_SKIP_VERIFY" default:"false"` - LogServerCertFile string `envconfig:"LOG_SERVER_CERT_FILE" default:""` - LogServerKeyFile string `envconfig:"LOG_SERVER_KEY_FILE" default:""` - LogServerCAFile string `envconfig:"LOG_SERVER_CA_FILE" default:""` - DisableSecretCreation bool `envconfig:"DISABLE_SECRET_CREATION" default:"false"` - TestkubeExecutionNamespaces string `envconfig:"TESTKUBE_EXECUTION_NAMESPACES" default:""` + + TestkubeProTLSSecret string `envconfig:"TESTKUBE_PRO_TLS_SECRET" default:""` + TestkubeProRunnerCustomCASecret string `envconfig:"TESTKUBE_PRO_RUNNER_CUSTOM_CA_SECRET" default:""` + TestkubeWatcherNamespaces string `envconfig:"TESTKUBE_WATCHER_NAMESPACES" default:""` + GraphqlPort string `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"` + TestkubeRegistry string `envconfig:"TESTKUBE_REGISTRY" default:""` + TestkubePodStartTimeout time.Duration `envconfig:"TESTKUBE_POD_START_TIMEOUT" default:"30m"` + CDEventsTarget string `envconfig:"CDEVENTS_TARGET" default:""` + TestkubeDashboardURI string `envconfig:"TESTKUBE_DASHBOARD_URI" default:""` + DisableReconciler bool `envconfig:"DISABLE_RECONCILER" default:"false"` + TestkubeClusterName string `envconfig:"TESTKUBE_CLUSTER_NAME" default:""` + CompressArtifacts bool `envconfig:"COMPRESSARTIFACTS" default:"false"` + TestkubeHelmchartVersion string `envconfig:"TESTKUBE_HELMCHART_VERSION" default:""` + DebugListenAddr string `envconfig:"DEBUG_LISTEN_ADDR" default:"0.0.0.0:1337"` + EnableDebugServer bool `envconfig:"ENABLE_DEBUG_SERVER" default:"false"` + EnableSecretsEndpoint bool `envconfig:"ENABLE_SECRETS_ENDPOINT" default:"false"` + DisableMongoMigrations bool `envconfig:"DISABLE_MONGO_MIGRATIONS" default:"false"` + Debug bool `envconfig:"DEBUG" default:"false"` + EnableImageDataPersistentCache bool `envconfig:"TESTKUBE_ENABLE_IMAGE_DATA_PERSISTENT_CACHE" default:"false"` + ImageDataPersistentCacheKey string `envconfig:"TESTKUBE_IMAGE_DATA_PERSISTENT_CACHE_KEY" default:"testkube-image-cache"` + LogServerGrpcAddress string `envconfig:"LOG_SERVER_GRPC_ADDRESS" default:":9090"` + LogServerSecure bool `envconfig:"LOG_SERVER_SECURE" default:"false"` + LogServerSkipVerify bool `envconfig:"LOG_SERVER_SKIP_VERIFY" default:"false"` + LogServerCertFile string `envconfig:"LOG_SERVER_CERT_FILE" default:""` + LogServerKeyFile string `envconfig:"LOG_SERVER_KEY_FILE" default:""` + LogServerCAFile string `envconfig:"LOG_SERVER_CA_FILE" default:""` + DisableSecretCreation bool `envconfig:"DISABLE_SECRET_CREATION" default:"false"` + TestkubeExecutionNamespaces string `envconfig:"TESTKUBE_EXECUTION_NAMESPACES" default:""` // DEPRECATED: Use TestkubeProAPIKey instead TestkubeCloudAPIKey string `envconfig:"TESTKUBE_CLOUD_API_KEY" default:""` @@ -119,6 +120,13 @@ type Config struct { TestkubeCloudOrgID string `envconfig:"TESTKUBE_CLOUD_ORG_ID" default:""` // DEPRECATED: Use TestkubeProMigrate instead TestkubeCloudMigrate string `envconfig:"TESTKUBE_CLOUD_MIGRATE" default:"false"` + + // TestkubeProCAFile is meant to provide a custom CA when making a TLS connection to + // the agent API. + // + // Deprecated: Instead mount a CA file into a directory and specify the diretory + // path with the SSL_CERT_DIR environment variable. + TestkubeProCAFile string `envconfig:"TESTKUBE_PRO_CA_FILE" default:""` } func Get() (*Config, error) { diff --git a/pkg/executor/client/common.go b/pkg/executor/client/common.go index 246e87f5e46..3f19247b0b9 100644 --- a/pkg/executor/client/common.go +++ b/pkg/executor/client/common.go @@ -23,18 +23,19 @@ const ( ) type ExecuteOptions struct { - ID string - TestName string - Namespace string - TestSpec testsv3.TestSpec - ExecutorName string - ExecutorSpec executorv1.ExecutorSpec - Request testkube.ExecutionRequest - Sync bool - Labels map[string]string - UsernameSecret *testkube.SecretRef - TokenSecret *testkube.SecretRef - CertificateSecret string + ID string + TestName string + Namespace string + TestSpec testsv3.TestSpec + ExecutorName string + ExecutorSpec executorv1.ExecutorSpec + Request testkube.ExecutionRequest + Sync bool + Labels map[string]string + UsernameSecret *testkube.SecretRef + TokenSecret *testkube.SecretRef + RunnerCustomCASecret string + CertificateSecret string // AgentAPITLSSecret is a secret name that contains TLS certificate for Agent (gRPC) API AgentAPITLSSecret string ImagePullSecretNames []string diff --git a/pkg/executor/client/job.go b/pkg/executor/client/job.go index a1df26ad53f..cfce8019f7d 100644 --- a/pkg/executor/client/job.go +++ b/pkg/executor/client/job.go @@ -171,6 +171,7 @@ type JobOptions struct { HTTPSProxy string UsernameSecret *testkube.SecretRef TokenSecret *testkube.SecretRef + RunnerCustomCASecret string CertificateSecret string AgentAPITLSSecret string Variables map[string]testkube.Variable @@ -630,6 +631,7 @@ func NewJobOptionsFromExecutionOptions(options ExecuteOptions) JobOptions { HTTPSProxy: options.Request.HttpsProxy, UsernameSecret: options.UsernameSecret, TokenSecret: options.TokenSecret, + RunnerCustomCASecret: options.RunnerCustomCASecret, CertificateSecret: options.CertificateSecret, ActiveDeadlineSeconds: options.Request.ActiveDeadlineSeconds, JobTemplateExtensions: options.Request.JobTemplate, diff --git a/pkg/executor/containerexecutor/containerexecutor.go b/pkg/executor/containerexecutor/containerexecutor.go index c7f3374b0dd..0a623d77607 100644 --- a/pkg/executor/containerexecutor/containerexecutor.go +++ b/pkg/executor/containerexecutor/containerexecutor.go @@ -167,6 +167,7 @@ type JobOptions struct { HTTPSProxy string UsernameSecret *testkube.SecretRef TokenSecret *testkube.SecretRef + RunnerCustomCASecret string CertificateSecret string AgentAPITLSSecret string Variables map[string]testkube.Variable @@ -703,6 +704,7 @@ func NewJobOptionsFromExecutionOptions(options client.ExecuteOptions) *JobOption HTTPSProxy: options.Request.HttpsProxy, UsernameSecret: options.UsernameSecret, TokenSecret: options.TokenSecret, + RunnerCustomCASecret: options.RunnerCustomCASecret, CertificateSecret: options.CertificateSecret, AgentAPITLSSecret: options.AgentAPITLSSecret, ActiveDeadlineSeconds: options.Request.ActiveDeadlineSeconds, diff --git a/pkg/executor/containerexecutor/templates/job.tmpl b/pkg/executor/containerexecutor/templates/job.tmpl index cfa6d76e4fd..838862b321d 100644 --- a/pkg/executor/containerexecutor/templates/job.tmpl +++ b/pkg/executor/containerexecutor/templates/job.tmpl @@ -22,6 +22,11 @@ spec: command: - "/bin/runner" - '{{ .Jsn }}' + {{- if .RunnerCustomCASecret }} + env: + - name: SSL_CERT_DIR + value: /etc/testkube/certs + {{- end }} volumeMounts: - name: data-volume mountPath: /data @@ -29,6 +34,12 @@ spec: - name: {{ .CertificateSecret }} mountPath: /etc/certs {{- end }} + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + mountPath: /etc/testkube/certs/testkube-custom-ca.pem + readOnly: true + subPath: ca.crt + {{- end }} {{- if .ArtifactRequest }} {{- if .ArtifactRequest.VolumeMountPath }} - name: artifact-volume @@ -49,7 +60,7 @@ spec: {{- end }} {{- end }} containers: - {{ if .Features.LogsV2 -}} + {{ if .Features.LogsV2 -}} - name: "{{ .Name }}-logs" image: {{ .Registry }}/{{ .LogSidecarImage }} env: @@ -84,6 +95,11 @@ spec: {{- if .WorkingDir }} workingDir: {{ .WorkingDir }} {{- end }} + {{- if .RunnerCustomCASecret }} + env: + - name: SSL_CERT_DIR + value: /etc/testkube/certs + {{- end }} volumeMounts: - name: data-volume mountPath: /data @@ -91,6 +107,12 @@ spec: - name: {{ .CertificateSecret }} mountPath: /etc/certs {{- end }} + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + mountPath: /etc/testkube/certs/testkube-custom-ca.pem + readOnly: true + subPath: ca.crt + {{- end }} {{- if .ArtifactRequest }} {{- if .ArtifactRequest.VolumeMountPath }} - name: artifact-volume @@ -117,6 +139,12 @@ spec: secret: secretName: {{ .CertificateSecret }} {{- end }} + {{- if .RunnerCustomCASecret }} + - name: {{ .RunnerCustomCASecret }} + secret: + secretName: {{ .RunnerCustomCASecret }} + defaultMode: 420 + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume diff --git a/pkg/scheduler/service.go b/pkg/scheduler/service.go index 2b50e3007ed..50f032c7ad3 100644 --- a/pkg/scheduler/service.go +++ b/pkg/scheduler/service.go @@ -46,6 +46,7 @@ type Scheduler struct { subscriptionChecker checktcl.SubscriptionChecker namespace string agentAPITLSSecret string + runnerCustomCASecret string } func NewScheduler( @@ -70,6 +71,7 @@ func NewScheduler( logsStream logsclient.Stream, namespace string, agentAPITLSSecret string, + runnerCustomCASecret string, ) *Scheduler { return &Scheduler{ metrics: metrics, @@ -93,6 +95,7 @@ func NewScheduler( logsStream: logsStream, namespace: namespace, agentAPITLSSecret: agentAPITLSSecret, + runnerCustomCASecret: runnerCustomCASecret, } } diff --git a/pkg/scheduler/test_scheduler.go b/pkg/scheduler/test_scheduler.go index 657a13fca64..cf7d58256cb 100644 --- a/pkg/scheduler/test_scheduler.go +++ b/pkg/scheduler/test_scheduler.go @@ -556,6 +556,7 @@ func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.Exe Labels: testCR.Labels, UsernameSecret: usernameSecret, TokenSecret: tokenSecret, + RunnerCustomCASecret: s.runnerCustomCASecret, CertificateSecret: certificateSecret, AgentAPITLSSecret: s.agentAPITLSSecret, ImagePullSecretNames: imagePullSecrets, diff --git a/pkg/triggers/executor_test.go b/pkg/triggers/executor_test.go index ee93be8a0df..bb2e376cf17 100644 --- a/pkg/triggers/executor_test.go +++ b/pkg/triggers/executor_test.go @@ -128,6 +128,7 @@ func TestExecute(t *testing.T) { mockLogsStream, "", "", + "", ) s := &Service{ triggerStatus: make(map[statusKey]*triggerStatus), diff --git a/pkg/triggers/service_test.go b/pkg/triggers/service_test.go index 1a7192c4ae6..feeeb52963d 100644 --- a/pkg/triggers/service_test.go +++ b/pkg/triggers/service_test.go @@ -141,6 +141,7 @@ func TestService_Run(t *testing.T) { mockLogsStream, "", "", + "", ) mockLeaseBackend := NewMockLeaseBackend(mockCtrl)