From caafe8fd5bb4742a595d4e03422d3398c46760c4 Mon Sep 17 00:00:00 2001 From: Upendra Vedullapalli Date: Tue, 3 Oct 2023 09:09:56 +0200 Subject: [PATCH 1/2] feat(datahub-ingestion-cron): added support for configuring securityContext for all ingestion crons --- charts/datahub/Chart.yaml | 4 +- .../datahub-ingestion-cron/Chart.yaml | 2 +- .../datahub-ingestion-cron/README.md | 66 ++++++++++--------- .../templates/cron.yaml | 6 ++ .../datahub-ingestion-cron/values.yaml | 12 ++++ 5 files changed, 55 insertions(+), 35 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 149e95c10..e3e8a4218 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.188 +version: 0.2.189 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.11.0 @@ -26,7 +26,7 @@ dependencies: repository: file://./subcharts/datahub-mce-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-ingestion-cron - version: 0.2.136 + version: 0.2.137 repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml index 7f70d6aad..6e4ebc44d 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.136 +version: 0.2.137 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/README.md b/charts/datahub/subcharts/datahub-ingestion-cron/README.md index 900b2b15a..c545de242 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/README.md +++ b/charts/datahub/subcharts/datahub-ingestion-cron/README.md @@ -4,35 +4,37 @@ A Helm chart for datahub's metadata-ingestion framework with kerberos authentica ## Chart Values -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"linkedin/datahub-ingestion"` | DataHub Ingestion image repository | -| image.tag | string | `"head"` | DataHub Ingestion image tag | -| imagePullSecrets | array | `[]` (does not add image pull secrets to deployed pods) | Docker registry secret names as an array | -| labels | string | `{}` | Metadata labels to be added to each crawling cron job | -| crons | type | `{}` | A map of crawling parameters per different technology being crawler, the key in the object will be used as the name for the new cron job | -| crons.schedule | string | `"0 0 * * *"` | Cron expression (default is daily at midnight) for crawler jobs | -| crons.recipe | object | `{}` | Recipe configuration to be executed (required) | -| crons.recipe.configmapName | string | `""` | Name of configmap to be mounted containing recipe to be executed | -| crons.recipe.fileName | string | `""` | Name of property within configMap referenced by `recipe.configName` with the concrete recipe definition | -| crons.command | array | `["/bin/sh", "-c", "datahub ingest -c /etc/recipe/"]` | Array of strings denoting the crawling command to be invoked in the cron job. By default it will execute the recipe defined in the `crons.recipe` object. Cron crawling customization is possible by having extra volumes with custom logic to be executed. | -| crons.hostAliases | array | `[]` | host aliases | -| crons.env | object | `{}` | Environment variables to add to the cronjob container | -| crons.envFromSecrets | object | `{}` | Environment variables from secrets to the cronjob container | -| crons.envFromSecrets*.secret | string | | secretKeyRef.name used for environment variable | -| crons.envFromSecrets*.key | string | | secretKeyRef.key used for environment variable | -| crons.extraVolumes | array | `[]` | Additional volumes to add to the pods | -| crons.extraVolumeMounts | array | `[]` | Additional volume mounts to add to the pods | -| crons.extraInitContainers | object | `{}` | Init containers to add to the cronjob container | -| crons.serviceAccountName | string | | Service account name used for the cronjob container | -| crons.podAnnotations | object | `{}` | Annotations to add to the pods | -| crons.restartPolicy | string | `"Always"` | Pod restart policy | -| crons.concurrencyPolicy | string | `"Allow"` | Specifies how to treat concurrent executions of a job | -| crons.failedJobsHistoryLimit | integer | `1` | Number of failed finished jobs to retain | -| crons.successfulJobsHistoryLimit | integer | `3` | Number of successful finished jobs to retain | -| crons.backoffLimit | integer | `6` | Number of retries before marking job failed | -| crons.nodeSelector | object | `{}` | Node labels for pod assignment | -| crons.affinity | object | `{}` | Affinity for pod assignment | -| crons.tolerations | list | `[]` | Tolerations for pod assignment | -| crons.extraSidecars | list | `[]` | Add sidecar containers to the pod | +| Key | Type | Default | Description | +|----------------------------------|---------|------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.repository | string | `"linkedin/datahub-ingestion"` | DataHub Ingestion image repository | +| image.tag | string | `"head"` | DataHub Ingestion image tag | +| imagePullSecrets | array | `[]` (does not add image pull secrets to deployed pods) | Docker registry secret names as an array | +| labels | string | `{}` | Metadata labels to be added to each crawling cron job | +| podSecurityContext | object | `{}` | Pod security context for cron jobs | +| crons | type | `{}` | A map of crawling parameters per different technology being crawler, the key in the object will be used as the name for the new cron job | +| crons.schedule | string | `"0 0 * * *"` | Cron expression (default is daily at midnight) for crawler jobs | +| crons.recipe | object | `{}` | Recipe configuration to be executed (required) | +| crons.recipe.configmapName | string | `""` | Name of configmap to be mounted containing recipe to be executed | +| crons.recipe.fileName | string | `""` | Name of property within configMap referenced by `recipe.configName` with the concrete recipe definition | +| crons.command | array | `["/bin/sh", "-c", "datahub ingest -c /etc/recipe/"]` | Array of strings denoting the crawling command to be invoked in the cron job. By default it will execute the recipe defined in the `crons.recipe` object. Cron crawling customization is possible by having extra volumes with custom logic to be executed. | +| crons.hostAliases | array | `[]` | host aliases | +| crons.env | object | `{}` | Environment variables to add to the cronjob container | +| crons.envFromSecrets | object | `{}` | Environment variables from secrets to the cronjob container | +| crons.envFromSecrets*.secret | string | | secretKeyRef.name used for environment variable | +| crons.envFromSecrets*.key | string | | secretKeyRef.key used for environment variable | +| crons.extraVolumes | array | `[]` | Additional volumes to add to the pods | +| crons.extraVolumeMounts | array | `[]` | Additional volume mounts to add to the pods | +| crons.extraInitContainers | object | `{}` | Init containers to add to the cronjob container | +| crons.serviceAccountName | string | | Service account name used for the cronjob container | +| crons.securityContext | object | `{}` | SecurityContext specific to each crawling cron job | +| crons.podAnnotations | object | `{}` | Annotations to add to the pods | +| crons.restartPolicy | string | `"Always"` | Pod restart policy | +| crons.concurrencyPolicy | string | `"Allow"` | Specifies how to treat concurrent executions of a job | +| crons.failedJobsHistoryLimit | integer | `1` | Number of failed finished jobs to retain | +| crons.successfulJobsHistoryLimit | integer | `3` | Number of successful finished jobs to retain | +| crons.backoffLimit | integer | `6` | Number of retries before marking job failed | +| crons.nodeSelector | object | `{}` | Node labels for pod assignment | +| crons.affinity | object | `{}` | Affinity for pod assignment | +| crons.tolerations | list | `[]` | Tolerations for pod assignment | +| crons.extraSidecars | list | `[]` | Add sidecar containers to the pod | diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml index 2f8001458..da33982dc 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml @@ -37,6 +37,8 @@ spec: {{- if .serviceAccountName }} serviceAccountName: {{ .serviceAccountName }} {{- end }} + securityContext: + {{- toYaml $.Values.podSecurityContext | nindent 12 }} containers: - name: {{ $jobName }}-crawler image: "{{ $.Values.image.repository }}:{{ required "Global or specific tag is required" ($.Values.image.tag | default $.Values.global.datahub.version) }}" @@ -48,6 +50,10 @@ spec: {{- toYaml .extraVolumeMounts | nindent 14 }} {{- end }} command: ["/bin/sh", "-c", {{ default $defaultCommand .command }} ] + {{- if .securityContext }} + securityContext: + {{- toYaml .securityContext | nindent 14 }} + {{- end }} env: {{- if .env }} {{- range $key,$value := .env }} diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml index 6f3825a8c..231dbc70b 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml @@ -9,6 +9,9 @@ image: imagePullSecrets: [] +podSecurityContext: {} +# fsGroup: 2000 + crons: {} #### Example data #hive: @@ -62,6 +65,15 @@ crons: {} ## #serviceAccountName: "my-cron-service" + ## If you want to specify container level security + #securityContext: + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + ## Add your own pod annotations. ## #podAnnotations: {} From 5a044b4f19e1ec24e249cd0ff61c55b2ada2a03e Mon Sep 17 00:00:00 2001 From: Upendra Vedullapalli Date: Tue, 3 Oct 2023 09:25:10 +0200 Subject: [PATCH 2/2] feat(datahub-ingestion-cron): Merge conflicts from master --- charts/datahub/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 778023a74..8cab3e366 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.189 +version: 0.2.190 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.11.0