From 30eee991bbad5b398051758c1a97a5d028430427 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 15 Aug 2023 07:59:18 -0500 Subject: [PATCH 01/16] docs(readme): document secrets randomization (#350) * docs(readme): Add notes about randomized keys and credentials --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 46355ed73..49027c0cb 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,26 @@ kubectl port-forward 9002:9002 You should be able to access the frontend via http://localhost:9002. Once you confirm that the pods are running well, you can set up ingress for datahub-frontend -to expose the 9002 port to the public. +to expose the 9002 port to the public. + +## Default Credentials + +There are a few keys and credentials created as part of the deployment using randomized values. They can be overridden using various configuration parameters. + +Also consider changing the default credentials used by any of the underlying data stores pulled in by the companion helm chart for +the prerequisites. Refer to the upstream helm charts or point to your own managed data stores for these components. + +### DataHub Login + +For controlling the default admin password, see the following [configuration](charts/datahub/values.yaml#L36). + +### Encryption Key + +Used by the Play framework and GMS to encrypt secrets at the application level, this can be configured [here](charts/datahub/values.yaml#L579). + +### Token Signing Key + +Used to sign tokens for authentication, see configuration [here](charts/datahub/values.yaml#L605). ## Contributing From ea8a17860f053c63387b8309e1f77c0e1462a1b3 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 17 Aug 2023 22:36:12 -0500 Subject: [PATCH 02/16] fix(auth-secrets): fix system update secrets (#351) * fix(auth-secret): remove auth secret from common template, cannot be used by all jobs --- charts/datahub/Chart.yaml | 2 +- charts/datahub/templates/datahub-upgrade/_upgrade.tpl | 9 --------- .../datahub-upgrade/datahub-cleanup-job-template.yml | 9 +++++++++ .../datahub-upgrade/datahub-nocode-migration-job.yml | 9 +++++++++ .../datahub-restore-indices-job-template.yml | 9 +++++++++ 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index da510e51f..43e2cf266 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.181 +version: 0.2.182 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.10.5 diff --git a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl index dda094575..e67ea962b 100644 --- a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl +++ b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl @@ -39,15 +39,6 @@ Return the env variables for upgrade jobs value: "{{ .Values.global.sql.datasource.url }}" - name: EBEAN_DATASOURCE_DRIVER value: "{{ .Values.global.sql.datasource.driver }}" -{{- if .Values.global.datahub.metadata_service_authentication.enabled }} -- name: DATAHUB_SYSTEM_CLIENT_ID - value: {{ .Values.global.datahub.metadata_service_authentication.systemClientId }} -- name: DATAHUB_SYSTEM_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} - key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} -{{- end }} - name: KAFKA_BOOTSTRAP_SERVER value: "{{ .Values.global.kafka.bootstrap.server }}" {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml index fdd539d04..54d51b8f6 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml @@ -58,6 +58,15 @@ spec: args: [ "-u", "NoCodeDataMigrationCleanup" ] env: {{- include "datahub.upgrade.env" . | nindent 16}} + {{- if .Values.global.datahub.metadata_service_authentication.enabled }} + - name: DATAHUB_SYSTEM_CLIENT_ID + value: {{ .Values.global.datahub.metadata_service_authentication.systemClientId }} + - name: DATAHUB_SYSTEM_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} + key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} + {{- end }} {{- with .Values.datahubUpgrade.extraEnvs }} {{- toYaml . | nindent 16 }} {{- end }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml index 7649584e8..311c1f3a5 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml @@ -68,6 +68,15 @@ spec: - "dbType={{ .Values.datahubUpgrade.noCodeDataMigration.sqlDbType }}" env: {{- include "datahub.upgrade.env" . | nindent 12}} + {{- if .Values.global.datahub.metadata_service_authentication.enabled }} + - name: DATAHUB_SYSTEM_CLIENT_ID + value: {{ .Values.global.datahub.metadata_service_authentication.systemClientId }} + - name: DATAHUB_SYSTEM_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} + key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} + {{- end }} {{- with .Values.datahubUpgrade.extraEnvs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml index 7fbdde2aa..167b0d63a 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml @@ -70,6 +70,15 @@ spec: - "batchDelayMs={{ .Values.datahubUpgrade.batchDelayMs }}" env: {{- include "datahub.upgrade.env" . | nindent 16}} + {{- if .Values.global.datahub.metadata_service_authentication.enabled }} + - name: DATAHUB_SYSTEM_CLIENT_ID + value: {{ .Values.global.datahub.metadata_service_authentication.systemClientId }} + - name: DATAHUB_SYSTEM_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} + key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} + {{- end }} {{- with .Values.datahubUpgrade.extraEnvs }} {{- toYaml . | nindent 16 }} {{- end }} From a8baacd6819a8ae2bdaf62f27670f74be47ea95b Mon Sep 17 00:00:00 2001 From: Kohei Watanabe Date: Wed, 6 Sep 2023 12:39:35 +0900 Subject: [PATCH 03/16] fix(datahub-system-update-job): enable `SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS` (#358) * fix(datahub-system-update-job): enable SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS --- charts/datahub/Chart.yaml | 2 +- .../templates/datahub-upgrade/datahub-system-update-job.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 43e2cf266..ecd6dc00e 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.182 +version: 0.2.183 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.10.5 diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml index e03e8b38f..261f0b921 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml @@ -70,7 +70,7 @@ spec: - name: SCHEMA_REGISTRY_SYSTEM_UPDATE value: "true" - name: SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS - value: "false" + value: "true" - name: SPRING_KAFKA_PROPERTIES_USE_LATEST_VERSION value: "true" {{- end }} From 0663c57198ec819895682ff5c1d939a46096412c Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Wed, 6 Sep 2023 09:19:52 -0500 Subject: [PATCH 04/16] feat(session): add session duration configuration (#361) * feat(session): add session duration configuration --- charts/datahub/Chart.yaml | 4 +- .../subcharts/datahub-frontend/Chart.yaml | 2 +- .../subcharts/datahub-frontend/README.md | 127 +++++++++--------- .../templates/deployment.yaml | 2 + .../subcharts/datahub-frontend/values.yaml | 2 + 5 files changed, 71 insertions(+), 66 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index ecd6dc00e..951ffba16 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.183 +version: 0.2.184 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.10.5 @@ -14,7 +14,7 @@ dependencies: repository: file://./subcharts/datahub-gms condition: datahub-gms.enabled - name: datahub-frontend - version: 0.2.139 + version: 0.2.140 repository: file://./subcharts/datahub-frontend condition: datahub-frontend.enabled - name: datahub-mae-consumer diff --git a/charts/datahub/subcharts/datahub-frontend/Chart.yaml b/charts/datahub/subcharts/datahub-frontend/Chart.yaml index 7437c207c..4b5d0f601 100644 --- a/charts/datahub/subcharts/datahub-frontend/Chart.yaml +++ b/charts/datahub/subcharts/datahub-frontend/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.139 +version: 0.2.140 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-frontend/README.md b/charts/datahub/subcharts/datahub-frontend/README.md index 7c14edbe7..3b04d39e7 100644 --- a/charts/datahub/subcharts/datahub-frontend/README.md +++ b/charts/datahub/subcharts/datahub-frontend/README.md @@ -6,66 +6,67 @@ Current chart version is `0.2.0` ## Chart Values -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | object | `{}` | | -| datahub.play.mem.buffer.size | string | `"10MB"` | | -| existingGmsSecret | object | {} | Reference to GMS secret if already exists | -| exporters.jmx.enabled | boolean | false | | -| extraLabels | object | `{}` | Extra labels for deployment configuration | -| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | -| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | -| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | -| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | -| fullnameOverride | string | `"datahub-frontend"` | | -| global.datahub_analytics_enabled | boolean | true | | -| global.datahub.gms.port | string | `"8080"` | | -| image.pullPolicy | string | `"IfNotPresent"` | | -| image.repository | string | `"linkedin/datahub-frontend-react"` | | -| image.tag | string | `"head"` | | -| imagePullSecrets | list | `[]` | | -| ingress.annotations | object | `{}` | | -| ingress.enabled | bool | `false` | | -| ingress.extraLabels | object | `{}` | provides extra labels for ingress configuration | -| ingress.hosts[0].host | string | `"chart-example.local"` | | -| ingress.hosts[0].paths | list | `[]` | | -| ingress.hosts[0].redirectPaths | list | `[]` | | -| ingress.tls | list | `[]` | | -| livenessProbe.initialDelaySeconds | int | `60` | | -| livenessProbe.periodSeconds | int | `30` | | -| livenessProbe.failureThreshold | int | `4` | | -| nameOverride | string | `""` | | -| nodeSelector | object | `{}` | | -| oidcAuthentication.enabled | boolean | `false` | Enable [OIDC authentication](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react) | -| oidcAuthentication.provider | string | `""` | One of the supported OIDC providers: [google](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-google), [okta](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-okta), or [azure](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-azure) | -| oidcAuthentication.clientId | string | `""` | A unique identifier for your application with the identity provider | -| oidcAuthentication.clientSecret | string | `""` | A shared secret to use for exchange between you and your identity provider | -| oidcAuthentication.clientSecretRef.secretRef | string | `"nil"` | Optional, this is the reference to the shared secret to use for exchange between you and your identity provider | -| oidcAuthentication.clientSecretRef.secretKey | string | `"nil"` | Optional, this is the key of the shared secret to use for exchange between you and your identity provider | -| oidcAuthentication.oktaDomain | string | `""` | Okta domain, e.g. `dev-12345.okta.com`; needed only if `provider` is set to `okta` | -| oidcAuthentication.azureTenantId | string | `""` | Azure directory (tenant) ID; neede only if `provider` is set to `azure` | -| podAnnotations | object | `{}` | | -| podSecurityContext | object | `{}` | | -| readinessProbe.initialDelaySeconds | int | `60` | | -| readinessProbe.periodSeconds | int | `30` | | -| readinessProbe.failureThreshold | int | `4` | | -| replicaCount | int | `1` | | -| revisionHistoryLimit | int | `10` | | -| lifecycle | object | `{}` | | -| resources | object | `{}` | | -| securityContext | object | `{}` | | -| service.port | int | `9001` | | -| service.nodePort | int | `""` | | -| service.type | string | `"LoadBalancer"` | | -| serviceAccount.annotations | object | `{}` | | -| serviceAccount.create | bool | `true` | | -| serviceAccount.name | string | `nil` | | -| serviceMonitor.create | bool | `false` | If set true and `global.datahub.monitoring.enablePrometheus` is set `true` it will create a ServiceMonitor resource | -| tolerations | list | `[]` | | -| global.elasticsearch.host | string | `"elasticsearch"` | | -| global.elasticsearch.port | string | `"9200"` | | -| global.kafka.bootstrap.server | string | `"broker:9092"` | | -| datahub.metadata_service_authentication.enabled | bool | `false` | Whether Metadata Service Authentication is enabled. | -| global.datahub.metadata_service_authentication.systemClientId | string | `"__datahub_system"` | The internal system id that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | -| global.datahub.metadata_service_authentication.systemClientSecret.secretRef | string | `nil` | The reference to a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | -| global.datahub.metadata_service_authentication.systemClientSecret.secretKey | string | `nil` | The key of a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | +| Key | Type | Default | Description | +|-----------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| affinity | object | `{}` | | +| datahub.play.mem.buffer.size | string | `"10MB"` | | +| existingGmsSecret | object | {} | Reference to GMS secret if already exists | +| exporters.jmx.enabled | boolean | false | | +| extraLabels | object | `{}` | Extra labels for deployment configuration | +| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | +| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | +| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | +| fullnameOverride | string | `"datahub-frontend"` | | +| global.datahub_analytics_enabled | boolean | true | | +| global.datahub.gms.port | string | `"8080"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"linkedin/datahub-frontend-react"` | | +| image.tag | string | `"head"` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.enabled | bool | `false` | | +| ingress.extraLabels | object | `{}` | provides extra labels for ingress configuration | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths | list | `[]` | | +| ingress.hosts[0].redirectPaths | list | `[]` | | +| ingress.tls | list | `[]` | | +| livenessProbe.initialDelaySeconds | int | `60` | | +| livenessProbe.periodSeconds | int | `30` | | +| livenessProbe.failureThreshold | int | `4` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| auth.sessionTTLHours | string | `24` | Configures the length of time a session token is valid for after creation. | +| oidcAuthentication.enabled | boolean | `false` | Enable [OIDC authentication](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react) | +| oidcAuthentication.provider | string | `""` | One of the supported OIDC providers: [google](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-google), [okta](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-okta), or [azure](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-azure) | +| oidcAuthentication.clientId | string | `""` | A unique identifier for your application with the identity provider | +| oidcAuthentication.clientSecret | string | `""` | A shared secret to use for exchange between you and your identity provider | +| oidcAuthentication.clientSecretRef.secretRef | string | `"nil"` | Optional, this is the reference to the shared secret to use for exchange between you and your identity provider | +| oidcAuthentication.clientSecretRef.secretKey | string | `"nil"` | Optional, this is the key of the shared secret to use for exchange between you and your identity provider | +| oidcAuthentication.oktaDomain | string | `""` | Okta domain, e.g. `dev-12345.okta.com`; needed only if `provider` is set to `okta` | +| oidcAuthentication.azureTenantId | string | `""` | Azure directory (tenant) ID; neede only if `provider` is set to `azure` | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| readinessProbe.initialDelaySeconds | int | `60` | | +| readinessProbe.periodSeconds | int | `30` | | +| readinessProbe.failureThreshold | int | `4` | | +| replicaCount | int | `1` | | +| revisionHistoryLimit | int | `10` | | +| lifecycle | object | `{}` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.port | int | `9001` | | +| service.nodePort | int | `""` | | +| service.type | string | `"LoadBalancer"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `nil` | | +| serviceMonitor.create | bool | `false` | If set true and `global.datahub.monitoring.enablePrometheus` is set `true` it will create a ServiceMonitor resource | +| tolerations | list | `[]` | | +| global.elasticsearch.host | string | `"elasticsearch"` | | +| global.elasticsearch.port | string | `"9200"` | | +| global.kafka.bootstrap.server | string | `"broker:9092"` | | +| datahub.metadata_service_authentication.enabled | bool | `false` | Whether Metadata Service Authentication is enabled. | +| global.datahub.metadata_service_authentication.systemClientId | string | `"__datahub_system"` | The internal system id that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | +| global.datahub.metadata_service_authentication.systemClientSecret.secretRef | string | `nil` | The reference to a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | +| global.datahub.metadata_service_authentication.systemClientSecret.secretKey | string | `nil` | The key of a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | diff --git a/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml b/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml index 914c97f81..5db08afbb 100644 --- a/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml @@ -174,6 +174,8 @@ spec: name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} {{- end }} + - name: AUTH_SESSION_TTL_HOURS + value: {{ .Values.auth.sessionTTLHours | quote }} {{- with .Values.oidcAuthentication }} {{- if .enabled }} - name: AUTH_OIDC_ENABLED diff --git a/charts/datahub/subcharts/datahub-frontend/values.yaml b/charts/datahub/subcharts/datahub-frontend/values.yaml index 2f4291106..520622b12 100644 --- a/charts/datahub/subcharts/datahub-frontend/values.yaml +++ b/charts/datahub/subcharts/datahub-frontend/values.yaml @@ -68,6 +68,8 @@ ingress: # hosts: # - chart-example.local +auth: + sessionTTLHours: "24" # OIDC auth based on https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react oidcAuthentication: enabled: false From f4cd1ffc74e0d8809b006b90fa1009c1af73030d Mon Sep 17 00:00:00 2001 From: sachinsaju <33017477+sachinsaju@users.noreply.github.com> Date: Thu, 7 Sep 2023 20:04:32 +0530 Subject: [PATCH 05/16] fix(ingestion-cron): fix indentation in ingestion cron template (#356) * fix: remove unused and irrelevant sidecar configuration * fix: correct sidecar configuration in cron job template * fix: update ingestion cron sidecar parameter in README --------- Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- charts/datahub/Chart.yaml | 4 ++-- .../subcharts/datahub-ingestion-cron/Chart.yaml | 2 +- .../subcharts/datahub-ingestion-cron/README.md | 2 +- .../datahub-ingestion-cron/templates/cron.yaml | 6 +++--- .../subcharts/datahub-ingestion-cron/values.yaml | 12 ++++++------ 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 951ffba16..609737bf9 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.184 +version: 0.2.185 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.10.5 @@ -26,7 +26,7 @@ dependencies: repository: file://./subcharts/datahub-mce-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-ingestion-cron - version: 0.2.134 + version: 0.2.135 repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml index 95eca2002..941fb2053 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.134 +version: 0.2.135 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.10.0 diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/README.md b/charts/datahub/subcharts/datahub-ingestion-cron/README.md index df7feddda..d82a6f5cf 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/README.md +++ b/charts/datahub/subcharts/datahub-ingestion-cron/README.md @@ -27,4 +27,4 @@ A Helm chart for datahub's metadata-ingestion framework with kerberos authentica | crons.extraInitContainers | object | `{}` | Init containers to add to the cronjob container | | crons.serviceAccountName | string | | Service account name used for the cronjob container | | crons.podAnnotations | object | `{}` | Annotations to add to the pods | -| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | +| crons.extraSidecars | list | `[]` | Add sidecar containers to the pod | diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml index 5b2c417db..2f8001458 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml @@ -64,6 +64,9 @@ spec: key: {{ $value.key | quote}} {{- end }} {{- end }} + {{- if .extraSidecars }} + {{- toYaml .extraSidecars | nindent 10 }} + {{- end }} restartPolicy: {{ default "OnFailure" .restartPolicy }} {{- if .nodeSelector }} nodeSelector: @@ -84,8 +87,5 @@ spec: {{- if .extraVolumes }} {{- toYaml .extraVolumes | nindent 12 }} {{- end }} - {{- if .extraSidecars }} - {{- toYaml .extraSidecars | nindent 10 }} - {{- end }} --- {{- end }} diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml index f4ad84c1f..4e703a6be 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml @@ -100,12 +100,12 @@ crons: {} ## #tolerations: [] -# Add extra sidecar containers to deployment pod -extraSidecars: [] - # - name: my-image-name - # image: my-image - # imagePullPolicy: Always + ## Add extra sidecar containers to deployment pod + #extraSidecars: + # - name: my-image-name + # image: my-image + # imagePullPolicy: Always global: datahub: - version: head \ No newline at end of file + version: head From 062a629a075729ffdf35b3e3944cb52c7777fcd7 Mon Sep 17 00:00:00 2001 From: sachinsaju <33017477+sachinsaju@users.noreply.github.com> Date: Thu, 7 Sep 2023 20:05:33 +0530 Subject: [PATCH 06/16] docs(ingestion-cron): add documentation for ingestion cron job values (#355) --- charts/datahub/subcharts/datahub-ingestion-cron/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/README.md b/charts/datahub/subcharts/datahub-ingestion-cron/README.md index d82a6f5cf..900b2b15a 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/README.md +++ b/charts/datahub/subcharts/datahub-ingestion-cron/README.md @@ -27,4 +27,12 @@ A Helm chart for datahub's metadata-ingestion framework with kerberos authentica | crons.extraInitContainers | object | `{}` | Init containers to add to the cronjob container | | crons.serviceAccountName | string | | Service account name used for the cronjob container | | crons.podAnnotations | object | `{}` | Annotations to add to the pods | +| crons.restartPolicy | string | `"Always"` | Pod restart policy | +| crons.concurrencyPolicy | string | `"Allow"` | Specifies how to treat concurrent executions of a job | +| crons.failedJobsHistoryLimit | integer | `1` | Number of failed finished jobs to retain | +| crons.successfulJobsHistoryLimit | integer | `3` | Number of successful finished jobs to retain | +| crons.backoffLimit | integer | `6` | Number of retries before marking job failed | +| crons.nodeSelector | object | `{}` | Node labels for pod assignment | +| crons.affinity | object | `{}` | Affinity for pod assignment | +| crons.tolerations | list | `[]` | Tolerations for pod assignment | | crons.extraSidecars | list | `[]` | Add sidecar containers to the pod | From ff3ea3ac1694daa149b0c030574800c376b46812 Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Mon, 11 Sep 2023 18:23:50 -0300 Subject: [PATCH 07/16] Helm changes for 0.11.0 release (#366) Co-authored-by: Indy Prentice --- charts/datahub/Chart.yaml | 4 ++-- charts/datahub/quickstart-values-with-neo4j.yaml | 4 ++-- charts/datahub/values.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 609737bf9..ee2f81e6a 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,10 +4,10 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.185 +version: 0.2.186 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.10.5 +appVersion: 0.11.0 dependencies: - name: datahub-gms version: 0.2.150 diff --git a/charts/datahub/quickstart-values-with-neo4j.yaml b/charts/datahub/quickstart-values-with-neo4j.yaml index d616daf7a..95a62c8b9 100644 --- a/charts/datahub/quickstart-values-with-neo4j.yaml +++ b/charts/datahub/quickstart-values-with-neo4j.yaml @@ -107,7 +107,7 @@ global: secretKey: mysql-root-password datahub: - version: v0.10.5 + version: v0.11.0 gms: port: "8080" mae_consumer: @@ -116,4 +116,4 @@ global: managed_ingestion: enabled: true - defaultCliVersion: "0.10.5.4" + defaultCliVersion: "0.11.0" diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 0c840997a..47c2ba3fd 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -554,7 +554,7 @@ global: # value: password datahub: - version: v0.10.5 + version: v0.11.0 gms: port: "8080" nodePort: "30001" @@ -586,7 +586,7 @@ global: managed_ingestion: enabled: true - defaultCliVersion: "0.10.5.4" + defaultCliVersion: "0.11.0" metadata_service_authentication: enabled: false From 935171e26592497818d2b329c886c3a5827ee597 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 14 Sep 2023 10:04:06 -0500 Subject: [PATCH 08/16] Release v0.11.0 updates (#367) * feat(release): updates for release v0.11.0 --- charts/datahub/Chart.yaml | 12 ++++----- charts/datahub/README.md | 18 ++++++------- .../datahub/quickstart-values-with-neo4j.yaml | 14 +++++----- .../subcharts/datahub-frontend/Chart.yaml | 4 +-- .../datahub/subcharts/datahub-gms/Chart.yaml | 4 +-- .../datahub/subcharts/datahub-gms/README.md | 2 +- .../datahub-gms/templates/deployment.yaml | 2 +- .../templates/hazelcastService.yaml | 2 +- .../datahub-ingestion-cron/Chart.yaml | 4 +-- .../subcharts/datahub-mae-consumer/Chart.yaml | 4 +-- .../subcharts/datahub-mce-consumer/Chart.yaml | 4 +-- charts/datahub/values.yaml | 26 +++++++++---------- 12 files changed, 48 insertions(+), 48 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index ee2f81e6a..5c32de242 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,29 +4,29 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.186 +version: 0.2.187 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.11.0 dependencies: - name: datahub-gms - version: 0.2.150 + version: 0.2.151 repository: file://./subcharts/datahub-gms condition: datahub-gms.enabled - name: datahub-frontend - version: 0.2.140 + version: 0.2.141 repository: file://./subcharts/datahub-frontend condition: datahub-frontend.enabled - name: datahub-mae-consumer - version: 0.2.146 + version: 0.2.147 repository: file://./subcharts/datahub-mae-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-mce-consumer - version: 0.2.149 + version: 0.2.150 repository: file://./subcharts/datahub-mce-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-ingestion-cron - version: 0.2.135 + version: 0.2.136 repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions diff --git a/charts/datahub/README.md b/charts/datahub/README.md index b077dedf2..986a84bd4 100644 --- a/charts/datahub/README.md +++ b/charts/datahub/README.md @@ -22,17 +22,17 @@ helm install datahub datahub/datahub --values <> |-----|------|---------|-------------| | datahub-frontend.enabled | bool | `true` | Enable Datahub Front-end | | datahub-frontend.image.repository | string | `"linkedin/datahub-frontend-react"` | Image repository for datahub-frontend | -| datahub-frontend.image.tag | string | `"v0.10.4"` | Image tag for datahub-frontend | +| datahub-frontend.image.tag | string | `"v0.11.0"` | Image tag for datahub-frontend | | datahub-frontend.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-frontend | | datahub-gms.enabled | bool | `true` | Enable GMS | | datahub-gms.image.repository | string | `"linkedin/datahub-gms"` | Image repository for datahub-gms | -| datahub-gms.image.tag | string | `"v0.10.4"` | Image tag for datahub-gms | +| datahub-gms.image.tag | string | `"v0.11.0"` | Image tag for datahub-gms | | datahub-gms.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-gms | | datahub-mae-consumer.image.repository | string | `"linkedin/datahub-mae-consumer"` | Image repository for datahub-mae-consumer | -| datahub-mae-consumer.image.tag | string | `"v0.10.4"` | Image tag for datahub-mae-consumer | +| datahub-mae-consumer.image.tag | string | `"v0.11.0"` | Image tag for datahub-mae-consumer | | datahub-mae-consumer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-mae-consumer | | datahub-mce-consumer.image.repository | string | `"linkedin/datahub-mce-consumer"` | Image repository for datahub-mce-consumer | -| datahub-mce-consumer.image.tag | string | `"v0.10.4"` | Image tag for datahub-mce-consumer | +| datahub-mce-consumer.image.tag | string | `"v0.11.0"` | Image tag for datahub-mce-consumer | | datahub-mce-consumer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-mce-consumer | | datahub-ingestion-cron.enabled | bool | `false` | Enable cronjob for periodic ingestion | | datahubUpgrade.podSecurityContext | object | `{}` | Pod security context for datahubUpgrade jobs | @@ -44,7 +44,7 @@ helm install datahub datahub/datahub --values <> | datahubUpgrade.restoreIndices.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | elasticsearchSetupJob.enabled | bool | `true` | Enable setup job for elasicsearch | | elasticsearchSetupJob.image.repository | string | `"linkedin/datahub-elasticsearch-setup"` | Image repository for elasticsearchSetupJob | -| elasticsearchSetupJob.image.tag | string | `"v0.10.4"` | Image repository for elasticsearchSetupJob | +| elasticsearchSetupJob.image.tag | string | `"v0.11.0"` | Image repository for elasticsearchSetupJob | | elasticsearchSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for elasticsearchSetupJob | | elasticsearchSetupJob.resources | object | '{}' | Kube Resource definitions for elasticsearchSetupJob | | elasticsearchSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for elasticsearchSetupJob | @@ -53,7 +53,7 @@ helm install datahub datahub/datahub --values <> | elasticsearchSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | kafkaSetupJob.enabled | bool | `true` | Enable setup job for kafka | | kafkaSetupJob.image.repository | string | `"linkedin/datahub-kafka-setup"` | Image repository for kafkaSetupJob | -| kafkaSetupJob.image.tag | string | `"v0.10.4"` | Image repository for kafkaSetupJob | +| kafkaSetupJob.image.tag | string | `"v0.11.0"` | Image repository for kafkaSetupJob | | kafkaSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for kafkaSetupJob| | kafkaSetupJob.resources | object | '{}' | Kube Resource definitions for kafkaSetupJob | | kafkaSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for kafkaSetupJob | @@ -62,7 +62,7 @@ helm install datahub datahub/datahub --values <> | kafkaSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | mysqlSetupJob.enabled | bool | `false` | Enable setup job for mysql | | mysqlSetupJob.image.repository | string | `"acryldata/datahub-mysql-setup"` | Image repository for mysqlSetupJob | -| mysqlSetupJob.image.tag | string | `"v0.10.4"` | Image repository for mysqlSetupJob | +| mysqlSetupJob.image.tag | string | `"v0.11.0"` | Image repository for mysqlSetupJob | | mysqlSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for mysqlSetupJob| | mysqlSetupJob.resources | object | '{}' | Kube Resource definitions for mysqlSetupJob | | mysqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | @@ -71,7 +71,7 @@ helm install datahub datahub/datahub --values <> | mysqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | postgresqlSetupJob.enabled | bool | `false` | Enable setup job for postgresql | | postgresqlSetupJob.image.repository | string | `"acryldata/datahub-postgres-setup"` | Image repository for postgresqlSetupJob | -| postgresqlSetupJob.image.tag | string | `"v0.10.4"` | Image repository for postgresqlSetupJob | +| postgresqlSetupJob.image.tag | string | `"v0.11.0"` | Image repository for postgresqlSetupJob | | postgresqlSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for postgresqlSetupJob| | postgresqlSetupJob.resources | object | '{}' | Kube Resource definitions for postgresqlSetupJob | | postgresqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | @@ -164,7 +164,7 @@ helm install datahub datahub/datahub --values <> | global.datahub.managed_ingestion.enabled | bool | `true` | Whether or not UI-based ingestion experience is enabled. | | global.datahub.encryptionKey.secretRef | string | `datahub-encryption-secrets` | The reference to a secret containing an alpha-numeric encryption key, which is used to encrypt Secrets on DataHub. If a secret reference is not provided, a random one will be generated for you in a Kubernetes secret named `datahub-encryption-secrets`. | | global.datahub.encryptionKey.secretKey | string | `encryption_key_secret` | The key of a secret containing an alpha-numeric encryption key, which is used to encrypt Secrets on DataHub. If a secret reference is not provided, a random one will be generated for you in a Kubernetes secret value named `encryption_key_secret` within a secret named `datahub-encryption-secrets`. | -| global.datahub.managed_ingestion.defaultCliVersion | string | `` |0.10.0 This is the version of the DataHub CLI to use for UI ingestion, by default. | +| global.datahub.managed_ingestion.defaultCliVersion | string | `` |0.11.0 This is the version of the DataHub CLI to use for UI ingestion, by default. | | global.datahub.encryptionKey.provisionSecret.enabled | bool | `true` | Whether an encryption key secret should be provisioned on the first deployment for you. Set this to false if you are overriding global.datahub.encryptionKey.secretRef. | | global.datahub.encryptionKey.provisionSecret.autoGenerate | bool | `true` | Whether an encryption key secret should be provisioned for you **with a random seed** on the first deployment for you. Set this to false and use `global.datahub.encryptionKey.provisionSecret.secretValues.encryptionKey` if you would like to specify the secret values directly. | | global.datahub.encryptionKey.provisionSecret.secretValues.encryptionKey | string | `` | The encryption key value to be used if specified directly. | diff --git a/charts/datahub/quickstart-values-with-neo4j.yaml b/charts/datahub/quickstart-values-with-neo4j.yaml index 95a62c8b9..2d96a889c 100644 --- a/charts/datahub/quickstart-values-with-neo4j.yaml +++ b/charts/datahub/quickstart-values-with-neo4j.yaml @@ -4,14 +4,14 @@ datahub-gms: enabled: true image: repository: linkedin/datahub-gms - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version datahub-frontend: enabled: true image: repository: linkedin/datahub-frontend-react - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version # Set up ingress to expose react front-end ingress: @@ -40,35 +40,35 @@ elasticsearchSetupJob: enabled: true image: repository: linkedin/datahub-elasticsearch-setup - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version kafkaSetupJob: enabled: true image: repository: linkedin/datahub-kafka-setup - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version mysqlSetupJob: enabled: true image: repository: acryldata/datahub-mysql-setup - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version datahubUpgrade: enabled: true image: repository: acryldata/datahub-upgrade - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version datahub-ingestion-cron: enabled: false image: repository: acryldata/datahub-ingestion - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version global: diff --git a/charts/datahub/subcharts/datahub-frontend/Chart.yaml b/charts/datahub/subcharts/datahub-frontend/Chart.yaml index 4b5d0f601..1c07a61b5 100644 --- a/charts/datahub/subcharts/datahub-frontend/Chart.yaml +++ b/charts/datahub/subcharts/datahub-frontend/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.140 +version: 0.2.141 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml index 1c8eb0cbd..b9fcd685d 100644 --- a/charts/datahub/subcharts/datahub-gms/Chart.yaml +++ b/charts/datahub/subcharts/datahub-gms/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for LinkedIn DataHub's datahub-gms component type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.150 +version: 0.2.151 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-gms/README.md b/charts/datahub/subcharts/datahub-gms/README.md index 24df024ae..e258611cd 100644 --- a/charts/datahub/subcharts/datahub-gms/README.md +++ b/charts/datahub/subcharts/datahub-gms/README.md @@ -81,5 +81,5 @@ Current chart version is `0.2.0` | global.datahub.managed_ingestion.enabled | bool | `true` | Whether or not UI-based ingestion experience is enabled. | | global.datahub.encryptionKey.secretRef | string | `nil` | The reference to a secret containing an alpha-numeric encryption key, which is used to encrypt Secrets on DataHub. Required if managed_ingestion_enabled is 'true'. | | global.datahub.encryptionKey.secretKey | string | `nil` | The key of a secret containing an alpha-numeric encryption key, which is used to encrypt Secrets on DataHub. Required if managed_ingestion_enabled is 'true'. | -| global.datahub.managed_ingestion.defaultCliVersion | string | `0.10.0` | This is the version of the DataHub CLI to use for UI ingestion, by default. You do not need to explicitly provide this. By default the underlying datahub-gms container will provide a latest version compatible with the server. | +| global.datahub.managed_ingestion.defaultCliVersion | string | `0.11.0` | This is the version of the DataHub CLI to use for UI ingestion, by default. You do not need to explicitly provide this. By default the underlying datahub-gms container will provide a latest version compatible with the server. | | global.datahub.enable_retention | bool | `false` | Whether or not to enable retention on local DB | diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml index 45524b096..d4d55367b 100644 --- a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml @@ -113,7 +113,7 @@ spec: - name: SEARCH_SERVICE_CACHE_IMPLEMENTATION value: "hazelcast" - name: SEARCH_SERVICE_HAZELCAST_SERVICE_NAME - value: {{ printf "%s-%s" .Release.Name "hazelcast-service" }} + value: {{ printf "%s-%s-%s" .Release.Name (regexReplaceAll "\\W+" .Values.global.datahub.version "-") "hazelcast-svc" | trunc 63 | trimSuffix "-" }} {{- end}} {{- if .Values.global.datahub.systemUpdate.enabled }} - name: DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID diff --git a/charts/datahub/subcharts/datahub-gms/templates/hazelcastService.yaml b/charts/datahub/subcharts/datahub-gms/templates/hazelcastService.yaml index 50abdc524..b4a1aa427 100644 --- a/charts/datahub/subcharts/datahub-gms/templates/hazelcastService.yaml +++ b/charts/datahub/subcharts/datahub-gms/templates/hazelcastService.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ printf "%s-%s" .Release.Name "hazelcast-service" }} + name: {{ printf "%s-%s-%s" .Release.Name (regexReplaceAll "\\W+" .Values.global.datahub.version "-") "hazelcast-svc" | trunc 63 | trimSuffix "-" }} spec: clusterIP: None ports: diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml index 941fb2053..7f70d6aad 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.135 +version: 0.2.136 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml index 00939a712..e2ad31b49 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.146 +version: 0.2.147 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml index f2985ec66..647ff4d54 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.149 +version: 0.2.150 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 47c2ba3fd..597993ecb 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -4,7 +4,7 @@ datahub-gms: enabled: true image: repository: linkedin/datahub-gms - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version resources: limits: memory: 2Gi @@ -23,7 +23,7 @@ datahub-frontend: enabled: true image: repository: linkedin/datahub-frontend-react - # tag: "v0.10.0" # # defaults to .global.datahub.version + # tag: "v0.11.0" # # defaults to .global.datahub.version resources: limits: memory: 1400Mi @@ -59,7 +59,7 @@ acryl-datahub-actions: datahub-mae-consumer: image: repository: linkedin/datahub-mae-consumer - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: memory: 1536Mi @@ -70,7 +70,7 @@ datahub-mae-consumer: datahub-mce-consumer: image: repository: linkedin/datahub-mce-consumer - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: memory: 1536Mi @@ -82,13 +82,13 @@ datahub-ingestion-cron: enabled: false image: repository: acryldata/datahub-ingestion - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version elasticsearchSetupJob: enabled: true image: repository: linkedin/datahub-elasticsearch-setup - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: cpu: 500m @@ -118,7 +118,7 @@ kafkaSetupJob: enabled: true image: repository: linkedin/datahub-kafka-setup - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: cpu: 500m @@ -148,7 +148,7 @@ mysqlSetupJob: enabled: true image: repository: acryldata/datahub-mysql-setup - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: cpu: 500m @@ -183,7 +183,7 @@ postgresqlSetupJob: enabled: false image: repository: acryldata/datahub-postgres-setup - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: cpu: 500m @@ -223,7 +223,7 @@ datahubUpgrade: enabled: true image: repository: acryldata/datahub-upgrade - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version batchSize: 1000 batchDelayMs: 100 noCodeDataMigration: @@ -620,9 +620,9 @@ global: ## Values specific to the unified search and browse feature. search_and_browse: - show_search_v2: false # If on, show the new search filters experience as of v0.10.5 - show_browse_v2: false # If on, show the new browse experience as of v0.10.5 - backfill_browse_v2: false # If on, run the backfill upgrade job that generates default browse paths for relevant entities + show_search_v2: true # If on, show the new search filters experience as of v0.10.5 + show_browse_v2: true # If on, show the new browse experience as of v0.10.5 + backfill_browse_v2: true # If on, run the backfill upgrade job that generates default browse paths for relevant entities # hostAliases: # - ip: "192.168.0.104" From d8e8c551284675b892e7efe6a102338de47390a7 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 19 Sep 2023 04:05:31 -0500 Subject: [PATCH 09/16] doc(values): remove duplicate doc (#363) * doc(values): remove duplicate doc * Update Chart.yaml --------- Co-authored-by: Pedro Silva --- charts/datahub/Chart.yaml | 2 +- charts/datahub/subcharts/datahub-ingestion-cron/values.yaml | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 5c32de242..149e95c10 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.187 +version: 0.2.188 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.11.0 diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml index 4e703a6be..6f3825a8c 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml @@ -100,12 +100,6 @@ crons: {} ## #tolerations: [] - ## Add extra sidecar containers to deployment pod - #extraSidecars: - # - name: my-image-name - # image: my-image - # imagePullPolicy: Always - global: datahub: version: head From ef63fae2d436ff099f600c8909a96a1e98c90c5b Mon Sep 17 00:00:00 2001 From: Dimitri <36767102+dim-ops@users.noreply.github.com> Date: Tue, 19 Sep 2023 11:07:59 +0200 Subject: [PATCH 10/16] chore(gms): setup readiness & liveness from main chart (#314) * fix(gms): increase initialDelaySeconds readiness & liveness * chore(gms): setup readiness & liveness from main chart --------- Co-authored-by: Dimitri GRISARD --- charts/datahub/values.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 597993ecb..d0394cb68 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -11,6 +11,14 @@ datahub-gms: requests: cpu: 100m memory: 1Gi + livenessProbe: + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 8 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 30 + failureThreshold: 8 # Optionally set a GMS specific SQL login (defaults to global login) # sql: # datasource: From e725085d99dc5e48cd1706cfcd52636f9ed5ed83 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Fri, 29 Sep 2023 14:41:18 -0500 Subject: [PATCH 11/16] fix(secret): improve secret default behavior (#369) * fix(secret): improve length of default key to be more brute force resilient * Update Chart.yaml --- charts/datahub/Chart.yaml | 4 ++-- charts/datahub/subcharts/datahub-frontend/Chart.yaml | 2 +- .../datahub/subcharts/datahub-frontend/templates/secrets.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 149e95c10..5954c3240 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.188 +version: 0.2.189 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.11.0 @@ -14,7 +14,7 @@ dependencies: repository: file://./subcharts/datahub-gms condition: datahub-gms.enabled - name: datahub-frontend - version: 0.2.141 + version: 0.2.142 repository: file://./subcharts/datahub-frontend condition: datahub-frontend.enabled - name: datahub-mae-consumer diff --git a/charts/datahub/subcharts/datahub-frontend/Chart.yaml b/charts/datahub/subcharts/datahub-frontend/Chart.yaml index 1c07a61b5..98bec17fd 100644 --- a/charts/datahub/subcharts/datahub-frontend/Chart.yaml +++ b/charts/datahub/subcharts/datahub-frontend/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.141 +version: 0.2.142 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-frontend/templates/secrets.yaml b/charts/datahub/subcharts/datahub-frontend/templates/secrets.yaml index 9faeb32e0..e3bc54221 100644 --- a/charts/datahub/subcharts/datahub-frontend/templates/secrets.yaml +++ b/charts/datahub/subcharts/datahub-frontend/templates/secrets.yaml @@ -7,5 +7,5 @@ metadata: {{- include "datahub-frontend.labels" . | nindent 4 }} type: Opaque data: - datahub.gms.secret: {{ randAlphaNum 10 | b64enc | quote }} + datahub.gms.secret: {{ randAscii 32 | b64enc | quote }} {{- end -}} \ No newline at end of file From 68400a96049d2cc7c300845ff26dc45ede04d354 Mon Sep 17 00:00:00 2001 From: Glenn Schuurman Date: Wed, 4 Oct 2023 18:28:53 +0200 Subject: [PATCH 12/16] feat(neo4j): update neo4j chart (#365) * feat(neo4j): update neo4j chart Update neo4j to the latest chart version supplied by the neo4j organisation. The new chart offers build in support for enterprise and comunity deployments. This also improves security by exposing the podSecurityContext and containerSecurityContext to the user. BREAKING CHANGE: Removal of neo4j-comunity chart and values Change neo4j parameters are now under neo4j.neo4j Rename neo4jPassword to password Rename existingPasswordSecret to passwordFromSecret Change passwordFromSecret expects: neo4j-password and NEO4J_AUTH keys Require PersistentVolume from values Closes: https://github.com/acryldata/datahub-helm/issues/364 --------- Co-authored-by: Glenn Schuurman Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- .github/workflows/release.yaml | 3 +- README.md | 6 +-- charts/datahub/Chart.yaml | 2 +- .../datahub/quickstart-values-with-neo4j.yaml | 4 +- charts/datahub/values.yaml | 4 +- charts/prerequisites/Chart.yaml | 13 ++--- charts/prerequisites/values.yaml | 48 ++++++++++++------- 7 files changed, 45 insertions(+), 35 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7343b932c..2c96e44c7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -31,8 +31,7 @@ jobs: - name: Add Helm repos run: | helm repo add elasticsearch https://helm.elastic.co - helm repo add neo4j https://neo4j-contrib.github.io/neo4j-helm - helm repo add neo4j-community https://equinor.github.io/helm-charts/charts + helm repo add neo4j https://helm.neo4j.com/neo4j helm repo add mysql https://charts.bitnami.com/bitnami helm repo add cp-helm-charts https://confluentinc.github.io/cp-helm-charts helm repo add kafka https://charts.bitnami.com/bitnami diff --git a/README.md b/README.md index 49027c0cb..722640da4 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Assuming kubectl context points to the correct kubernetes cluster, first create ```(shell) kubectl create secret generic mysql-secrets --from-literal=mysql-root-password=datahub -kubectl create secret generic neo4j-secrets --from-literal=neo4j-password=datahub +kubectl create secret generic neo4j-secrets --from-literal=neo4j-password=datahub --from-literal=NEO4J_AUTH=neo4j/datahub ``` The above commands sets the passwords to "datahub" as an example. Change to any password of choice. @@ -79,7 +79,7 @@ elasticsearch-master-0 1/1 Running 0 prerequisites-cp-schema-registry-cf79bfccf-kvjtv 2/2 Running 1 63m prerequisites-kafka-0 1/1 Running 2 62m prerequisites-mysql-0 1/1 Running 1 62m -prerequisites-neo4j-community-0 1/1 Running 0 52m +prerequisites-neo4j-0 1/1 Running 0 52m prerequisites-zookeeper-0 1/1 Running 0 62m ``` @@ -109,7 +109,7 @@ elasticsearch-master-0 1/1 Running 0 prerequisites-cp-schema-registry-cf79bfccf-kvjtv 2/2 Running 1 99m prerequisites-kafka-0 1/1 Running 2 97m prerequisites-mysql-0 1/1 Running 1 97m -prerequisites-neo4j-community-0 1/1 Running 0 88m +prerequisites-neo4j-0 1/1 Running 0 88m prerequisites-zookeeper-0 1/1 Running 0 97m ``` diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 5954c3240..03577c468 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.189 +version: 0.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.11.0 diff --git a/charts/datahub/quickstart-values-with-neo4j.yaml b/charts/datahub/quickstart-values-with-neo4j.yaml index 2d96a889c..e37768374 100644 --- a/charts/datahub/quickstart-values-with-neo4j.yaml +++ b/charts/datahub/quickstart-values-with-neo4j.yaml @@ -87,8 +87,8 @@ global: url: "http://prerequisites-cp-schema-registry:8081" neo4j: - host: "prerequisites-neo4j-community:7474" - uri: "bolt://prerequisites-neo4j-community" + host: "prerequisites-neo4j:7474" + uri: "bolt://prerequisites-neo4j" username: "neo4j" password: secretRef: neo4j-secrets diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index d0394cb68..e9f285a6a 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -506,8 +506,8 @@ global: # registry: datahub neo4j: - host: "prerequisites-neo4j-community:7474" - uri: "bolt://prerequisites-neo4j-community" + host: "prerequisites-neo4j:7474" + uri: "bolt://prerequisites-neo4j" username: "neo4j" password: secretRef: neo4j-secrets diff --git a/charts/prerequisites/Chart.yaml b/charts/prerequisites/Chart.yaml index 86b95a471..c9378a561 100644 --- a/charts/prerequisites/Chart.yaml +++ b/charts/prerequisites/Chart.yaml @@ -4,22 +4,17 @@ description: A Helm chart for packages that Datahub depends on type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.0.18 +version: 0.1.1 dependencies: - name: elasticsearch version: 7.17.3 repository: https://helm.elastic.co condition: elasticsearch.enabled - # This chart deploys an enterprise version of neo4j that requires commercial license + # This chart deploys an enterprise or community version of neo4j - name: neo4j - version: 4.2.2-1 - repository: https://neo4j-contrib.github.io/neo4j-helm/ + version: 5.11.0 + repository: https://helm.neo4j.com/neo4j condition: neo4j.enabled - # This chart deploys a community version of neo4j - - name: neo4j-community - version: 1.2.5 - repository: https://equinor.github.io/helm-charts/charts/ - condition: neo4j-community.enabled - name: mysql version: 9.1.8 repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami diff --git a/charts/prerequisites/values.yaml b/charts/prerequisites/values.yaml index 620e1e897..beeee6a4b 100644 --- a/charts/prerequisites/values.yaml +++ b/charts/prerequisites/values.yaml @@ -34,24 +34,40 @@ elasticsearch: # requests: # storage: 100M -# Official neo4j chart uses the Neo4j Enterprise Edition which requires a license +# Official neo4j chart, supports both community and enterprise editions +# see https://neo4j.com/docs/operations-manual/current/kubernetes/ for more information +# source: https://github.com/neo4j/helm-charts neo4j: - enabled: false # set this to true, if you have a license for the enterprise edition - acceptLicenseAgreement: "yes" - defaultDatabase: "graph.db" - neo4jPassword: "datahub" - # For better security, add password to neo4j-secrets k8s secret and uncomment below - # existingPasswordSecret: neo4j-secrets - core: - standalone: true + enabled: true + nameOverride: neo4j + neo4j: + name: neo4j + edition: "community" + acceptLicenseAgreement: "yes" + defaultDatabase: "graph.db" + password: "datahub" + # For better security, add password to neo4j-secrets k8s secret with neo4j-username neo4j-passwordn and NEO4J_AUTH and uncomment below + # NEO4J_AUTH: should be composed like so: {Username}/{Password} + # passwordFromSecret: neo4j-secrets + + # Set security context for pod + securityContext: + runAsNonRoot: true + runAsUser: 7474 + runAsGroup: 7474 + fsGroup: 7474 + fsGroupChangePolicy: "Always" + + # Disallow privilegeEscalation on container level + containerSecurityContext: + allowPrivilegeEscalation: false -# Deploys neo4j community version. Only supports single node -neo4j-community: - enabled: false # set this to true, if you want to run neo4j community edition - acceptLicenseAgreement: "yes" - defaultDatabase: "graph.db" - # For better security, add neo4j-secrets k8s secret with neo4j-password and uncomment below - existingPasswordSecret: neo4j-secrets + # Create a volume for neo4j, SSD storage is recommended + volumes: {} + # data: + # mode: "dynamic" + # dynamic: + # storageClassName: managed-csi-premium mysql: enabled: true From ae3db71b53e08731e54bf21a954dd969ceece677 Mon Sep 17 00:00:00 2001 From: Upendra Rao Vedullapalli Date: Wed, 4 Oct 2023 19:30:16 +0200 Subject: [PATCH 13/16] =?UTF-8?q?feat(datahub-ingestion-cron):=20added=20s?= =?UTF-8?q?upport=20for=20configuring=20securityC=E2=80=A6=20(#373)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(datahub-ingestion-cron): added support for configuring securityContext for all ingestion crons * feat(datahub-ingestion-cron): Merge conflicts from master --------- Co-authored-by: RyanHolstien --- charts/datahub/Chart.yaml | 4 +- .../datahub-ingestion-cron/Chart.yaml | 2 +- .../datahub-ingestion-cron/README.md | 66 ++++++++++--------- .../templates/cron.yaml | 6 ++ .../datahub-ingestion-cron/values.yaml | 12 ++++ 5 files changed, 55 insertions(+), 35 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 03577c468..4bbcc538e 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.3.0 +version: 0.3.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.11.0 @@ -26,7 +26,7 @@ dependencies: repository: file://./subcharts/datahub-mce-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-ingestion-cron - version: 0.2.136 + version: 0.2.137 repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml index 7f70d6aad..6e4ebc44d 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.136 +version: 0.2.137 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/README.md b/charts/datahub/subcharts/datahub-ingestion-cron/README.md index 900b2b15a..c545de242 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/README.md +++ b/charts/datahub/subcharts/datahub-ingestion-cron/README.md @@ -4,35 +4,37 @@ A Helm chart for datahub's metadata-ingestion framework with kerberos authentica ## Chart Values -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"linkedin/datahub-ingestion"` | DataHub Ingestion image repository | -| image.tag | string | `"head"` | DataHub Ingestion image tag | -| imagePullSecrets | array | `[]` (does not add image pull secrets to deployed pods) | Docker registry secret names as an array | -| labels | string | `{}` | Metadata labels to be added to each crawling cron job | -| crons | type | `{}` | A map of crawling parameters per different technology being crawler, the key in the object will be used as the name for the new cron job | -| crons.schedule | string | `"0 0 * * *"` | Cron expression (default is daily at midnight) for crawler jobs | -| crons.recipe | object | `{}` | Recipe configuration to be executed (required) | -| crons.recipe.configmapName | string | `""` | Name of configmap to be mounted containing recipe to be executed | -| crons.recipe.fileName | string | `""` | Name of property within configMap referenced by `recipe.configName` with the concrete recipe definition | -| crons.command | array | `["/bin/sh", "-c", "datahub ingest -c /etc/recipe/"]` | Array of strings denoting the crawling command to be invoked in the cron job. By default it will execute the recipe defined in the `crons.recipe` object. Cron crawling customization is possible by having extra volumes with custom logic to be executed. | -| crons.hostAliases | array | `[]` | host aliases | -| crons.env | object | `{}` | Environment variables to add to the cronjob container | -| crons.envFromSecrets | object | `{}` | Environment variables from secrets to the cronjob container | -| crons.envFromSecrets*.secret | string | | secretKeyRef.name used for environment variable | -| crons.envFromSecrets*.key | string | | secretKeyRef.key used for environment variable | -| crons.extraVolumes | array | `[]` | Additional volumes to add to the pods | -| crons.extraVolumeMounts | array | `[]` | Additional volume mounts to add to the pods | -| crons.extraInitContainers | object | `{}` | Init containers to add to the cronjob container | -| crons.serviceAccountName | string | | Service account name used for the cronjob container | -| crons.podAnnotations | object | `{}` | Annotations to add to the pods | -| crons.restartPolicy | string | `"Always"` | Pod restart policy | -| crons.concurrencyPolicy | string | `"Allow"` | Specifies how to treat concurrent executions of a job | -| crons.failedJobsHistoryLimit | integer | `1` | Number of failed finished jobs to retain | -| crons.successfulJobsHistoryLimit | integer | `3` | Number of successful finished jobs to retain | -| crons.backoffLimit | integer | `6` | Number of retries before marking job failed | -| crons.nodeSelector | object | `{}` | Node labels for pod assignment | -| crons.affinity | object | `{}` | Affinity for pod assignment | -| crons.tolerations | list | `[]` | Tolerations for pod assignment | -| crons.extraSidecars | list | `[]` | Add sidecar containers to the pod | +| Key | Type | Default | Description | +|----------------------------------|---------|------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.repository | string | `"linkedin/datahub-ingestion"` | DataHub Ingestion image repository | +| image.tag | string | `"head"` | DataHub Ingestion image tag | +| imagePullSecrets | array | `[]` (does not add image pull secrets to deployed pods) | Docker registry secret names as an array | +| labels | string | `{}` | Metadata labels to be added to each crawling cron job | +| podSecurityContext | object | `{}` | Pod security context for cron jobs | +| crons | type | `{}` | A map of crawling parameters per different technology being crawler, the key in the object will be used as the name for the new cron job | +| crons.schedule | string | `"0 0 * * *"` | Cron expression (default is daily at midnight) for crawler jobs | +| crons.recipe | object | `{}` | Recipe configuration to be executed (required) | +| crons.recipe.configmapName | string | `""` | Name of configmap to be mounted containing recipe to be executed | +| crons.recipe.fileName | string | `""` | Name of property within configMap referenced by `recipe.configName` with the concrete recipe definition | +| crons.command | array | `["/bin/sh", "-c", "datahub ingest -c /etc/recipe/"]` | Array of strings denoting the crawling command to be invoked in the cron job. By default it will execute the recipe defined in the `crons.recipe` object. Cron crawling customization is possible by having extra volumes with custom logic to be executed. | +| crons.hostAliases | array | `[]` | host aliases | +| crons.env | object | `{}` | Environment variables to add to the cronjob container | +| crons.envFromSecrets | object | `{}` | Environment variables from secrets to the cronjob container | +| crons.envFromSecrets*.secret | string | | secretKeyRef.name used for environment variable | +| crons.envFromSecrets*.key | string | | secretKeyRef.key used for environment variable | +| crons.extraVolumes | array | `[]` | Additional volumes to add to the pods | +| crons.extraVolumeMounts | array | `[]` | Additional volume mounts to add to the pods | +| crons.extraInitContainers | object | `{}` | Init containers to add to the cronjob container | +| crons.serviceAccountName | string | | Service account name used for the cronjob container | +| crons.securityContext | object | `{}` | SecurityContext specific to each crawling cron job | +| crons.podAnnotations | object | `{}` | Annotations to add to the pods | +| crons.restartPolicy | string | `"Always"` | Pod restart policy | +| crons.concurrencyPolicy | string | `"Allow"` | Specifies how to treat concurrent executions of a job | +| crons.failedJobsHistoryLimit | integer | `1` | Number of failed finished jobs to retain | +| crons.successfulJobsHistoryLimit | integer | `3` | Number of successful finished jobs to retain | +| crons.backoffLimit | integer | `6` | Number of retries before marking job failed | +| crons.nodeSelector | object | `{}` | Node labels for pod assignment | +| crons.affinity | object | `{}` | Affinity for pod assignment | +| crons.tolerations | list | `[]` | Tolerations for pod assignment | +| crons.extraSidecars | list | `[]` | Add sidecar containers to the pod | diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml index 2f8001458..da33982dc 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml @@ -37,6 +37,8 @@ spec: {{- if .serviceAccountName }} serviceAccountName: {{ .serviceAccountName }} {{- end }} + securityContext: + {{- toYaml $.Values.podSecurityContext | nindent 12 }} containers: - name: {{ $jobName }}-crawler image: "{{ $.Values.image.repository }}:{{ required "Global or specific tag is required" ($.Values.image.tag | default $.Values.global.datahub.version) }}" @@ -48,6 +50,10 @@ spec: {{- toYaml .extraVolumeMounts | nindent 14 }} {{- end }} command: ["/bin/sh", "-c", {{ default $defaultCommand .command }} ] + {{- if .securityContext }} + securityContext: + {{- toYaml .securityContext | nindent 14 }} + {{- end }} env: {{- if .env }} {{- range $key,$value := .env }} diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml index 6f3825a8c..231dbc70b 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml @@ -9,6 +9,9 @@ image: imagePullSecrets: [] +podSecurityContext: {} +# fsGroup: 2000 + crons: {} #### Example data #hive: @@ -62,6 +65,15 @@ crons: {} ## #serviceAccountName: "my-cron-service" + ## If you want to specify container level security + #securityContext: + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + ## Add your own pod annotations. ## #podAnnotations: {} From 83d67144553ff914db082b49964ee2335009b5a9 Mon Sep 17 00:00:00 2001 From: Xuelei Li <115022112+lix-mms@users.noreply.github.com> Date: Tue, 10 Oct 2023 18:54:14 +0200 Subject: [PATCH 14/16] fix(neo4j): enable APOC plugin (#353) * fix(neo4j): enable APOC plugin * config: fix format of adding env var for neo4j * config: update prerequisites chart version to 0.1.2 --- charts/prerequisites/Chart.yaml | 2 +- charts/prerequisites/values.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/prerequisites/Chart.yaml b/charts/prerequisites/Chart.yaml index c9378a561..991d8e493 100644 --- a/charts/prerequisites/Chart.yaml +++ b/charts/prerequisites/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for packages that Datahub depends on type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.1.1 +version: 0.1.2 dependencies: - name: elasticsearch version: 7.17.3 diff --git a/charts/prerequisites/values.yaml b/charts/prerequisites/values.yaml index beeee6a4b..b295ac164 100644 --- a/charts/prerequisites/values.yaml +++ b/charts/prerequisites/values.yaml @@ -69,6 +69,9 @@ neo4j: # dynamic: # storageClassName: managed-csi-premium + env: + NEO4J_PLUGINS: '["apoc"]' + mysql: enabled: true auth: From 826bf3be405a6f2decedee2c6cbb2cfe28ceb735 Mon Sep 17 00:00:00 2001 From: jan-wacker <75388864+jan-wacker@users.noreply.github.com> Date: Thu, 12 Oct 2023 13:12:43 +0200 Subject: [PATCH 15/16] fix(kafka): increase kafka chart version due to kraft enabled error (#378) * fix: increase kafka chart version due to kraft enabled error * increse prerequisites helm chart version --------- Co-authored-by: Wacker --- charts/prerequisites/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/prerequisites/Chart.yaml b/charts/prerequisites/Chart.yaml index 991d8e493..1b8423341 100644 --- a/charts/prerequisites/Chart.yaml +++ b/charts/prerequisites/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for packages that Datahub depends on type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.1.2 +version: 0.1.3 dependencies: - name: elasticsearch version: 7.17.3 @@ -35,7 +35,7 @@ dependencies: condition: cp-helm-charts.enabled # This chart deploys a community version of kafka - name: kafka - version: 22.1.3 + version: 22.1.6 repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami condition: kafka.enabled maintainers: From 4a19805ba95af9fbb349fc837d37b88be3daff6d Mon Sep 17 00:00:00 2001 From: Dheeraj Rampally <84748107+dheerajrampally@users.noreply.github.com> Date: Thu, 12 Oct 2023 08:32:43 -0700 Subject: [PATCH 16/16] feat(datahub-upgrade): adding concurrency policy for the cronjobs (#376) * adding concurrency policy for the cronjobs * parameterizing the concurrencypolicy * fixing the chart by removing IfNotPresent * addressing the code review comments. --- charts/datahub/Chart.yaml | 2 +- charts/datahub/README.md | 196 +++++++++--------- .../datahub-cleanup-job-template.yml | 1 + .../datahub-restore-indices-job-template.yml | 1 + charts/datahub/values.yaml | 4 + 5 files changed, 106 insertions(+), 98 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 4bbcc538e..0208dcf97 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.3.1 +version: 0.3.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.11.0 diff --git a/charts/datahub/README.md b/charts/datahub/README.md index 986a84bd4..5f1ad720d 100644 --- a/charts/datahub/README.md +++ b/charts/datahub/README.md @@ -18,103 +18,105 @@ helm install datahub datahub/datahub --values <> ## Chart Values -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| datahub-frontend.enabled | bool | `true` | Enable Datahub Front-end | -| datahub-frontend.image.repository | string | `"linkedin/datahub-frontend-react"` | Image repository for datahub-frontend | -| datahub-frontend.image.tag | string | `"v0.11.0"` | Image tag for datahub-frontend | -| datahub-frontend.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-frontend | -| datahub-gms.enabled | bool | `true` | Enable GMS | -| datahub-gms.image.repository | string | `"linkedin/datahub-gms"` | Image repository for datahub-gms | -| datahub-gms.image.tag | string | `"v0.11.0"` | Image tag for datahub-gms | -| datahub-gms.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-gms | -| datahub-mae-consumer.image.repository | string | `"linkedin/datahub-mae-consumer"` | Image repository for datahub-mae-consumer | -| datahub-mae-consumer.image.tag | string | `"v0.11.0"` | Image tag for datahub-mae-consumer | -| datahub-mae-consumer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-mae-consumer | -| datahub-mce-consumer.image.repository | string | `"linkedin/datahub-mce-consumer"` | Image repository for datahub-mce-consumer | -| datahub-mce-consumer.image.tag | string | `"v0.11.0"` | Image tag for datahub-mce-consumer | -| datahub-mce-consumer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-mce-consumer | -| datahub-ingestion-cron.enabled | bool | `false` | Enable cronjob for periodic ingestion | -| datahubUpgrade.podSecurityContext | object | `{}` | Pod security context for datahubUpgrade jobs | -| datahubUpgrade.securityContext | object | `{}` | Container security context for datahubUpgrade jobs | -| datahubUpgrade.podAnnotations | object | `{}` | Pod annotations for datahubUpgrade jobs | -| datahubUpgrade.cleanupJob.resources | object | '{}' | Kube Resource definitions for the datahub upgrade job 'cleanupJob' | -| datahubUpgrade.cleanupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | -| datahubUpgrade.restoreIndices.resources | object | '{}' | Kube Resource definitions for the datahub upgrade job 'restore indices' | -| datahubUpgrade.restoreIndices.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | -| elasticsearchSetupJob.enabled | bool | `true` | Enable setup job for elasicsearch | -| elasticsearchSetupJob.image.repository | string | `"linkedin/datahub-elasticsearch-setup"` | Image repository for elasticsearchSetupJob | -| elasticsearchSetupJob.image.tag | string | `"v0.11.0"` | Image repository for elasticsearchSetupJob | -| elasticsearchSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for elasticsearchSetupJob | -| elasticsearchSetupJob.resources | object | '{}' | Kube Resource definitions for elasticsearchSetupJob | -| elasticsearchSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for elasticsearchSetupJob | -| elasticsearchSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for elasticsearchSetupJob | -| elasticsearchSetupJob.podAnnotations | object | `{}` | Pod annotations for elasticsearchSetupJob | -| elasticsearchSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | -| kafkaSetupJob.enabled | bool | `true` | Enable setup job for kafka | -| kafkaSetupJob.image.repository | string | `"linkedin/datahub-kafka-setup"` | Image repository for kafkaSetupJob | -| kafkaSetupJob.image.tag | string | `"v0.11.0"` | Image repository for kafkaSetupJob | -| kafkaSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for kafkaSetupJob| -| kafkaSetupJob.resources | object | '{}' | Kube Resource definitions for kafkaSetupJob | -| kafkaSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for kafkaSetupJob | -| kafkaSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for kafkaSetupJob | -| kafkaSetupJob.podAnnotations | object | `{}` | Pod annotations for kafkaSetupJob | -| kafkaSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | -| mysqlSetupJob.enabled | bool | `false` | Enable setup job for mysql | -| mysqlSetupJob.image.repository | string | `"acryldata/datahub-mysql-setup"` | Image repository for mysqlSetupJob | -| mysqlSetupJob.image.tag | string | `"v0.11.0"` | Image repository for mysqlSetupJob | -| mysqlSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for mysqlSetupJob| -| mysqlSetupJob.resources | object | '{}' | Kube Resource definitions for mysqlSetupJob | -| mysqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | -| mysqlSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for mysqlSetupJob | -| mysqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob | -| mysqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | -| postgresqlSetupJob.enabled | bool | `false` | Enable setup job for postgresql | -| postgresqlSetupJob.image.repository | string | `"acryldata/datahub-postgres-setup"` | Image repository for postgresqlSetupJob | -| postgresqlSetupJob.image.tag | string | `"v0.11.0"` | Image repository for postgresqlSetupJob | -| postgresqlSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for postgresqlSetupJob| -| postgresqlSetupJob.resources | object | '{}' | Kube Resource definitions for postgresqlSetupJob | -| postgresqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | -| postgresqlSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for mysqlSetupJob | -| postgresqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob | -| postgresqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | -| datahubSystemUpdate.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | -| global.strict_mode | boolean | true | Enables validations in helm charts to ensure features work as expected. Recommended NOT TO CHANGE. | -| global.datahub_standalone_consumers_enabled | boolean | true | Enable standalone consumers for kafka | -| global.datahub_analytics_enabled | boolean | true | Enable datahub usage analytics | -| global.datahub.appVersion | string | `"1.0"` | App version for annotation | -| global.datahub.gms.port | string | `"8080"` | Port of GMS service | -| global.elasticsearch.host | string | `"elasticsearch-master"` | Elasticsearch host name (endpoint) | -| global.elasticsearch.port | string | `"9200"` | Elasticsearch port | -| global.kafka.bootstrap.server | string | `"prerequisites-broker:9092"` | Kafka bootstrap servers (with port) | -| global.kafka.zookeeper.server | string | `"prerequisites-zookeeper:2181"` | Kafka zookeeper servers (with port) | -| global.kafka.topics.metadata_change_event_name | string | `"MetadataChangeEvent_v4"` | Kafka topic name for Metadata Change Events (deprecated) | -| global.kafka.topics.failed_metadata_change_event_name | string | `"FailedMetadataChangeEvent_v4"` | Kafka topic name for Failed Metadata Change events (deprecated) | -| global.kafka.topics.metadata_audit_event_name | string | `"MetadataAuditEvent_v4"` | Kafka topic name for Metadata Audit events (deprecated) | -| global.kafka.topics.datahub_usage_event_name | string | `"DataHubUsageEvent_v1"` | Kafka topic name for DataHub Usage events | -| global.kafka.topics.metadata_change_proposal_topic_name | string | `"MetadataChangeProposal_v1"` | Kafka topic name for Metadata Change Proposal events | -| global.kafka.topics.failed_metadata_change_proposal_topic_name | string | `"FailedMetadataChangeProposal_v1"` | Kafka topic name for Failed Metadata Change Proposal events | -| global.kafka.topics.metadata_change_log_versioned_topic_name | string | `"MetadataChangeLog_Versioned_v1"` | Kafka topic name for Versioned Metadata Change Log events | -| global.kafka.topics.metadata_change_log_timeseries_topic_name | string | `"MetadataChangeLog_Timeseries_v1"` | Kafka topic name for Timeseries Metadata Change Log events | -| global.kafka.topics.platform_event_topic_name | string | `"PlatformEvent_v1"` | Kafka topic name for Platform events | -| global.kafka.schemaregistry.url | string | `` | URL to kafka schema registry if using `KAFKA` type | -| global.neo4j.host | string | `"prerequisites-neo4j:7474"` | Neo4j host address (with port) | -| global.neo4j.uri | string | `"bolt://prerequisites-neo4j"` | Neo4j URI | -| global.neo4j.username | string | `"neo4j"` | Neo4j user name | -| global.neo4j.password.secretRef | string | `"neo4j-secrets"` | Secret that contains the Neo4j password | -| global.neo4j.password.secretKey | string | `"neo4j-password"` | Secret key that contains the Neo4j password | -| global.sql.datasource.driver | string | `"com.mysql.cj.jdbc.Driver"` | Driver for the SQL database | -| global.sql.datasource.host | string | `"prerequisites-mysql:3306"` | SQL database host (with port) | -| global.sql.datasource.hostForMysqlClient | string | `"prerequisites-mysql"` | SQL database host (without port) | -| global.sql.datasource.port | string | `"3306"` | SQL database port | -| global.sql.datasource.url | string | `"jdbc:mysql://prerequisites-mysql:3306/datahub?verifyServerCertificate=false\u0026useSSL=true"` | URL to access SQL database | -| global.sql.datasource.username | string | `"root"` | SQL user name | -| global.sql.datasource.username.secretRef | string | `"mysql-secrets"` | Secret that contains the MySQL username | -| global.sql.datasource.username.secretKey | string | `"mysql-username"` | Secret key that contains the MySQL username | -| global.sql.datasource.password.secretRef | string | `"mysql-secrets"` | Secret that contains the MySQL password | -| global.sql.datasource.password.secretKey | string | `"mysql-password"` | Secret key that contains the MySQL password | -| global.sql.datasource.password.value | string | `"mysql-password"` | Alternative to using the secret above, uses raw string value instead | -| global.graph_service_impl | string | `neo4j` | One of `neo4j` or `elasticsearch`. Determines which backend to use for the GMS graph service. Elastic is recommended for a simplified deployment. Neo4j will be the default for now to maintain backwards compatibility | +| Key | Type | Default | Description | +|-----|------|--------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| datahub-frontend.enabled | bool | `true` | Enable Datahub Front-end | +| datahub-frontend.image.repository | string | `"linkedin/datahub-frontend-react"` | Image repository for datahub-frontend | +| datahub-frontend.image.tag | string | `"v0.11.0"` | Image tag for datahub-frontend | +| datahub-frontend.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-frontend | +| datahub-gms.enabled | bool | `true` | Enable GMS | +| datahub-gms.image.repository | string | `"linkedin/datahub-gms"` | Image repository for datahub-gms | +| datahub-gms.image.tag | string | `"v0.11.0"` | Image tag for datahub-gms | +| datahub-gms.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-gms | +| datahub-mae-consumer.image.repository | string | `"linkedin/datahub-mae-consumer"` | Image repository for datahub-mae-consumer | +| datahub-mae-consumer.image.tag | string | `"v0.11.0"` | Image tag for datahub-mae-consumer | +| datahub-mae-consumer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-mae-consumer | +| datahub-mce-consumer.image.repository | string | `"linkedin/datahub-mce-consumer"` | Image repository for datahub-mce-consumer | +| datahub-mce-consumer.image.tag | string | `"v0.11.0"` | Image tag for datahub-mce-consumer | +| datahub-mce-consumer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-mce-consumer | +| datahub-ingestion-cron.enabled | bool | `false` | Enable cronjob for periodic ingestion | +| datahubUpgrade.podSecurityContext | object | `{}` | Pod security context for datahubUpgrade jobs | +| datahubUpgrade.securityContext | object | `{}` | Container security context for datahubUpgrade jobs | +| datahubUpgrade.podAnnotations | object | `{}` | Pod annotations for datahubUpgrade jobs | +| datahubUpgrade.cleanupJob.resources | object | '{}' | Kube Resource definitions for the datahub upgrade job 'cleanupJob' | +| datahubUpgrade.cleanupJob.concurrencyPolicy | string | `Allow, Forbid, Replace` | Add concurrencyPolicy for the clean up cron job | +| datahubUpgrade.cleanupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | +| datahubUpgrade.restoreIndices.resources | object | '{}' | Kube Resource definitions for the datahub upgrade job 'restore indices' | +| datahubUpgrade.restoreIndices.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | +| datahubUpgrade.restoreIndices.concurrencyPolicy | string | `Allow, Forbid, Replace` | Add concurrencyPolicy for the restoreIndicies cron job | +| elasticsearchSetupJob.enabled | bool | `true` | Enable setup job for elasicsearch | +| elasticsearchSetupJob.image.repository | string | `"linkedin/datahub-elasticsearch-setup"` | Image repository for elasticsearchSetupJob | +| elasticsearchSetupJob.image.tag | string | `"v0.11.0"` | Image repository for elasticsearchSetupJob | +| elasticsearchSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for elasticsearchSetupJob | +| elasticsearchSetupJob.resources | object | '{}' | Kube Resource definitions for elasticsearchSetupJob | +| elasticsearchSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for elasticsearchSetupJob | +| elasticsearchSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for elasticsearchSetupJob | +| elasticsearchSetupJob.podAnnotations | object | `{}` | Pod annotations for elasticsearchSetupJob | +| elasticsearchSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | +| kafkaSetupJob.enabled | bool | `true` | Enable setup job for kafka | +| kafkaSetupJob.image.repository | string | `"linkedin/datahub-kafka-setup"` | Image repository for kafkaSetupJob | +| kafkaSetupJob.image.tag | string | `"v0.11.0"` | Image repository for kafkaSetupJob | +| kafkaSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for kafkaSetupJob | +| kafkaSetupJob.resources | object | '{}' | Kube Resource definitions for kafkaSetupJob | +| kafkaSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for kafkaSetupJob | +| kafkaSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for kafkaSetupJob | +| kafkaSetupJob.podAnnotations | object | `{}` | Pod annotations for kafkaSetupJob | +| kafkaSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | +| mysqlSetupJob.enabled | bool | `false` | Enable setup job for mysql | +| mysqlSetupJob.image.repository | string | `"acryldata/datahub-mysql-setup"` | Image repository for mysqlSetupJob | +| mysqlSetupJob.image.tag | string | `"v0.11.0"` | Image repository for mysqlSetupJob | +| mysqlSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for mysqlSetupJob | +| mysqlSetupJob.resources | object | '{}' | Kube Resource definitions for mysqlSetupJob | +| mysqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | +| mysqlSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for mysqlSetupJob | +| mysqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob | +| mysqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | +| postgresqlSetupJob.enabled | bool | `false` | Enable setup job for postgresql | +| postgresqlSetupJob.image.repository | string | `"acryldata/datahub-postgres-setup"` | Image repository for postgresqlSetupJob | +| postgresqlSetupJob.image.tag | string | `"v0.11.0"` | Image repository for postgresqlSetupJob | +| postgresqlSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for postgresqlSetupJob | +| postgresqlSetupJob.resources | object | '{}' | Kube Resource definitions for postgresqlSetupJob | +| postgresqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | +| postgresqlSetupJob.securityContext | object | `{"runAsUser": 1000}` | Container security context for mysqlSetupJob | +| postgresqlSetupJob.podAnnotations | object | `{}` | Pod annotations for mysqlSetupJob | +| postgresqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | +| datahubSystemUpdate.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | +| global.strict_mode | boolean | true | Enables validations in helm charts to ensure features work as expected. Recommended NOT TO CHANGE. | +| global.datahub_standalone_consumers_enabled | boolean | true | Enable standalone consumers for kafka | +| global.datahub_analytics_enabled | boolean | true | Enable datahub usage analytics | +| global.datahub.appVersion | string | `"1.0"` | App version for annotation | +| global.datahub.gms.port | string | `"8080"` | Port of GMS service | +| global.elasticsearch.host | string | `"elasticsearch-master"` | Elasticsearch host name (endpoint) | +| global.elasticsearch.port | string | `"9200"` | Elasticsearch port | +| global.kafka.bootstrap.server | string | `"prerequisites-broker:9092"` | Kafka bootstrap servers (with port) | +| global.kafka.zookeeper.server | string | `"prerequisites-zookeeper:2181"` | Kafka zookeeper servers (with port) | +| global.kafka.topics.metadata_change_event_name | string | `"MetadataChangeEvent_v4"` | Kafka topic name for Metadata Change Events (deprecated) | +| global.kafka.topics.failed_metadata_change_event_name | string | `"FailedMetadataChangeEvent_v4"` | Kafka topic name for Failed Metadata Change events (deprecated) | +| global.kafka.topics.metadata_audit_event_name | string | `"MetadataAuditEvent_v4"` | Kafka topic name for Metadata Audit events (deprecated) | +| global.kafka.topics.datahub_usage_event_name | string | `"DataHubUsageEvent_v1"` | Kafka topic name for DataHub Usage events | +| global.kafka.topics.metadata_change_proposal_topic_name | string | `"MetadataChangeProposal_v1"` | Kafka topic name for Metadata Change Proposal events | +| global.kafka.topics.failed_metadata_change_proposal_topic_name | string | `"FailedMetadataChangeProposal_v1"` | Kafka topic name for Failed Metadata Change Proposal events | +| global.kafka.topics.metadata_change_log_versioned_topic_name | string | `"MetadataChangeLog_Versioned_v1"` | Kafka topic name for Versioned Metadata Change Log events | +| global.kafka.topics.metadata_change_log_timeseries_topic_name | string | `"MetadataChangeLog_Timeseries_v1"` | Kafka topic name for Timeseries Metadata Change Log events | +| global.kafka.topics.platform_event_topic_name | string | `"PlatformEvent_v1"` | Kafka topic name for Platform events | +| global.kafka.schemaregistry.url | string | `` | URL to kafka schema registry if using `KAFKA` type | +| global.neo4j.host | string | `"prerequisites-neo4j:7474"` | Neo4j host address (with port) | +| global.neo4j.uri | string | `"bolt://prerequisites-neo4j"` | Neo4j URI | +| global.neo4j.username | string | `"neo4j"` | Neo4j user name | +| global.neo4j.password.secretRef | string | `"neo4j-secrets"` | Secret that contains the Neo4j password | +| global.neo4j.password.secretKey | string | `"neo4j-password"` | Secret key that contains the Neo4j password | +| global.sql.datasource.driver | string | `"com.mysql.cj.jdbc.Driver"` | Driver for the SQL database | +| global.sql.datasource.host | string | `"prerequisites-mysql:3306"` | SQL database host (with port) | +| global.sql.datasource.hostForMysqlClient | string | `"prerequisites-mysql"` | SQL database host (without port) | +| global.sql.datasource.port | string | `"3306"` | SQL database port | +| global.sql.datasource.url | string | `"jdbc:mysql://prerequisites-mysql:3306/datahub?verifyServerCertificate=false\u0026useSSL=true"` | URL to access SQL database | +| global.sql.datasource.username | string | `"root"` | SQL user name | +| global.sql.datasource.username.secretRef | string | `"mysql-secrets"` | Secret that contains the MySQL username | +| global.sql.datasource.username.secretKey | string | `"mysql-username"` | Secret key that contains the MySQL username | +| global.sql.datasource.password.secretRef | string | `"mysql-secrets"` | Secret that contains the MySQL password | +| global.sql.datasource.password.secretKey | string | `"mysql-password"` | Secret key that contains the MySQL password | +| global.sql.datasource.password.value | string | `"mysql-password"` | Alternative to using the secret above, uses raw string value instead | +| global.graph_service_impl | string | `neo4j` | One of `neo4j` or `elasticsearch`. Determines which backend to use for the GMS graph service. Elastic is recommended for a simplified deployment. Neo4j will be the default for now to maintain backwards compatibility | ## Optional Chart Values diff --git a/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml index 54d51b8f6..982a34838 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml @@ -12,6 +12,7 @@ metadata: spec: schedule: "* * * * *" suspend: true + concurrencyPolicy: {{ .Values.datahubUpgrade.cleanupJob.concurrencyPolicy | default "Allow" }} jobTemplate: spec: template: diff --git a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml index 167b0d63a..e0f962e72 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml @@ -12,6 +12,7 @@ metadata: spec: schedule: "* * * * *" suspend: true + concurrencyPolicy: {{ .Values.datahubUpgrade.restoreIndices.concurrencyPolicy | default "Allow" }} jobTemplate: spec: template: diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index e9f285a6a..d564509af 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -261,6 +261,8 @@ datahubUpgrade: requests: cpu: 300m memory: 256Mi + # Add the concurrency Policy flexibility via values + concurrencyPolicy: Allow # Add extra sidecar containers to job pod extraSidecars: [] # - name: my-image-name @@ -274,6 +276,8 @@ datahubUpgrade: requests: cpu: 300m memory: 256Mi + # Add the concurrency Policy flexibility via values + concurrencyPolicy: Allow # Add extra sidecar containers to job pod extraSidecars: [] # - name: my-image-name