diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ce8753ae27..1e8b7c9644a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -823,27 +823,33 @@ jobs: run: | ct list-changed --target-branch ${{ github.event.repository.default_branch }} - - name: Run chart-testing (lint) - run: ct lint --debug --charts ./helm/nessie - - name: Run 'helm template' validation - run: + run: | cd helm/nessie - for f in values.yaml ci/*.yaml ; do + for f in values.yaml ci/*.yaml; do echo "::group::helm template $f" - helm template --debug --namespace nessie-ns $f . + helm template --debug --namespace nessie-ns --values $f . echo "::endgroup::" done + - name: Run chart-testing (lint) + run: ct lint --debug --charts ./helm/nessie + - name: Show pods run: kubectl get pods -A + - name: Install secrets + run: | + kubectl create namespace nessie-ns + kubectl apply --namespace nessie-ns $(find helm/nessie/ci/secrets -name "*.yaml" -printf '-f %p ') + - name: Run chart-testing (install) run: | echo "Using image: ${DOCKER_IMAGE}" echo " tag: ${DOCKER_VERSION}" - + ct install \ + --namespace nessie-ns \ --helm-extra-set-args "--set=image.repository=${DOCKER_IMAGE} --set=image.tag=${DOCKER_VERSION}" \ --debug --charts ./helm/nessie diff --git a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/AbstractMapBasedSecretsProvider.java b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/AbstractMapBasedSecretsProvider.java index 19996b7c56c..bd5d6dad7a4 100644 --- a/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/AbstractMapBasedSecretsProvider.java +++ b/catalog/secrets/api/src/main/java/org/projectnessie/catalog/secrets/AbstractMapBasedSecretsProvider.java @@ -44,14 +44,13 @@ public Optional getSecret( @Nonnull String name, @Nonnull SecretType secretType, // only used for type-safety at the call site - @SuppressWarnings("unused") @Nonnull Class secretJavaType) { + @Nonnull Class secretJavaType) { Map secretData = resolveSecret(name); if (secretData == null) { return Optional.empty(); } - @SuppressWarnings("unchecked") - S secret = (S) secretType.fromValueMap(secretData); + S secret = secretJavaType.cast(secretType.fromValueMap(secretData)); checkState(secret != null, "Invalid %s secret definition for %s", secretType.name(), name); return Optional.of(secret); } diff --git a/helm/nessie/ci/catalog-secrets-values.yaml b/helm/nessie/ci/catalog-secrets-values.yaml index 0a950eafcdc..566c07c94cd 100644 --- a/helm/nessie/ci/catalog-secrets-values.yaml +++ b/helm/nessie/ci/catalog-secrets-values.yaml @@ -180,7 +180,7 @@ catalog: configOverrides: override1: ov1 # In rare cases it might be legit to turn off the object-stores readiness check. - objectStoresHealthCheckEnabled: true + objectStoresHealthCheckEnabled: false # -- Catalog storage settings. storage: @@ -196,7 +196,7 @@ catalog: region: us-west-2 # -- Endpoint URI, required for private clouds. Optional; if not provided, the default is # used. - endpoint: ~ # "https://bucket1.s3.amazonaws.com" + endpoint: https://localhost/ # -- Endpoint URI, required for private clouds. Optional; if not provided, the default is # used. If the endpoint URIs for the Nessie server and clients differ, this one defines the # endpoint used for the Nessie server. @@ -204,7 +204,7 @@ catalog: # -- Whether to use path-style access. Optional; if not provided, the default is used. If # true, path-style access will be used, as in: https:///. If false, a # virtual-hosted style will be used instead, as in: https://.. - pathStyleAccess: false + pathStyleAccess: true # -- AWS Access point for this bucket. Access points can be used to perform S3 operations by # specifying a mapping of bucket to access points. This is useful for multi-region access, # cross-region access, disaster recovery, etc. See @@ -273,7 +273,7 @@ catalog: # -- AWS credentials. Required when serverAuthenticationMode is STATIC. accessKeySecret: # -- The secret name to pull AWS credentials from. - name: defaultCreds + name: default-creds # -- The secret key storing the AWS secret key id. awsAccessKeyId: defaultAccessKeyId # -- The secret key storing the AWS secret access key. @@ -332,7 +332,7 @@ catalog: # -- The default endpoint override to use. The endpoint is almost always used for testing # purposes. If the endpoint URIs for the Nessie server and clients differ, this one defines # the endpoint used for the Nessie server. - host: ~ + host: http://localhost/ # -- When using a specific endpoint, see host, and the endpoint URIs for the Nessie server # differ, you can specify the URI passed down to clients using this setting. Otherwise, # clients will receive the value from the host setting. @@ -353,18 +353,18 @@ catalog: # SERVICE_ACCOUNT. authCredentialsJsonSecret: # -- The secret name to pull a valid Google Cloud service account key from. - name: gcsJson + name: gcs-json # -- The secret key storing the Google Cloud service account JSON key. key: gcsJsonKey # -- The oauth2 token secret. This is required when authType is ACCESS_TOKEN. oauth2TokenSecret: # # -- The secret name to pull a valid Google Cloud service account key from. - name: gcsOauth2Name + name: gcs-oauth2-name # # -- The secret key storing the token. token: gcsOauth2Token # # -- The secret key storing the token's expiresAt value (optional). - expiresAt: ~ + expiresAt: gcsOauth2TokenExpires # -- Customer-supplied AES256 key for blob encryption when writing. Currently unsupported. encryptionKey: gcsEncKey @@ -384,19 +384,19 @@ catalog: authType: ACCESS_TOKEN oauth2TokenSecret: name: gcs-creds - key: token + token: token expiresAt: expiresAt authCredentialsJsonSecret: - name: gcsJsonBucket + name: gcs-json-bucket key: gcsJsonKeyBucket - name: bucket2 authType: ACCESS_TOKEN oauth2TokenSecret: name: gcs-creds2 - key: token2 + token: token2 expiresAt: expiresAt2 authCredentialsJsonSecret: - name: gcsJsonBucket2 + name: gcs-json-bucket2 key: gcsJsonKeyBucket2 # -- GCS transport settings. Not overridable on a per-bucket basis. @@ -429,7 +429,7 @@ catalog: # Global ADLS settings. Can be overridden on a per-filesystem basis below. defaultOptions: # -- Custom HTTP endpoint. In case clients need to use a different URI, use externalEndpoint. - endpoint: ~ + endpoint: http://localhost/foo/bar # -- Custom HTTP endpoint to be used by clients. If not set, the endpoint value is used. externalEndpoint: ~ # -- The retry strategy to use. Valid values are: NONE, EXPONENTIAL_BACKOFF, FIXED_DELAY. @@ -457,7 +457,7 @@ catalog: # STORAGE_SHARED_KEY. accountSecret: # -- Name of the secret containing the account name and key. - name: adlsName + name: adls-name # -- Secret key containing the fully-qualified account name, e.g. "myaccount.dfs.core.windows.net". accountName: adlsaccountName # -- Secret key containing the account key. @@ -465,7 +465,7 @@ catalog: # -- A secret containing the SAS token to use. Required when authType is SAS_TOKEN. sasTokenSecret: # -- Name of the secret containing the SAS token. - name: adlsSAS + name: adls-sas # -- Secret key containing the SAS token. sasToken: adlsTOKEN @@ -478,7 +478,7 @@ catalog: accountName: accountName accountKey: accountKeyRef sasTokenSecret: - name: adlsSasFs + name: adls-sas-fs sasToken: adlsTokenFs - name: filesystem2 endpoint: http://localhost/adlsgen2/bucket2 @@ -487,7 +487,7 @@ catalog: accountName: accountName2 accountKey: accountKeyRef2 sasTokenSecret: - name: adlsSasFs2 + name: adls-sas-fs2 sasToken: adlsTokenFs2 # -- ADLS transport settings. Not overridable on a per-bucket basis. diff --git a/helm/nessie/ci/secrets/adls-account-secret.yaml b/helm/nessie/ci/secrets/adls-account-secret.yaml new file mode 100644 index 00000000000..3d16ef77c00 --- /dev/null +++ b/helm/nessie/ci/secrets/adls-account-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: adls-account-secret +type: Opaque +stringData: + accountName: "acct" + accountKeyRef: "key" diff --git a/helm/nessie/ci/secrets/adls-account-secret2.yaml b/helm/nessie/ci/secrets/adls-account-secret2.yaml new file mode 100644 index 00000000000..b183d9ad5d2 --- /dev/null +++ b/helm/nessie/ci/secrets/adls-account-secret2.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: adls-account-secret2 +type: Opaque +stringData: + accountName2: "acct" + accountKeyRef2: "key" diff --git a/helm/nessie/ci/secrets/adls-name.yaml b/helm/nessie/ci/secrets/adls-name.yaml new file mode 100644 index 00000000000..238076cca02 --- /dev/null +++ b/helm/nessie/ci/secrets/adls-name.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: adls-name +type: Opaque +stringData: + adlsaccountName: "acct" + adlsaccountKey: "key" diff --git a/helm/nessie/ci/secrets/adls-sas-fs.yaml b/helm/nessie/ci/secrets/adls-sas-fs.yaml new file mode 100644 index 00000000000..0780514cba5 --- /dev/null +++ b/helm/nessie/ci/secrets/adls-sas-fs.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: adls-sas-fs +type: Opaque +stringData: + adlsTokenFs: "tok" diff --git a/helm/nessie/ci/secrets/adls-sas-fs2.yaml b/helm/nessie/ci/secrets/adls-sas-fs2.yaml new file mode 100644 index 00000000000..49afef20300 --- /dev/null +++ b/helm/nessie/ci/secrets/adls-sas-fs2.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: adls-sas-fs2 +type: Opaque +stringData: + adlsTokenFs2: "tok" diff --git a/helm/nessie/ci/secrets/adls-sas.yaml b/helm/nessie/ci/secrets/adls-sas.yaml new file mode 100644 index 00000000000..3a5b4e03f1b --- /dev/null +++ b/helm/nessie/ci/secrets/adls-sas.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: adls-sas +type: Opaque +stringData: + adlsTOKEN: "tok" diff --git a/helm/nessie/ci/secrets/awscreds.yaml b/helm/nessie/ci/secrets/awscreds.yaml new file mode 100644 index 00000000000..a1c71ef108e --- /dev/null +++ b/helm/nessie/ci/secrets/awscreds.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: awscreds +type: Opaque +stringData: + aws_access_key_id: "access_key" + aws_secret_access_key: "secret_access_key" diff --git a/helm/nessie/ci/secrets/awscreds2.yaml b/helm/nessie/ci/secrets/awscreds2.yaml new file mode 100644 index 00000000000..8e60603d17e --- /dev/null +++ b/helm/nessie/ci/secrets/awscreds2.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: awscreds2 +type: Opaque +stringData: + aws_access_key_id2: "access_key2" + aws_secret_access_key2: "secret_access_key2" diff --git a/helm/nessie/ci/secrets/default-creds.yaml b/helm/nessie/ci/secrets/default-creds.yaml new file mode 100644 index 00000000000..14cc642d5ab --- /dev/null +++ b/helm/nessie/ci/secrets/default-creds.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: default-creds +type: Opaque +stringData: + defaultAccessKeyId: "defaultAccessKeyId" + defaultSecretAccessKey: "defaultSecretAccessKey" diff --git a/helm/nessie/ci/secrets/gcs-creds.yaml b/helm/nessie/ci/secrets/gcs-creds.yaml new file mode 100644 index 00000000000..c0d6f8dfa94 --- /dev/null +++ b/helm/nessie/ci/secrets/gcs-creds.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gcs-creds +type: Opaque +stringData: + token: "oath2-token" + expiresAt: "" diff --git a/helm/nessie/ci/secrets/gcs-creds2.yaml b/helm/nessie/ci/secrets/gcs-creds2.yaml new file mode 100644 index 00000000000..d3c2cf390b8 --- /dev/null +++ b/helm/nessie/ci/secrets/gcs-creds2.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gcs-creds2 +type: Opaque +stringData: + token2: "oath2-token" + expiresAt2: "" diff --git a/helm/nessie/ci/secrets/gcs-json-bucket.yaml b/helm/nessie/ci/secrets/gcs-json-bucket.yaml new file mode 100644 index 00000000000..ddcb543e116 --- /dev/null +++ b/helm/nessie/ci/secrets/gcs-json-bucket.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gcs-json-bucket +type: Opaque +stringData: + gcsJsonKeyBucket: "gcs-json-jey-bucket" diff --git a/helm/nessie/ci/secrets/gcs-json-bucket2.yaml b/helm/nessie/ci/secrets/gcs-json-bucket2.yaml new file mode 100644 index 00000000000..530daeeebd6 --- /dev/null +++ b/helm/nessie/ci/secrets/gcs-json-bucket2.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gcs-json-bucket2 +type: Opaque +stringData: + gcsJsonKeyBucket2: "gcs-json-jey-bucket2" diff --git a/helm/nessie/ci/secrets/gcs-json.yaml b/helm/nessie/ci/secrets/gcs-json.yaml new file mode 100644 index 00000000000..8ac00504505 --- /dev/null +++ b/helm/nessie/ci/secrets/gcs-json.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gcs-json +type: Opaque +stringData: + gcsJsonKey: "gcs-json-jey" diff --git a/helm/nessie/ci/secrets/gcs-oauth2-name.yaml b/helm/nessie/ci/secrets/gcs-oauth2-name.yaml new file mode 100644 index 00000000000..a4af1154003 --- /dev/null +++ b/helm/nessie/ci/secrets/gcs-oauth2-name.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gcs-oauth2-name +type: Opaque +stringData: + gcsOauth2Token: "oath2-token" + gcsOauth2TokenExpires: "" diff --git a/helm/nessie/templates/_helpers.tpl b/helm/nessie/templates/_helpers.tpl index b92072ea9db..06622f326e7 100644 --- a/helm/nessie/templates/_helpers.tpl +++ b/helm/nessie/templates/_helpers.tpl @@ -281,24 +281,24 @@ Define environkent variables for catalog storage options. {{- include "nessie.secretToEnv" (list $bucket.accessKeySecret "awsSecretAccessKey" (printf "s3.buckets.bucket%d.access-key" (add $i 1)) "secret" false . ) }} {{- end -}} {{- end -}} -{{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.authCredentialsJsonSecret "key" "gcs.default-options" "auth-credentials-json" true . ) }} +{{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.authCredentialsJsonSecret "key" "gcs.default-options.auth-credentials-json" "key" true . ) }} {{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.oauth2TokenSecret "token" "gcs.default-options.oauth-token" "token" true . ) }} -{{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.oauth2TokenSecret "expiresAt" "gcs.default-options.oauth-token" "expires-at" false . ) }} +{{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.oauth2TokenSecret "expiresAt" "gcs.default-options.oauth-token" "expiresAt" false . ) }} {{- range $i, $bucket := .Values.catalog.storage.gcs.buckets -}} {{- with $global }} -{{- include "nessie.secretToEnv" (list $bucket.authCredentialsJsonSecret "key" (printf "gcs.buckets.bucket%d" (add $i 1)) "auth-credentials-json" true . ) }} -{{- include "nessie.secretToEnv" (list $bucket.oauth2TokenSecret "token" (printf "gcs.buckets.bucket%d.oauth-token" (add $i 1)) "oauth-token" true . ) }} -{{- include "nessie.secretToEnv" (list $bucket.oauth2TokenSecret "expiresAt" (printf "gcs.buckets.bucket%d.oauth-token" (add $i 1)) "expires-at" false . ) }} +{{- include "nessie.secretToEnv" (list $bucket.authCredentialsJsonSecret "key" (printf "gcs.buckets.bucket%d.auth-credentials-json" (add $i 1)) "key" true . ) }} +{{- include "nessie.secretToEnv" (list $bucket.oauth2TokenSecret "token" (printf "gcs.buckets.bucket%d.oauth-token" (add $i 1)) "token" true . ) }} +{{- include "nessie.secretToEnv" (list $bucket.oauth2TokenSecret "expiresAt" (printf "gcs.buckets.bucket%d.oauth-token" (add $i 1)) "expiresAt" false . ) }} {{- end -}} {{- end -}} {{ include "nessie.secretToEnv" (list .Values.catalog.storage.adls.defaultOptions.accountSecret "accountName" "adls.default-options.account" "name" true . ) }} {{- include "nessie.secretToEnv" (list .Values.catalog.storage.adls.defaultOptions.accountSecret "accountKey" "adls.default-options.account" "secret" false . ) }} -{{- include "nessie.secretToEnv" (list .Values.catalog.storage.adls.defaultOptions.sasTokenSecret "sasToken" "adls.default-options" "sas-token" true . ) }} +{{- include "nessie.secretToEnv" (list .Values.catalog.storage.adls.defaultOptions.sasTokenSecret "sasToken" "adls.default-options.sas-token" "token" true . ) }} {{- range $i, $filesystem := .Values.catalog.storage.adls.filesystems -}} {{- with $global }} {{- include "nessie.secretToEnv" (list $filesystem.accountSecret "accountName" (printf "adls.file-systems.filesystem%d.account" (add $i 1)) "name" true . ) }} {{- include "nessie.secretToEnv" (list $filesystem.accountSecret "accountKey" (printf "adls.file-systems.filesystem%d.account" (add $i 1)) "secret" false . ) }} -{{- include "nessie.secretToEnv" (list $filesystem.sasTokenSecret "sasToken" (printf "adls.file-systems.filesystem%d.sas-token" (add $i 1)) "sas-token" true . ) }} +{{- include "nessie.secretToEnv" (list $filesystem.sasTokenSecret "sasToken" (printf "adls.file-systems.filesystem%d.sas-token" (add $i 1)) "token" true . ) }} {{- end -}} {{- end -}} {{- end -}} @@ -330,13 +330,13 @@ config types know about that symbolic name and resolve it via a SecretsProvider, # - name: {{ (printf "nessie.catalog.service.%s" $midfix) | quote }} value: {{ (printf "nessie-catalog-secrets.%s" $midfix) | quote }} -{{ end -}} +{{- end }} - name: {{ (printf "nessie-catalog-secrets.%s.%s" $midfix $suffix) | quote }} valueFrom: secretKeyRef: name: {{ (tpl $secretName . ) | quote }} key: {{ (tpl $secretKey . ) | quote }} -{{- end }} +{{ end -}} {{- end -}} {{- end -}} {{- end -}}