review + fix

projectnessie · Sep 4, 2024 · 32970ee · 32970ee
1 parent f9adbad
commit 32970ee
Show file tree

Hide file tree

Showing 19 changed files with 155 additions and 36 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -823,27 +823,33 @@ jobs:
         run: |
           ct list-changed --target-branch ${{ github.event.repository.default_branch }}
 
-      - name: Run chart-testing (lint)
-        run: ct lint --debug --charts ./helm/nessie
-
       - name: Run 'helm template' validation
-        run:
+        run: |
           cd helm/nessie
-          for f in values.yaml ci/*.yaml ; do
+          for f in values.yaml ci/*.yaml; do
             echo "::group::helm template $f"
-            helm template --debug --namespace nessie-ns $f .
+            helm template --debug --namespace nessie-ns --values $f .
             echo "::endgroup::"
           done
 
+      - name: Run chart-testing (lint)
+        run: ct lint --debug --charts ./helm/nessie
+
       - name: Show pods
         run: kubectl get pods -A
 
+      - name: Install secrets
+        run: |
+          kubectl create namespace nessie-ns
+          kubectl apply --namespace nessie-ns $(find helm/nessie/ci/secrets -name "*.yaml" -printf '-f %p ')
+
       - name: Run chart-testing (install)
         run: |
           echo "Using image: ${DOCKER_IMAGE}"
           echo "        tag: ${DOCKER_VERSION}"
-      
+
           ct install \
+            --namespace nessie-ns \
             --helm-extra-set-args "--set=image.repository=${DOCKER_IMAGE} --set=image.tag=${DOCKER_VERSION}" \
             --debug --charts ./helm/nessie
 

diff --git a/.../api/src/main/java/org/projectnessie/catalog/secrets/AbstractMapBasedSecretsProvider.java b/.../api/src/main/java/org/projectnessie/catalog/secrets/AbstractMapBasedSecretsProvider.java
@@ -44,14 +44,13 @@ public <S extends Secret> Optional<S> getSecret(
       @Nonnull String name,
       @Nonnull SecretType secretType,
       // only used for type-safety at the call site
-      @SuppressWarnings("unused") @Nonnull Class<S> secretJavaType) {
+      @Nonnull Class<S> secretJavaType) {
     Map<String, String> secretData = resolveSecret(name);
     if (secretData == null) {
       return Optional.empty();
     }
 
-    @SuppressWarnings("unchecked")
-    S secret = (S) secretType.fromValueMap(secretData);
+    S secret = secretJavaType.cast(secretType.fromValueMap(secretData));
     checkState(secret != null, "Invalid %s secret definition for %s", secretType.name(), name);
     return Optional.of(secret);
   }

diff --git a/helm/nessie/ci/catalog-secrets-values.yaml b/helm/nessie/ci/catalog-secrets-values.yaml
@@ -180,7 +180,7 @@ catalog:
       configOverrides:
         override1: ov1
     # In rare cases it might be legit to turn off the object-stores readiness check.
-    objectStoresHealthCheckEnabled: true
+    objectStoresHealthCheckEnabled: false
 
   # -- Catalog storage settings.
   storage:
@@ -196,15 +196,15 @@ catalog:
         region: us-west-2
         # -- Endpoint URI, required for private clouds. Optional; if not provided, the default is
         # used.
-        endpoint: ~  # "https://bucket1.s3.amazonaws.com"
+        endpoint: https://localhost/
         # -- Endpoint URI, required for private clouds. Optional; if not provided, the default is
         # used. If the endpoint URIs for the Nessie server and clients differ, this one defines the
         # endpoint used for the Nessie server.
         externalEndpoint: ~
         # -- Whether to use path-style access. Optional; if not provided, the default is used. If
         # true, path-style access will be used, as in: https://<domain>/<bucket>. If false, a
         # virtual-hosted style will be used instead, as in: https://<bucket>.<domain>.
-        pathStyleAccess: false
+        pathStyleAccess: true
         # -- AWS Access point for this bucket. Access points can be used to perform S3 operations by
         # specifying a mapping of bucket to access points. This is useful for multi-region access,
         # cross-region access, disaster recovery, etc. See
@@ -273,7 +273,7 @@ catalog:
         # -- AWS credentials. Required when serverAuthenticationMode is STATIC.
         accessKeySecret:
           # -- The secret name to pull AWS credentials from.
-          name: defaultCreds
+          name: default-creds
           # -- The secret key storing the AWS secret key id.
           awsAccessKeyId: defaultAccessKeyId
           # -- The secret key storing the AWS secret access key.
@@ -332,7 +332,7 @@ catalog:
         # -- The default endpoint override to use. The endpoint is almost always used for testing
         # purposes. If the endpoint URIs for the Nessie server and clients differ, this one defines
         # the endpoint used for the Nessie server.
-        host: ~
+        host: http://localhost/
         # -- When using a specific endpoint, see host, and the endpoint URIs for the Nessie server
         # differ, you can specify the URI passed down to clients using this setting. Otherwise,
         # clients will receive the value from the host setting.
@@ -353,18 +353,18 @@ catalog:
         # SERVICE_ACCOUNT.
         authCredentialsJsonSecret:
           # -- The secret name to pull a valid Google Cloud service account key from.
-          name: gcsJson
+          name: gcs-json
           # -- The secret key storing the Google Cloud service account JSON key.
           key: gcsJsonKey
 
         # -- The oauth2 token secret. This is required when authType is ACCESS_TOKEN.
         oauth2TokenSecret:
           # # -- The secret name to pull a valid Google Cloud service account key from.
-          name: gcsOauth2Name
+          name: gcs-oauth2-name
           # # -- The secret key storing the token.
           token: gcsOauth2Token
           # # -- The secret key storing the token's expiresAt value (optional).
-          expiresAt: ~
+          expiresAt: gcsOauth2TokenExpires
 
         # -- Customer-supplied AES256 key for blob encryption when writing. Currently unsupported.
         encryptionKey: gcsEncKey
@@ -384,19 +384,19 @@ catalog:
         authType: ACCESS_TOKEN
         oauth2TokenSecret:
           name: gcs-creds
-          key: token
+          token: token
           expiresAt: expiresAt
         authCredentialsJsonSecret:
-          name: gcsJsonBucket
+          name: gcs-json-bucket
           key: gcsJsonKeyBucket
       - name: bucket2
         authType: ACCESS_TOKEN
         oauth2TokenSecret:
           name: gcs-creds2
-          key: token2
+          token: token2
           expiresAt: expiresAt2
         authCredentialsJsonSecret:
-          name: gcsJsonBucket2
+          name: gcs-json-bucket2
           key: gcsJsonKeyBucket2
 
       # -- GCS transport settings. Not overridable on a per-bucket basis.
@@ -429,7 +429,7 @@ catalog:
       # Global ADLS settings. Can be overridden on a per-filesystem basis below.
       defaultOptions:
         # -- Custom HTTP endpoint. In case clients need to use a different URI, use externalEndpoint.
-        endpoint: ~
+        endpoint: http://localhost/foo/bar
         # -- Custom HTTP endpoint to be used by clients. If not set, the endpoint value is used.
         externalEndpoint: ~
         # -- The retry strategy to use. Valid values are: NONE, EXPONENTIAL_BACKOFF, FIXED_DELAY.
@@ -457,15 +457,15 @@ catalog:
         # STORAGE_SHARED_KEY.
         accountSecret:
           # -- Name of the secret containing the account name and key.
-          name: adlsName
+          name: adls-name
           # -- Secret key containing the fully-qualified account name, e.g. "myaccount.dfs.core.windows.net".
           accountName: adlsaccountName
           # -- Secret key containing the account key.
           accountKey: adlsaccountKey
         # -- A secret containing the SAS token to use. Required when authType is SAS_TOKEN.
         sasTokenSecret:
           # -- Name of the secret containing the SAS token.
-          name: adlsSAS
+          name: adls-sas
           # -- Secret key containing the SAS token.
           sasToken: adlsTOKEN
 
@@ -478,7 +478,7 @@ catalog:
           accountName: accountName
           accountKey: accountKeyRef
         sasTokenSecret:
-          name: adlsSasFs
+          name: adls-sas-fs
           sasToken: adlsTokenFs
       - name: filesystem2
         endpoint: http://localhost/adlsgen2/bucket2
@@ -487,7 +487,7 @@ catalog:
           accountName: accountName2
           accountKey: accountKeyRef2
         sasTokenSecret:
-          name: adlsSasFs2
+          name: adls-sas-fs2
           sasToken: adlsTokenFs2
 
       # -- ADLS transport settings. Not overridable on a per-bucket basis.

diff --git a/helm/nessie/ci/secrets/adls-account-secret.yaml b/helm/nessie/ci/secrets/adls-account-secret.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: adls-account-secret
+type: Opaque
+stringData:
+  accountName: "acct"
+  accountKeyRef: "key"
diff --git a/helm/nessie/ci/secrets/adls-account-secret2.yaml b/helm/nessie/ci/secrets/adls-account-secret2.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: adls-account-secret2
+type: Opaque
+stringData:
+  accountName2: "acct"
+  accountKeyRef2: "key"
diff --git a/helm/nessie/ci/secrets/adls-name.yaml b/helm/nessie/ci/secrets/adls-name.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: adls-name
+type: Opaque
+stringData:
+  adlsaccountName: "acct"
+  adlsaccountKey: "key"
diff --git a/helm/nessie/ci/secrets/adls-sas-fs.yaml b/helm/nessie/ci/secrets/adls-sas-fs.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: adls-sas-fs
+type: Opaque
+stringData:
+  adlsTokenFs: "tok"
diff --git a/helm/nessie/ci/secrets/adls-sas-fs2.yaml b/helm/nessie/ci/secrets/adls-sas-fs2.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: adls-sas-fs2
+type: Opaque
+stringData:
+  adlsTokenFs2: "tok"
diff --git a/helm/nessie/ci/secrets/adls-sas.yaml b/helm/nessie/ci/secrets/adls-sas.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: adls-sas
+type: Opaque
+stringData:
+  adlsTOKEN: "tok"
diff --git a/helm/nessie/ci/secrets/awscreds.yaml b/helm/nessie/ci/secrets/awscreds.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: awscreds
+type: Opaque
+stringData:
+  aws_access_key_id: "access_key"
+  aws_secret_access_key: "secret_access_key"
diff --git a/helm/nessie/ci/secrets/awscreds2.yaml b/helm/nessie/ci/secrets/awscreds2.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: awscreds2
+type: Opaque
+stringData:
+  aws_access_key_id2: "access_key2"
+  aws_secret_access_key2: "secret_access_key2"
diff --git a/helm/nessie/ci/secrets/default-creds.yaml b/helm/nessie/ci/secrets/default-creds.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: default-creds
+type: Opaque
+stringData:
+  defaultAccessKeyId: "defaultAccessKeyId"
+  defaultSecretAccessKey: "defaultSecretAccessKey"
diff --git a/helm/nessie/ci/secrets/gcs-creds.yaml b/helm/nessie/ci/secrets/gcs-creds.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcs-creds
+type: Opaque
+stringData:
+  token: "oath2-token"
+  expiresAt: ""
diff --git a/helm/nessie/ci/secrets/gcs-creds2.yaml b/helm/nessie/ci/secrets/gcs-creds2.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcs-creds2
+type: Opaque
+stringData:
+  token2: "oath2-token"
+  expiresAt2: ""
diff --git a/helm/nessie/ci/secrets/gcs-json-bucket.yaml b/helm/nessie/ci/secrets/gcs-json-bucket.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcs-json-bucket
+type: Opaque
+stringData:
+  gcsJsonKeyBucket: "gcs-json-jey-bucket"
diff --git a/helm/nessie/ci/secrets/gcs-json-bucket2.yaml b/helm/nessie/ci/secrets/gcs-json-bucket2.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcs-json-bucket2
+type: Opaque
+stringData:
+  gcsJsonKeyBucket2: "gcs-json-jey-bucket2"
diff --git a/helm/nessie/ci/secrets/gcs-json.yaml b/helm/nessie/ci/secrets/gcs-json.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcs-json
+type: Opaque
+stringData:
+  gcsJsonKey: "gcs-json-jey"
diff --git a/helm/nessie/ci/secrets/gcs-oauth2-name.yaml b/helm/nessie/ci/secrets/gcs-oauth2-name.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcs-oauth2-name
+type: Opaque
+stringData:
+  gcsOauth2Token: "oath2-token"
+  gcsOauth2TokenExpires: ""
diff --git a/helm/nessie/templates/_helpers.tpl b/helm/nessie/templates/_helpers.tpl
@@ -281,24 +281,24 @@ Define environkent variables for catalog storage options.
 {{- include "nessie.secretToEnv" (list $bucket.accessKeySecret "awsSecretAccessKey" (printf "s3.buckets.bucket%d.access-key" (add $i 1)) "secret" false . ) }}
 {{- end -}}
 {{- end -}}
-{{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.authCredentialsJsonSecret "key" "gcs.default-options" "auth-credentials-json" true . ) }}
+{{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.authCredentialsJsonSecret "key" "gcs.default-options.auth-credentials-json" "key" true . ) }}
 {{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.oauth2TokenSecret "token" "gcs.default-options.oauth-token" "token" true . ) }}
-{{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.oauth2TokenSecret "expiresAt" "gcs.default-options.oauth-token" "expires-at" false . ) }}
+{{- include "nessie.secretToEnv" (list .Values.catalog.storage.gcs.defaultOptions.oauth2TokenSecret "expiresAt" "gcs.default-options.oauth-token" "expiresAt" false . ) }}
 {{- range $i, $bucket := .Values.catalog.storage.gcs.buckets -}}
 {{- with $global }}
-{{- include "nessie.secretToEnv" (list $bucket.authCredentialsJsonSecret "key" (printf "gcs.buckets.bucket%d" (add $i 1)) "auth-credentials-json" true . ) }}
-{{- include "nessie.secretToEnv" (list $bucket.oauth2TokenSecret "token" (printf "gcs.buckets.bucket%d.oauth-token" (add $i 1)) "oauth-token" true . ) }}
-{{- include "nessie.secretToEnv" (list $bucket.oauth2TokenSecret "expiresAt" (printf "gcs.buckets.bucket%d.oauth-token" (add $i 1)) "expires-at" false . ) }}
+{{- include "nessie.secretToEnv" (list $bucket.authCredentialsJsonSecret "key" (printf "gcs.buckets.bucket%d.auth-credentials-json" (add $i 1)) "key" true . ) }}
+{{- include "nessie.secretToEnv" (list $bucket.oauth2TokenSecret "token" (printf "gcs.buckets.bucket%d.oauth-token" (add $i 1)) "token" true . ) }}
+{{- include "nessie.secretToEnv" (list $bucket.oauth2TokenSecret "expiresAt" (printf "gcs.buckets.bucket%d.oauth-token" (add $i 1)) "expiresAt" false . ) }}
 {{- end -}}
 {{- end -}}
 {{ include "nessie.secretToEnv" (list .Values.catalog.storage.adls.defaultOptions.accountSecret "accountName" "adls.default-options.account" "name" true . ) }}
 {{- include "nessie.secretToEnv" (list .Values.catalog.storage.adls.defaultOptions.accountSecret "accountKey" "adls.default-options.account" "secret" false . ) }}
-{{- include "nessie.secretToEnv" (list .Values.catalog.storage.adls.defaultOptions.sasTokenSecret "sasToken" "adls.default-options" "sas-token" true . ) }}
+{{- include "nessie.secretToEnv" (list .Values.catalog.storage.adls.defaultOptions.sasTokenSecret "sasToken" "adls.default-options.sas-token" "token" true . ) }}
 {{- range $i, $filesystem := .Values.catalog.storage.adls.filesystems -}}
 {{- with $global }}
 {{- include "nessie.secretToEnv" (list $filesystem.accountSecret "accountName" (printf "adls.file-systems.filesystem%d.account" (add $i 1)) "name" true . ) }}
 {{- include "nessie.secretToEnv" (list $filesystem.accountSecret "accountKey" (printf "adls.file-systems.filesystem%d.account" (add $i 1)) "secret" false . ) }}
-{{- include "nessie.secretToEnv" (list $filesystem.sasTokenSecret "sasToken" (printf "adls.file-systems.filesystem%d.sas-token" (add $i 1)) "sas-token" true . ) }}
+{{- include "nessie.secretToEnv" (list $filesystem.sasTokenSecret "sasToken" (printf "adls.file-systems.filesystem%d.sas-token" (add $i 1)) "token" true . ) }}
 {{- end -}}
 {{- end -}}
 {{- end -}}
@@ -330,13 +330,13 @@ config types know about that symbolic name and resolve it via a SecretsProvider,
 #
 - name: {{ (printf "nessie.catalog.service.%s" $midfix) | quote }}
   value: {{ (printf "nessie-catalog-secrets.%s" $midfix) | quote }}
-{{ end -}}
+{{- end }}
 - name: {{ (printf "nessie-catalog-secrets.%s.%s" $midfix $suffix) | quote }}
   valueFrom:
     secretKeyRef:
       name: {{ (tpl $secretName . ) | quote }}
       key: {{ (tpl $secretKey . ) | quote }}
-{{- end }}
+{{ end -}}
 {{- end -}}
 {{- end -}}
 {{- end -}}