Helm chart: redesign logging options (#9775)

.github/workflows/ci.yml

@@ -838,6 +838,11 @@ jobs:
             echo "::endgroup::"
           done
 
+      - name: Run Helm unit tests
+        run: |
+          helm plugin install https://github.com/helm-unittest/helm-unittest.git || true
+          helm unittest helm/nessie
+
       - name: Run chart-testing (lint)
         run: ct lint --debug --charts ./helm/nessie
 

CHANGELOG.md

@@ -10,6 +10,15 @@ as necessary. Empty sections will not end in the release notes.
 
 ### Upgrade notes
 
+- Helm chart: the old `logLevel` field has been replaced with a new `log` section with many more
+  options to configure logging. You can now configure console- and file-based logging separately. It
+  is also possible to enable JSON logging instead of plain text (but this feature requires Nessie >=
+  0.99.1). For file-based logging, it is also possible to configure rotation and retention policies,
+  and a persistent volume claim is now automatically created when file-based logging is enabled.
+  Furthermore, Sentry integration can also be enabled and configured. And finally, it is now
+  possible to configure the log level for specific loggers, not just the root logger. The old
+  `logLevel` field is still supported, but will be removed in a future release.
+
 ### Breaking changes
 
 ### New Features

helm/nessie/README.md

@@ -269,7 +269,36 @@ ct install --charts ./helm/nessie --namespace nessie-ns --debug
 | livenessProbe.successThreshold | int | `1` | Minimum consecutive successes for the probe to be considered successful after having failed. Minimum value is 1. |
 | livenessProbe.terminationGracePeriodSeconds | int | `30` | Optional duration in seconds the pod needs to terminate gracefully upon probe failure. Minimum value is 1. |
 | livenessProbe.timeoutSeconds | int | `10` | Number of seconds after which the probe times out. Minimum value is 1. |
-| logLevel | string | `"INFO"` | The minimum log level for the Nessie server. If you need to debug Nessie, set this to DEBUG, then add the following to the advancedConfig section: `quarkus.log.category."<logger>".level: DEBUG` where `<logger>` is the logger you want to debug. E.g. to debug issues with the Nessie configuration, add: `quarkus.log.category."io.smallrye.config".level: DEBUG` |
+| log | object | `{"categories":{"org.projectnessie":"INFO"},"console":{"enabled":true,"format":"%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] (%t) %s%e%n","json":false,"threshold":"ALL"},"file":{"enabled":false,"fileName":"nessie.log","format":"%d{yyyy-MM-dd HH:mm:ss,SSS} %h %N[%i] %-5p [%X{traceId},%X{spanId},%X{sampled}] [%c{3.}] (%t) %s%e%n","json":false,"logsDir":"/deployments/logs","rotation":{"fileSuffix":null,"maxBackupIndex":5,"maxFileSize":"100Mi"},"storage":{"className":"standard","selectorLabels":{},"size":"512Gi"},"threshold":"ALL"},"level":"INFO","sentry":{"dsn":null,"enabled":false,"environment":null,"inAppPackages":["org.projectnessie"],"level":"ERROR","release":null}}` | Logging configuration. |
+| log.categories | object | `{"org.projectnessie":"INFO"}` | Configuration for specific log categories. |
+| log.console | object | `{"enabled":true,"format":"%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] (%t) %s%e%n","json":false,"threshold":"ALL"}` | Configuration for the console appender. |
+| log.console.enabled | bool | `true` | Whether to enable the console appender. |
+| log.console.format | string | `"%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] (%t) %s%e%n"` | The log format to use. Ignored if JSON format is enabled. See https://quarkus.io/guides/logging#logging-format for details. |
+| log.console.json | bool | `false` | Whether to log in JSON format. |
+| log.console.threshold | string | `"ALL"` | The log level of the console appender. |
+| log.file | object | `{"enabled":false,"fileName":"nessie.log","format":"%d{yyyy-MM-dd HH:mm:ss,SSS} %h %N[%i] %-5p [%X{traceId},%X{spanId},%X{sampled}] [%c{3.}] (%t) %s%e%n","json":false,"logsDir":"/deployments/logs","rotation":{"fileSuffix":null,"maxBackupIndex":5,"maxFileSize":"100Mi"},"storage":{"className":"standard","selectorLabels":{},"size":"512Gi"},"threshold":"ALL"}` | Configuration for the file appender. |
+| log.file.enabled | bool | `false` | Whether to enable the file appender. |
+| log.file.fileName | string | `"nessie.log"` | The log file name. |
+| log.file.format | string | `"%d{yyyy-MM-dd HH:mm:ss,SSS} %h %N[%i] %-5p [%X{traceId},%X{spanId},%X{sampled}] [%c{3.}] (%t) %s%e%n"` | The log format to use. Ignored if JSON format is enabled. See https://quarkus.io/guides/logging#logging-format for details. |
+| log.file.json | bool | `false` | Whether to log in JSON format. |
+| log.file.logsDir | string | `"/deployments/logs"` | The local directory where log files are stored. The persistent volume claim will be mounted here. |
+| log.file.rotation | object | `{"fileSuffix":null,"maxBackupIndex":5,"maxFileSize":"100Mi"}` | Log rotation configuration. |
+| log.file.rotation.fileSuffix | string | `nil` | An optional suffix to append to the rotated log files. If present, the rotated log files will be grouped in time buckets, and each bucket will contain at most maxBackupIndex files. The suffix must be in a date-time format that is understood by DateTimeFormatter. If the suffix ends with .gz or .zip, the rotated files will also be compressed using the corresponding algorithm. |
+| log.file.rotation.maxBackupIndex | int | `5` | The maximum number of backup files to keep. |
+| log.file.rotation.maxFileSize | string | `"100Mi"` | The maximum size of the log file before it is rotated. Should be expressed as a Kubernetes quantity. |
+| log.file.storage | object | `{"className":"standard","selectorLabels":{},"size":"512Gi"}` | The log storage configuration. A persistent volume claim will be created using these settings. |
+| log.file.storage.className | string | `"standard"` | The storage class name of the persistent volume claim to create. |
+| log.file.storage.selectorLabels | object | `{}` | Labels to add to the persistent volume claim spec selector; a persistent volume with matching labels must exist. Leave empty if using dynamic provisioning. |
+| log.file.storage.size | string | `"512Gi"` | The size of the persistent volume claim to create. |
+| log.file.threshold | string | `"ALL"` | The log level of the file appender. |
+| log.level | string | `"INFO"` | The log level of the root category, which is used as the default log level for all categories. |
+| log.sentry | object | `{"dsn":null,"enabled":false,"environment":null,"inAppPackages":["org.projectnessie"],"level":"ERROR","release":null}` | Configuration for the Sentry appender. See https://sentry.io and https://docs.quarkiverse.io/quarkus-logging-sentry/dev for more information. |
+| log.sentry.dsn | string | `nil` | The Sentry DSN. Required. |
+| log.sentry.enabled | bool | `false` | Whether to enable the Sentry appender. |
+| log.sentry.environment | string | `nil` | The environment to report to Sentry. Optional. |
+| log.sentry.inAppPackages | list | `["org.projectnessie"]` | Package prefixes that belong to your application. |
+| log.sentry.level | string | `"ERROR"` | The log level of the Sentry appender. |
+| log.sentry.release | string | `nil` | The release version to report to Sentry. Optional. |
 | managementService | object | `{"annotations":{},"portName":"nessie-mgmt","portNumber":9000}` | Management service settings. These settings are used to configure liveness and readiness probes, and to configure the dedicated headless service that will expose health checks and metrics, e.g. for metrics scraping and service monitoring. |
 | managementService.annotations | object | `{}` | Annotations to add to the service. |
 | managementService.portName | string | `"nessie-mgmt"` | The name of the management port. Required. |

helm/nessie/ci/inmemory-values.yaml

@@ -1,5 +1,25 @@
 ---
 versionStoreType: IN_MEMORY
+log:
+  level: INFO
+  console:
+    enabled: true
+    json: false
+  file:
+    enabled: true
+    json: true
+    threshold: WARN
+    rotation:
+      maxFileSize: 1Mi
+      maxBackupIndex: 5
+      fileSuffix: ~
+    storage:
+      className: standard
+      size: 100Mi
+  categories:
+    org:
+      projectnessie: DEBUG
+    io.smallrye.config: DEBUG
 tracing:
   enabled: true
 metrics:

helm/nessie/templates/_helpers.tpl

@@ -466,3 +466,55 @@ ports:
   protocol: TCP
 {{ end -}}
 {{ end -}}
+
+{{/*
+Shared - Converts a Kubernetes quantity to a number (int64 if possible or float64 otherwise).
+It handles raw numbers as well as quantities with suffixes
+like m, k, M, G, T, P, E, ki, Mi, Gi, Ti, Pi, Ei.
+It also handles scientific notation.
+Quantities should be positive, so negative values, zero, or any unparseable number
+will result in a failure.
+https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/
+*/}}
+{{- define "nessie.quantity" -}}
+{{- $quantity := . -}}
+{{- $n := $quantity | float64 -}}
+{{- if kindIs "string" $quantity -}}
+{{- if hasSuffix "m" $quantity -}}
+{{- $n = divf (trimSuffix "m" $quantity | float64) 1000.0 -}}
+{{- else if hasSuffix "k" $quantity -}}
+{{- $n = trimSuffix "k" $quantity | int64 | mul 1000 -}}
+{{- else if hasSuffix "M" $quantity -}}
+{{- $n = trimSuffix "M" $quantity | int64 | mul 1000000 -}}
+{{- else if hasSuffix "G" $quantity -}}
+{{- $n = trimSuffix "G" $quantity | int64 | mul 1000000000 -}}
+{{- else if hasSuffix "T" $quantity -}}
+{{- $n = trimSuffix "T" $quantity | int64 | mul 1000000000000 -}}
+{{- else if hasSuffix "P" $quantity -}}
+{{- $n = trimSuffix "P" $quantity | int64 | mul 1000000000000000 -}}
+{{- else if hasSuffix "E" $quantity -}}
+{{- $n = trimSuffix "E" $quantity | int64 | mul 1000000000000000000 -}}
+{{- else if hasSuffix "ki" $quantity -}}
+{{- $n = trimSuffix "ki" $quantity | int64 | mul 1024 -}}
+{{- else if hasSuffix "Mi" $quantity -}}
+{{- $n = trimSuffix "Mi" $quantity | int64 | mul 1048576 -}}
+{{- else if hasSuffix "Gi" $quantity -}}
+{{- $n = trimSuffix "Gi" $quantity | int64 | mul 1073741824 -}}
+{{- else if hasSuffix "Ti" $quantity -}}
+{{- $n = trimSuffix "Ti" $quantity | int64 | mul 1099511627776 -}}
+{{- else if hasSuffix "Pi" $quantity -}}
+{{- $n = trimSuffix "Pi" $quantity | int64 | mul 1125899906842624 -}}
+{{- else if hasSuffix "Ei" $quantity -}}
+{{- $n = trimSuffix "Ei" $quantity | int64 | mul 1152921504606846976 -}}
+{{- end -}}
+{{- end -}}
+{{- if le ($n | float64) 0.0 -}}
+{{- fail (print "invalid quantity: " $quantity) -}}
+{{- end -}}
+{{- if kindIs "float64" $n -}}
+{{- printf "%f" $n -}}
+{{- else -}}
+{{- printf "%v" $n -}}
+{{- end -}}
+{{- end -}}
+

helm/nessie/templates/configmap.yaml

@@ -39,9 +39,54 @@ data:
     {{- $_ = set $map "quarkus.http.port" (get (first .Values.service.ports) "number") -}}
     {{- $_ = set $map "quarkus.management.port" .Values.managementService.portNumber -}}
 
-    {{- if .Values.logLevel -}}
-    {{- $_ = set $map "quarkus.log.console.level" .Values.logLevel -}}
-    {{- $_ = set $map "quarkus.log.file.level" .Values.logLevel -}}
+    {{- $_ = set $map "quarkus.log.level" (coalesce .Values.logLevel .Values.log.level) -}}
+    {{- if .Values.log.console.enabled -}}
+    {{- $_ = set $map "quarkus.log.console.enable" "true" -}}
+    {{- $_ = set $map "quarkus.log.console.level" .Values.log.console.threshold -}}
+    {{- if .Values.log.console.json -}}
+    {{- $_ = set $map "quarkus.log.console.json" "true" -}}
+    {{- else -}}
+    {{- $_ = set $map "quarkus.log.console.format" .Values.log.console.format -}}
+    {{- end -}}
+    {{- else -}}
+    {{- $_ = set $map "quarkus.log.console.enable" "false" -}}
+    {{- end -}}
+    {{- if .Values.log.file.enabled -}}
+    {{- $_ = set $map "quarkus.log.file.enable" "true" -}}
+    {{- $_ = set $map "quarkus.log.file.level" .Values.log.file.threshold -}}
+    {{- $_ = set $map "quarkus.log.file.path" (printf "%s/%s" .Values.log.file.logsDir .Values.log.file.fileName) -}}
+    {{- $_ = set $map "quarkus.log.file.rotation.max-file-size" (include "nessie.quantity" .Values.log.file.rotation.maxFileSize) -}}
+    {{- $_ = set $map "quarkus.log.file.rotation.max-backup-index" .Values.log.file.rotation.maxBackupIndex -}}
+    {{- if .Values.log.file.rotation.fileSuffix -}}
+    {{- $_ = set $map "quarkus.log.file.rotation.file-suffix" .Values.log.file.rotation.fileSuffix -}}
+    {{- end -}}
+    {{- if .Values.log.file.json -}}
+    {{- $_ = set $map "quarkus.log.file.json" "true" -}}
+    {{- else -}}
+    {{- $_ = set $map "quarkus.log.file.format" .Values.log.file.format -}}
+    {{- end -}}
+    {{- else -}}
+    {{- $_ = set $map "quarkus.log.file.enable" "false" -}}
+    {{- end -}}
+    {{- if .Values.log.sentry.enabled -}}
+    {{- $_ = set $map "quarkus.log.sentry" "true" -}}
+    {{- if not .Values.log.sentry.dsn -}}{{- required "log.sentry.dsn is required when log.sentry.enabled is true" .Values.log.sentry.dsn -}}{{- end -}}
+    {{- $_ = set $map "quarkus.log.sentry.dsn" .Values.log.sentry.dsn -}}
+    {{- $_ = set $map "quarkus.log.sentry.level" .Values.log.sentry.level -}}
+    {{- if .Values.log.sentry.environment -}}
+    {{- $_ = set $map "quarkus.log.sentry.environment" .Values.log.sentry.environment -}}
+    {{- end -}}
+    {{- if .Values.log.sentry.release -}}
+    {{- $_ = set $map "quarkus.log.sentry.release" .Values.log.sentry.release -}}
+    {{- end -}}
+    {{- $_ = set $map "quarkus.log.sentry.in-app-packages" ( join "," .Values.log.sentry.inAppPackages ) -}}
+    {{- else -}}
+    {{- $_ = set $map "quarkus.log.sentry" "false" -}}
+    {{- end -}}
+    {{- $categories := dict -}}
+    {{- list .Values.log.categories "" $categories | include "nessie.mergeAdvancedConfig" -}}
+    {{- range $k, $v := $categories -}}
+    {{- $_ = set $map (printf "quarkus.log.category.\"%s\".level" $k) $v -}}
     {{- end -}}
 
     {{- if hasPrefix "DYNAMODB" .Values.versionStoreType -}}

helm/nessie/templates/deployment.yaml

@@ -71,6 +71,11 @@ spec:
               mountPath: /rocks-nessie
               readOnly: false
           {{- end }}
+          {{- if .Values.log.file.enabled }}
+            - name: logs-storage
+              mountPath: {{ .Values.log.file.logsDir }}
+              readOnly: false
+          {{- end }}
           {{- if and (hasPrefix "BIGTABLE" .Values.versionStoreType) (.Values.bigtable.secret) }}
             - name: bigtable-creds
               mountPath: /bigtable-nessie
@@ -139,7 +144,6 @@ spec:
             successThreshold: {{ .Values.readinessProbe.successThreshold }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
             timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
-
           resources:
             {{- tpl (toYaml .Values.resources) . | nindent 12 }}
       volumes:
@@ -153,6 +157,11 @@ spec:
           persistentVolumeClaim:
             claimName: {{ include "nessie.fullname" . }}
       {{- end }}
+      {{- if .Values.log.file.enabled }}
+        - name: logs-storage
+          persistentVolumeClaim:
+            claimName: {{ include "nessie.fullname" . }}-logs
+      {{- end }}
       {{- if and (hasPrefix "BIGTABLE" .Values.versionStoreType) (.Values.bigtable.secret) }}
         - name: bigtable-creds
           secret:

helm/nessie/templates/storage.yaml

@@ -31,10 +31,33 @@ spec:
   storageClassName: {{ .Values.rocksdb.storageClassName }}
   resources:
     requests:
-      storage: {{ .Values.rocksdb.storageSize }}
+      storage: "{{ .Values.rocksdb.storageSize }}"
 {{- if .Values.rocksdb.selectorLabels }}
   selector:
     matchLabels:
       {{- toYaml .Values.rocksdb.selectorLabels | nindent 6 }}
 {{- end }}
 {{- end }}
+---
+{{- if .Values.log.file.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "nessie.fullname" . }}-logs
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nessie.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  storageClassName: {{ .Values.log.file.storage.className }}
+  resources:
+    requests:
+      storage: "{{ .Values.log.file.storage.size }}"
+{{- if .Values.log.file.storage.selectorLabels }}
+  selector:
+    matchLabels:
+      {{- toYaml .Values.log.file.storage.selectorLabels | nindent 6 }}
+{{- end }}
+{{- end }}

helm/nessie/tests/logging_storage_test.yaml

@@ -0,0 +1,103 @@
+##
+## Copyright (C) 2024 Dremio
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+
+release:
+  name: nessie
+  namespace: nessie-ns
+
+tests:
+  - it: should create PVC for file logging
+    templates:
+      - storage.yaml
+    set:
+      log.file.enabled: true
+      log.file.logsDir: /var/log
+      log.file.fileName: app.log
+      log.file.storage.className: standard-rwo
+      log.file.storage.size: 100Gi
+    asserts:
+      - containsDocument:
+          kind: PersistentVolumeClaim
+          apiVersion: v1
+          name: nessie-logs
+          namespace: nessie-ns
+      - equal:
+          path: spec
+          value:
+            accessModes:
+              - ReadWriteOnce
+            volumeMode: Filesystem
+            storageClassName: standard-rwo
+            resources:
+              requests:
+                storage: 100Gi
+      - notExists:
+          path: spec.selector
+  - it: should create PVC for file logging with selector labels
+    templates:
+      - storage.yaml
+    set:
+      log.file.enabled: true
+      log.file.logsDir: /var/log
+      log.file.fileName: app.log
+      log.file.storage.className: standard-rwo
+      log.file.storage.size: 100Gi
+      log.file.storage.selectorLabels:
+        app: nessie
+    documentSelector:
+      path: metadata.name
+      value: nessie-logs
+    asserts:
+      - containsDocument:
+          kind: PersistentVolumeClaim
+          apiVersion: v1
+          name: nessie-logs
+          namespace: nessie-ns
+      - equal:
+          path: spec
+          value:
+            accessModes:
+              - ReadWriteOnce
+            volumeMode: Filesystem
+            storageClassName: standard-rwo
+            resources:
+              requests:
+                storage: 100Gi
+            selector:
+              matchLabels:
+                app: nessie
+  - it: should mount PVC for file logging
+    templates:
+      - deployment.yaml
+      - configmap.yaml
+    set:
+      log.file.enabled: true
+      log.file.logsDir: /var/log
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: logs-storage
+            mountPath: /var/log
+            readOnly: false
+        template: deployment.yaml
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: logs-storage
+            persistentVolumeClaim:
+              claimName: nessie-logs
+        template: deployment.yaml