refactor: allow config from yaml file on chart values

charts/sorting_hat_step/Chart.yaml

@@ -3,4 +3,4 @@ appVersion: 6.2.1a80
 description: Sorting Hat deployment chart
 name: sorting-hat
 type: application
-version: 2.6.161
+version: 3.0.0

charts/sorting_hat_step/templates/configmap.yaml

@@ -2,33 +2,8 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: '{{ include "sorting-hat.fullname" . }}'
+  name: config-yaml
   namespace: {{ .Values.namespace }}
 data:
-  consumer-topics: "{{ .Values.configmap.consumerTopics }}"
-  topic-strategy-topic-format: "{{ .Values.configmap.topicStrategy.topicFormat }}"
-  topic-strategy-date-format: "{{ .Values.configmap.topicStrategy.dateFormat }}"
-  topic-strategy-change-hour: "{{ .Values.configmap.topicStrategy.changeHour }}"
-  consumer-group-id: {{ .Values.configmap.consumerGroupId | default (printf "%s" (include "sorting-hat.fullname" .)) }}
-  consumer-server: {{ .Values.configmap.consumerServer }}
-  consumer-class: {{ .Values.configmap.consumerClass }}
-  producer-server: {{ .Values.configmap.producerServer }}
-  producer-topic: {{ .Values.configmap.producerTopic }}
-  producer-message-max-bytes: "{{ .Values.configmap.producerMessageMaxBytes }}"
-  pyroscope-server: {{ .Values.configmap.pyroscopeServer }}
-  metrics-server: {{ .Values.configmap.metricsServer }}
-  metrics-topic: {{ .Values.configmap.metricsTopic }}
-  mongo-secret-name: {{ .Values.configmap.mongoSecretName }}
-  {{- if .Values.featureFlags.usePsql }}
-  psql-secret-name: {{ .Values.configmap.psqlSecretName }}
-  {{- end }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: 'feature-flags'
-  namespace: {{ .Values.namespace }}
-data:
-  run-conesearch: '{{ .Values.featureFlags.runConesearch }}'
-  use-profiling: '{{ .Values.featureFlags.useProfiling }}'
-  use-psql: '{{ .Values.featureFlags.usePsql }}'
+  config.yaml: |
+    {{- toYaml .Values.configYaml | nindent 4 }}

charts/sorting_hat_step/templates/deployment.yaml

@@ -38,146 +38,23 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           env:
-            {{- if .Values.loggingDebug }}
-            - name: LOGGING_DEBUG
-              value: "YES"
-            {{- end}}
-            {{- if .Values.configmap.topicStrategy.enabled }}
-            - name: TOPIC_STRATEGY_TOPIC_FORMAT
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: topic-strategy-topic-format
-            - name: TOPIC_STRATEGY_DATE_FORMAT
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: topic-strategy-date-format
-            - name: TOPIC_STRATEGY_CHANGE_HOUR
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: topic-strategy-change-hour
-            {{- end }}
-            {{- if not .Values.configmap.topicStrategy.enabled }}
-            - name: CONSUMER_TOPICS
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: consumer-topics
-            {{- end}}
-            - name: CONSUMER_SERVER
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: consumer-server
-            - name: CONSUMER_CLASS
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: consumer-class
-            - name: CONSUMER_GROUP_ID
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: consumer-group-id
-            - name: PRODUCER_SERVER
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: producer-server
-            - name: PRODUCER_TOPIC
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: producer-topic
-            - name: PRODUCER_MESSAGE_MAX_BYTES
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: producer-message-max-bytes
-            - name: MONGODB_SECRET_NAME
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: mongo-secret-name
-            {{- if .Values.featureFlags.usePsql }}
-            - name: PSQL_SECRET_NAME
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: psql-secret-name
-            {{- end }}
-            - name: METRICS_HOST
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: metrics-server
-            - name: METRICS_TOPIC
-              valueFrom:
-                configMapKeyRef:
-                  name: '{{ include "sorting-hat.fullname" . }}'
-                  key: metrics-topic
-            - name: RUN_CONESEARCH
-              valueFrom:
-                configMapKeyRef:
-                  name: 'feature-flags'
-                  key: run-conesearch
-            - name: USE_PROFILING
-              valueFrom:
-                configMapKeyRef:
-                  name: 'feature-flags'
-                  key: use-profiling
-            - name: USE_PSQL
-              valueFrom:
-                configMapKeyRef:
-                  name: 'feature-flags'
-                  key: use-psql
-            - name: PYROSCOPE_SERVER
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ include "sorting-hat.fullname" . }}
-                  key: pyroscope-server
-            {{- if .Values.secrets.kafkaAuth.consumer.enabled }}
-            - name: CONSUMER_KAFKA_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: consumer-username
-            - name: CONSUMER_KAFKA_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: consumer-password
-            {{- end }}
-            {{- if .Values.secrets.kafkaAuth.producer.enabled }}
-            - name: PRODUCER_KAFKA_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: producer-username
-            - name: PRODUCER_KAFKA_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: producer-password
-            {{- end }}
-            {{- if .Values.secrets.kafkaAuth.metrics.enabled }}
-            - name: METRICS_KAFKA_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: metrics-username
-            - name: METRICS_KAFKA_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: kafka-auth
-                  key: metrics-password
-            {{- end }}
-            {{- with .Values.extraEnvVariables }}
-            {{- tpl (toYaml .) $ | nindent 12 }}
-            {{- end}}
+            {{- toYaml .Values.envVariables | nindent 12 }}
+          {{- if .Values.configYaml.enabled }}
+          volumeMounts:
+            - name: config
+              mountPath: "/config"
+              readOnly: true
+          {{- end}}
       {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if .Values.configYaml.enabled }}
+      volumes:
+        - name: config
+          configMap:
+            name: config-yaml
+            items:
+              - key: "config.yaml"
+                path: "config.yaml"
+      {{- end }}

charts/sorting_hat_step/values.yaml

@@ -20,10 +20,6 @@ fullnameOverride: ""
 
 podAnnotations: {}
 
-service:
-  type: ClusterIP
-  port: 8000
-
 resources:
   requests:
     memory: 750M
@@ -45,45 +41,51 @@ affinity:
               ## Override this value with the NodeGroup tag accordingly
               values: []
 
-loggingDebug: false
-configmap:
-  consumerTopics: ""
-  topicStrategy:
-    enabled: false
-    topicFormat: ""
-    dateFormat: ""
-    changeHour: ""
-  consumerGroupId: ""
-  consumerServer: ""
-  consumerClass: ""
-  producerServer: ""
-  producerTopic: ""
-  producerMessageMaxBytes: ""
-  metricsTopic: ""
-  metricsServer: ""
-  mongoSecretName: ""
-  psqlSecretName: ""
-  pyroscopeServer: "http://pyroscope.pyroscope:4040"
-
-featureFlags:
-  runConesearch: ""
-  useProfiling: ""
-  usePsql: "False"
+configYaml:
+  enabled: true
+  FEATURE_FLAGS:
+    RUN_CONESEARCH: true
+    USE_PROFILING: true
+    USE_PSQL: true
+    PROMETHEUS: false
+  MONGO_SECRET_NAME: ""
+  PSQL_SECRET_NAME: ""
+  CONSUMER_CONFIG:
+    CLASS: "apf.consumers.KafkaConsumer"
+    SCHEMA_PATH: ""
+    TOPICS: [""]
+    PARAMS:
+      bootstrap.servers: ""
+      group.id: ""
+      auto.offset.reset: "beginning"
+      enable.partition.eof: false
+      security.protocol: "SASL_SSL"
+      sasl.mechanism: "SCRAM-SHA-512"
+      sasl.username: ""
+      sasl.password: ""
+    consume.timeout: 10
+    consume.messages: 100
+  PRODUCER_CONFIG:
+    PARAMS:
+      bootstrap.servers: ""
+      security.protocol: "SASL_PLAINTEXT"
+      sasl.mechanism: "SCRAM-SHA-256"
+      sasl.username: ""
+      sasl.password: ""
+      message.max.bytes: 6291456
+    CLASS: "apf.producers.kafka.KafkaProducer"
+    TOPIC: ""
+    SCHEMA_PATH: "/schemas/sorting_hat_step/output.avsc"
+  METRICS_CONFIG:
+    CLASS: "apf.metrics.KafkaMetricsProducer"
+    PARAMS:
+      PARAMS:
+        bootstrap.servers: ""
+      TOPIC: "metrics"
+      SCHEMA_PATH: "/schemas/lc_classification_step/metrics.json"
+  LOGGING_DEBUG: false
+  PYROSCOPE_SERVER: "http://pyroscope.pyroscope:4040"
 
-secrets:
-  kafkaAuth:
-    consumer:
-      enabled: false
-      username: ""
-      password: ""
-    producer:
-      enabled: false
-      username: ""
-      password: ""
-    metrics:
-      enabled: false
-      username: ""
-      password: ""
 
 imageCredentials:
   registry: ""
@@ -93,8 +95,9 @@ imageCredentials:
 
 # Variables to add to deployment template
 # These are yaml formatted as received by `env` field of the container spec
-extraEnvVariables:
-  ##
+envVariables:
+  - name: CONFIG_FROM_YAML
+    value: "yes"
   ## - name: METRICS_SOURCE
   ##   value: {{ .Release.Name }}
   ## - name: METRICS_SURVEY

sorting_hat_step/credentials.py

@@ -37,7 +37,7 @@ def __setitem__(self, key, value):
         super().__setitem__("".join(klist), value)
 
 
-def get_secret(secret_name: str):
+def get_secret(secret_name: str) -> str:
     secret_name = secret_name
     region_name = "us-east-1"
 
@@ -54,7 +54,7 @@ def get_secret(secret_name: str):
     return get_secret_value_response["SecretString"]
 
 
-def get_credentials(secret_name: str, secret_type="mongo"):
+def get_credentials(secret_name: str, secret_type="mongo") -> UserDict | dict:
     secret = get_secret(secret_name)
     secret = json.loads(secret)
     # check if config is valid