From c4d2a991af69a299ba0d2d4bb76ed3e92e6fbcaa Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Wed, 15 Nov 2023 12:07:32 -0300 Subject: [PATCH] refactor: allow config from yaml file on chart values --- charts/sorting_hat_step/Chart.yaml | 2 +- .../sorting_hat_step/templates/configmap.yaml | 31 +--- .../templates/deployment.yaml | 155 ++---------------- charts/sorting_hat_step/values.yaml | 91 +++++----- sorting_hat_step/credentials.py | 4 +- sorting_hat_step/scripts/run_step.py | 28 ++-- sorting_hat_step/settings.py | 24 +-- sorting_hat_step/sorting_hat_step/step.py | 5 +- 8 files changed, 103 insertions(+), 237 deletions(-) diff --git a/charts/sorting_hat_step/Chart.yaml b/charts/sorting_hat_step/Chart.yaml index 363abd616..83f165fd5 100644 --- a/charts/sorting_hat_step/Chart.yaml +++ b/charts/sorting_hat_step/Chart.yaml @@ -3,4 +3,4 @@ appVersion: 6.2.1a80 description: Sorting Hat deployment chart name: sorting-hat type: application -version: 2.6.161 +version: 3.0.0 diff --git a/charts/sorting_hat_step/templates/configmap.yaml b/charts/sorting_hat_step/templates/configmap.yaml index 0789cb7bc..3f38b8a70 100644 --- a/charts/sorting_hat_step/templates/configmap.yaml +++ b/charts/sorting_hat_step/templates/configmap.yaml @@ -2,33 +2,8 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ include "sorting-hat.fullname" . }}' + name: config-yaml namespace: {{ .Values.namespace }} data: - consumer-topics: "{{ .Values.configmap.consumerTopics }}" - topic-strategy-topic-format: "{{ .Values.configmap.topicStrategy.topicFormat }}" - topic-strategy-date-format: "{{ .Values.configmap.topicStrategy.dateFormat }}" - topic-strategy-change-hour: "{{ .Values.configmap.topicStrategy.changeHour }}" - consumer-group-id: {{ .Values.configmap.consumerGroupId | default (printf "%s" (include "sorting-hat.fullname" .)) }} - consumer-server: {{ .Values.configmap.consumerServer }} - consumer-class: {{ .Values.configmap.consumerClass }} - producer-server: {{ .Values.configmap.producerServer }} - producer-topic: {{ .Values.configmap.producerTopic }} - producer-message-max-bytes: "{{ .Values.configmap.producerMessageMaxBytes }}" - pyroscope-server: {{ .Values.configmap.pyroscopeServer }} - metrics-server: {{ .Values.configmap.metricsServer }} - metrics-topic: {{ .Values.configmap.metricsTopic }} - mongo-secret-name: {{ .Values.configmap.mongoSecretName }} - {{- if .Values.featureFlags.usePsql }} - psql-secret-name: {{ .Values.configmap.psqlSecretName }} - {{- end }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: 'feature-flags' - namespace: {{ .Values.namespace }} -data: - run-conesearch: '{{ .Values.featureFlags.runConesearch }}' - use-profiling: '{{ .Values.featureFlags.useProfiling }}' - use-psql: '{{ .Values.featureFlags.usePsql }}' + config.yaml: | + {{- toYaml .Values.configYaml | nindent 4 }} diff --git a/charts/sorting_hat_step/templates/deployment.yaml b/charts/sorting_hat_step/templates/deployment.yaml index 19ef7fd5a..21ab3743a 100644 --- a/charts/sorting_hat_step/templates/deployment.yaml +++ b/charts/sorting_hat_step/templates/deployment.yaml @@ -38,146 +38,23 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} env: - {{- if .Values.loggingDebug }} - - name: LOGGING_DEBUG - value: "YES" - {{- end}} - {{- if .Values.configmap.topicStrategy.enabled }} - - name: TOPIC_STRATEGY_TOPIC_FORMAT - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: topic-strategy-topic-format - - name: TOPIC_STRATEGY_DATE_FORMAT - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: topic-strategy-date-format - - name: TOPIC_STRATEGY_CHANGE_HOUR - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: topic-strategy-change-hour - {{- end }} - {{- if not .Values.configmap.topicStrategy.enabled }} - - name: CONSUMER_TOPICS - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: consumer-topics - {{- end}} - - name: CONSUMER_SERVER - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: consumer-server - - name: CONSUMER_CLASS - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: consumer-class - - name: CONSUMER_GROUP_ID - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: consumer-group-id - - name: PRODUCER_SERVER - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: producer-server - - name: PRODUCER_TOPIC - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: producer-topic - - name: PRODUCER_MESSAGE_MAX_BYTES - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: producer-message-max-bytes - - name: MONGODB_SECRET_NAME - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: mongo-secret-name - {{- if .Values.featureFlags.usePsql }} - - name: PSQL_SECRET_NAME - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: psql-secret-name - {{- end }} - - name: METRICS_HOST - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: metrics-server - - name: METRICS_TOPIC - valueFrom: - configMapKeyRef: - name: '{{ include "sorting-hat.fullname" . }}' - key: metrics-topic - - name: RUN_CONESEARCH - valueFrom: - configMapKeyRef: - name: 'feature-flags' - key: run-conesearch - - name: USE_PROFILING - valueFrom: - configMapKeyRef: - name: 'feature-flags' - key: use-profiling - - name: USE_PSQL - valueFrom: - configMapKeyRef: - name: 'feature-flags' - key: use-psql - - name: PYROSCOPE_SERVER - valueFrom: - configMapKeyRef: - name: {{ include "sorting-hat.fullname" . }} - key: pyroscope-server - {{- if .Values.secrets.kafkaAuth.consumer.enabled }} - - name: CONSUMER_KAFKA_USERNAME - valueFrom: - secretKeyRef: - name: kafka-auth - key: consumer-username - - name: CONSUMER_KAFKA_PASSWORD - valueFrom: - secretKeyRef: - name: kafka-auth - key: consumer-password - {{- end }} - {{- if .Values.secrets.kafkaAuth.producer.enabled }} - - name: PRODUCER_KAFKA_USERNAME - valueFrom: - secretKeyRef: - name: kafka-auth - key: producer-username - - name: PRODUCER_KAFKA_PASSWORD - valueFrom: - secretKeyRef: - name: kafka-auth - key: producer-password - {{- end }} - {{- if .Values.secrets.kafkaAuth.metrics.enabled }} - - name: METRICS_KAFKA_USERNAME - valueFrom: - secretKeyRef: - name: kafka-auth - key: metrics-username - - name: METRICS_KAFKA_PASSWORD - valueFrom: - secretKeyRef: - name: kafka-auth - key: metrics-password - {{- end }} - {{- with .Values.extraEnvVariables }} - {{- tpl (toYaml .) $ | nindent 12 }} - {{- end}} + {{- toYaml .Values.envVariables | nindent 12 }} + {{- if .Values.configYaml.enabled }} + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + {{- end}} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.configYaml.enabled }} + volumes: + - name: config + configMap: + name: config-yaml + items: + - key: "config.yaml" + path: "config.yaml" + {{- end }} diff --git a/charts/sorting_hat_step/values.yaml b/charts/sorting_hat_step/values.yaml index 8574a0764..c137c1bb1 100644 --- a/charts/sorting_hat_step/values.yaml +++ b/charts/sorting_hat_step/values.yaml @@ -20,10 +20,6 @@ fullnameOverride: "" podAnnotations: {} -service: - type: ClusterIP - port: 8000 - resources: requests: memory: 750M @@ -45,45 +41,51 @@ affinity: ## Override this value with the NodeGroup tag accordingly values: [] -loggingDebug: false -configmap: - consumerTopics: "" - topicStrategy: - enabled: false - topicFormat: "" - dateFormat: "" - changeHour: "" - consumerGroupId: "" - consumerServer: "" - consumerClass: "" - producerServer: "" - producerTopic: "" - producerMessageMaxBytes: "" - metricsTopic: "" - metricsServer: "" - mongoSecretName: "" - psqlSecretName: "" - pyroscopeServer: "http://pyroscope.pyroscope:4040" - -featureFlags: - runConesearch: "" - useProfiling: "" - usePsql: "False" +configYaml: + enabled: true + FEATURE_FLAGS: + RUN_CONESEARCH: true + USE_PROFILING: true + USE_PSQL: true + PROMETHEUS: false + MONGO_SECRET_NAME: "" + PSQL_SECRET_NAME: "" + CONSUMER_CONFIG: + CLASS: "apf.consumers.KafkaConsumer" + SCHEMA_PATH: "" + TOPICS: [""] + PARAMS: + bootstrap.servers: "" + group.id: "" + auto.offset.reset: "beginning" + enable.partition.eof: false + security.protocol: "SASL_SSL" + sasl.mechanism: "SCRAM-SHA-512" + sasl.username: "" + sasl.password: "" + consume.timeout: 10 + consume.messages: 100 + PRODUCER_CONFIG: + PARAMS: + bootstrap.servers: "" + security.protocol: "SASL_PLAINTEXT" + sasl.mechanism: "SCRAM-SHA-256" + sasl.username: "" + sasl.password: "" + message.max.bytes: 6291456 + CLASS: "apf.producers.kafka.KafkaProducer" + TOPIC: "" + SCHEMA_PATH: "/schemas/sorting_hat_step/output.avsc" + METRICS_CONFIG: + CLASS: "apf.metrics.KafkaMetricsProducer" + PARAMS: + PARAMS: + bootstrap.servers: "" + TOPIC: "metrics" + SCHEMA_PATH: "/schemas/lc_classification_step/metrics.json" + LOGGING_DEBUG: false + PYROSCOPE_SERVER: "http://pyroscope.pyroscope:4040" -secrets: - kafkaAuth: - consumer: - enabled: false - username: "" - password: "" - producer: - enabled: false - username: "" - password: "" - metrics: - enabled: false - username: "" - password: "" imageCredentials: registry: "" @@ -93,8 +95,9 @@ imageCredentials: # Variables to add to deployment template # These are yaml formatted as received by `env` field of the container spec -extraEnvVariables: - ## +envVariables: + - name: CONFIG_FROM_YAML + value: "yes" ## - name: METRICS_SOURCE ## value: {{ .Release.Name }} ## - name: METRICS_SURVEY diff --git a/sorting_hat_step/credentials.py b/sorting_hat_step/credentials.py index e49944477..a2c0fed95 100644 --- a/sorting_hat_step/credentials.py +++ b/sorting_hat_step/credentials.py @@ -37,7 +37,7 @@ def __setitem__(self, key, value): super().__setitem__("".join(klist), value) -def get_secret(secret_name: str): +def get_secret(secret_name: str) -> str: secret_name = secret_name region_name = "us-east-1" @@ -54,7 +54,7 @@ def get_secret(secret_name: str): return get_secret_value_response["SecretString"] -def get_credentials(secret_name: str, secret_type="mongo"): +def get_credentials(secret_name: str, secret_type="mongo") -> UserDict | dict: secret = get_secret(secret_name) secret = json.loads(secret) # check if config is valid diff --git a/sorting_hat_step/scripts/run_step.py b/sorting_hat_step/scripts/run_step.py index 47e2edb64..a8d881398 100644 --- a/sorting_hat_step/scripts/run_step.py +++ b/sorting_hat_step/scripts/run_step.py @@ -7,6 +7,8 @@ from prometheus_client import start_http_server from apf.metrics.prometheus import PrometheusMetrics from sorting_hat_step.database import MongoConnection, PsqlConnection +from apf.core.settings import config_from_yaml_file +from credentials import get_credentials SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -15,42 +17,48 @@ sys.path.append(PACKAGE_PATH) from settings import STEP_CONFIG +if os.getenv("CONFIG_FROM_YAML"): + STEP_CONFIG = config_from_yaml_file("/config/config.yaml") + STEP_CONFIG["METRICS_CONFIG"]["EXTRA_METRICS"] = [{"key":"candid", "format": lambda x: str(x)}] + level = logging.INFO -if os.getenv("LOGGING_DEBUG"): +if STEP_CONFIG.get("LOGGING_DEBUG"): level = logging.DEBUG logger = logging.getLogger("alerce") logger.setLevel(level) - fmt = logging.Formatter( "%(asctime)s %(levelname)7s %(name)36s: %(message)s", "%Y-%m-%d %H:%M:%S" ) handler = logging.StreamHandler() handler.setFormatter(fmt) handler.setLevel(level) - logger.addHandler(handler) from sorting_hat_step.step import SortingHatStep -mongo_database = MongoConnection(STEP_CONFIG["MONGO_CONFIG"]) +mongo_database = MongoConnection( + get_credentials(STEP_CONFIG["MONGODB_SECRET_NAME"], secret_type="mongo") +) -if bool(os.getenv("USE_PROFILING", True)): +if bool(STEP_CONFIG["FEATURE_FLAGS"].get("USE_PROFILING", True)): logger.info("Configuring Pyroscope profiling...") pyroscope.configure( application_name="steps.SortingHat", - server_address=os.getenv("PYROSCOPE_SERVER"), + server_address=STEP_CONFIG.get("PYROSCOPE_SERVER", ""), ) -prometheus_metrics = PrometheusMetrics() -start_http_server(8000) +prometheus_metrics = None +if STEP_CONFIG["FEATURE_FLAGS"]["PROMETHEUS"]: + prometheus_metrics = PrometheusMetrics() + start_http_server(8000) step = SortingHatStep( mongo_connection=mongo_database, config=STEP_CONFIG, prometheus_metrics=prometheus_metrics, ) -if STEP_CONFIG["USE_PSQL"].lower() == "true": - psql_database = PsqlConnection(STEP_CONFIG["PSQL_CONFIG"]) +if STEP_CONFIG["FEATURE_FLAGS"]["USE_PSQL"].lower() == "true": + psql_database = PsqlConnection(get_credentials(STEP_CONFIG["PSQL_SECRET_NAME"], secret_type="psql") step.set_psql_driver(psql_database) step.start() diff --git a/sorting_hat_step/settings.py b/sorting_hat_step/settings.py index cc8adf77c..3e056122f 100644 --- a/sorting_hat_step/settings.py +++ b/sorting_hat_step/settings.py @@ -21,9 +21,6 @@ # Export prometheus metrics PROMETHEUS = True -MONGO_CONFIG = get_credentials(os.environ["MONGODB_SECRET_NAME"], secret_type="mongo") -PSQL_CONFIG = {} - # Consumer configuration # Each consumer has different parameters and can be found in the documentation CONSUMER_CONFIG = { @@ -114,19 +111,24 @@ ) ## Feature flags -RUN_CONESEARCH = os.getenv("RUN_CONESEARCH", "True") -USE_PSQL = os.getenv("USE_PSQL", "False") +RUN_CONESEARCH = bool(os.getenv("RUN_CONESEARCH", False)) +USE_PSQL = bool(os.getenv("USE_PSQL", True)) -if USE_PSQL.lower() == "true": +if USE_PSQL: PSQL_CONFIG = get_credentials(os.environ["PSQL_SECRET_NAME"], secret_type="psql") # Step Configuration STEP_CONFIG = { - "PROMETHEUS": PROMETHEUS, - "MONGO_CONFIG": MONGO_CONFIG, - "PSQL_CONFIG": PSQL_CONFIG, + "FEATURE_FLAGS": { + "RUN_CONESEARCH": RUN_CONESEARCH, + "USE_PSQL": USE_PSQL, + "USE_PROFILING": os.getenv("USE_PROFILING", "True"), + "PROMETHEUS": PROMETHEUS, + }, + "MONGO_SECRET_NAME": os.getenv("MONGO_SECRET_NAME"), + "PSQL_SECRET_NAME": os.getenv("PSQL_SECRET_NAME", ""), "CONSUMER_CONFIG": CONSUMER_CONFIG, "PRODUCER_CONFIG": PRODUCER_CONFIG, "METRICS_CONFIG": METRICS_CONFIG, - "RUN_CONESEARCH": RUN_CONESEARCH, - "USE_PSQL": USE_PSQL, + "LOGGING_DEBUG": LOGGING_DEBUG, + "PYROSCOPE_SERVER": os.getenv("PYROSCOPE_SERVER", ""), } diff --git a/sorting_hat_step/sorting_hat_step/step.py b/sorting_hat_step/sorting_hat_step/step.py index 139ba5e96..65ba2a8c7 100644 --- a/sorting_hat_step/sorting_hat_step/step.py +++ b/sorting_hat_step/sorting_hat_step/step.py @@ -17,9 +17,10 @@ def __init__( ): super().__init__(config=config, **kwargs) self.mongo_driver = mongo_connection - self.run_conesearch = config["RUN_CONESEARCH"] != "False" self.parser = ALeRCEParser() - self.use_psql = config["USE_PSQL"].lower() == "true" + # feature flags + self.use_psql = config["FEATURE_FLAGS"]["USE_PSQL"] + self.run_conesearch = config["FEATURE_FLAGS"]["RUN_CONESEARCH"] def set_psql_driver(self, psql_connection: PsqlConnection): if not self.use_psql: