From bf02100a3668b10f50ff9338c36b7de54a8513ef Mon Sep 17 00:00:00 2001 From: Sam Arbid Date: Tue, 26 Nov 2024 14:49:25 +0100 Subject: [PATCH] config: granular env-based solution for connection strings Add logic to build connection urls from env vars if available needed for helm charts security best practices. includes: * Build db uri * Build redis url * Build mq url Partially closes: https://github.com/inveniosoftware/helm-invenio/issues/112 --- invenio_config/utils.py | 60 +++++++++++++++++ tests/test_invenio_config.py | 123 +++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) diff --git a/invenio_config/utils.py b/invenio_config/utils.py index 1ee5cf0..6c5c854 100644 --- a/invenio_config/utils.py +++ b/invenio_config/utils.py @@ -2,6 +2,7 @@ # # This file is part of Invenio. # Copyright (C) 2015-2018 CERN. +# Copyright (C) 2024 KTH Royal Institute of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -10,6 +11,8 @@ from __future__ import absolute_import, print_function +import os + from .default import InvenioConfigDefault from .entrypoint import InvenioConfigEntryPointModule from .env import InvenioConfigEnvironment @@ -77,3 +80,60 @@ def create_conf_loader(*args, **kwargs): # pragma: no cover DeprecationWarning, ) return create_config_loader(*args, **kwargs) + + +def build_db_uri(): + """Build the database URI.""" + DEFAULT_URI = "postgresql+psycopg2://invenio-app-rdm:invenio-app-rdm@localhost/invenio-app-rdm" + params = { + k: os.environ.get(f"DB_{k.upper()}") + for k in ["user", "password", "host", "port", "name"] + } + + if all(params.values()): + uri = f"postgresql+psycopg2://{params['user']}:{params['password']}@{params['host']}:{params['port']}/{params['name']}" + return uri + + uri = os.environ.get("SQLALCHEMY_DATABASE_URI") + if uri: + return uri + return DEFAULT_URI + + +def build_broker_url(): + """Build the broker URL.""" + DEFAULT_BROKER_URL = "amqp://guest:guest@localhost:5672/" + params = { + k: os.environ.get(f"BROKER_{k.upper()}") + for k in ["user", "password", "host", "port"] + } + + if all(params.values()): + uri = f"amqp://{params['user']}:{params['password']}@{params['host']}:{params['port']}/" + return uri + + uri = os.environ.get("BROKER_URL") + if uri: + return uri + + return DEFAULT_BROKER_URL + + +def build_redis_url(db=None): + """Build the Redis broker URL.""" + redis_host = os.environ.get("REDIS_HOST") + redis_port = os.environ.get("REDIS_PORT") + redis_password = os.environ.get("REDIS_PASSWORD") + db = db if db is not None else 0 + DEFAULT_BROKER_URL = f"redis://localhost:6379/{db}" + + if redis_host and redis_port: + password = f":{redis_password}@" if redis_password else "" + uri = f"redis://{password}{redis_host}:{redis_port}/{db}" + return uri + + uri = os.environ.get("BROKER_URL") + if uri and uri.startswith(("redis://", "rediss://", "unix://")): + return uri + + return DEFAULT_BROKER_URL diff --git a/tests/test_invenio_config.py b/tests/test_invenio_config.py index 4461648..957b62c 100644 --- a/tests/test_invenio_config.py +++ b/tests/test_invenio_config.py @@ -2,6 +2,7 @@ # # This file is part of Invenio. # Copyright (C) 2015-2018 CERN. +# Copyright (C) 2024 KTH Royal Institute of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -16,6 +17,7 @@ import warnings from os.path import join +import pytest from flask import Flask from mock import patch from pkg_resources import EntryPoint @@ -29,6 +31,7 @@ create_config_loader, ) from invenio_config.default import ALLOWED_HTML_ATTRS, ALLOWED_HTML_TAGS +from invenio_config.utils import build_broker_url, build_db_uri, build_redis_url class ConfigEP(EntryPoint): @@ -231,3 +234,123 @@ class Config(object): assert app.config["ENV"] == "env" finally: shutil.rmtree(tmppath) + + +@pytest.mark.parametrize( + "env_vars, expected_uri", + [ + ( + { + "DB_USER": "testuser", + "DB_PASSWORD": "testpassword", + "DB_HOST": "testhost", + "DB_PORT": "5432", + "DB_NAME": "testdb", + }, + "postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb", + ), + ( + {"SQLALCHEMY_DATABASE_URI": "sqlite:///test.db"}, + "sqlite:///test.db", + ), + ( + {}, + "postgresql+psycopg2://invenio-app-rdm:invenio-app-rdm@localhost/invenio-app-rdm", + ), + ], +) +def test_build_db_uri(monkeypatch, env_vars, expected_uri): + """Test building database URI.""" + for key in [ + "DB_USER", + "DB_PASSWORD", + "DB_HOST", + "DB_PORT", + "DB_NAME", + "SQLALCHEMY_DATABASE_URI", + ]: + monkeypatch.delenv(key, raising=False) + for key, value in env_vars.items(): + monkeypatch.setenv(key, value) + + assert build_db_uri() == expected_uri + + +@pytest.mark.parametrize( + "env_vars, expected_url", + [ + ( + { + "BROKER_USER": "testuser", + "BROKER_PASSWORD": "testpassword", + "BROKER_HOST": "testhost", + "BROKER_PORT": "5672", + }, + "amqp://testuser:testpassword@testhost:5672/", + ), + ( + {"BROKER_URL": "amqp://guest:guest@localhost:5672/"}, + "amqp://guest:guest@localhost:5672/", + ), + ( + {}, + "amqp://guest:guest@localhost:5672/", + ), + ], +) +def test_build_broker_url(monkeypatch, env_vars, expected_url): + """Test building broker URL.""" + for key in [ + "BROKER_USER", + "BROKER_PASSWORD", + "BROKER_HOST", + "BROKER_PORT", + "BROKER_URL", + ]: + monkeypatch.delenv(key, raising=False) + for key, value in env_vars.items(): + monkeypatch.setenv(key, value) + + assert build_broker_url() == expected_url + + +@pytest.mark.parametrize( + "env_vars, db, expected_url", + [ + ( + { + "REDIS_HOST": "testhost", + "REDIS_PORT": "6379", + "REDIS_PASSWORD": "testpassword", + }, + 2, + "redis://:testpassword@testhost:6379/2", + ), + ( + { + "REDIS_HOST": "testhost", + "REDIS_PORT": "6379", + }, + 1, + "redis://testhost:6379/1", + ), + ( + {"BROKER_URL": "redis://localhost:6379/0"}, + None, + "redis://localhost:6379/0", + ), + ( + {}, + 4, + "redis://localhost:6379/4", + ), + ], +) +def test_build_redis_url(monkeypatch, env_vars, db, expected_url): + """Test building Redis URL.""" + for key in ["REDIS_HOST", "REDIS_PORT", "REDIS_PASSWORD", "BROKER_URL"]: + monkeypatch.delenv(key, raising=False) + for key, value in env_vars.items(): + monkeypatch.setenv(key, value) + + assert build_redis_url(db=db) == expected_url