config: granular env-based solution for connection strings

Add logic to build connection urls from env vars if available
needed for helm charts security best practices. includes:
* Build db uri
* Build redis url
* Build mq url

Partially closes: inveniosoftware/helm-invenio#112

invenio_config/utils.py

@@ -2,6 +2,7 @@
 #
 # This file is part of Invenio.
 # Copyright (C) 2015-2018 CERN.
+# Copyright (C) 2024 KTH Royal Institute of Technology.
 #
 # Invenio is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -10,6 +11,8 @@
 
 from __future__ import absolute_import, print_function
 
+import os
+
 from .default import InvenioConfigDefault
 from .entrypoint import InvenioConfigEntryPointModule
 from .env import InvenioConfigEnvironment
@@ -77,3 +80,97 @@ def create_conf_loader(*args, **kwargs):  # pragma: no cover
         DeprecationWarning,
     )
     return create_config_loader(*args, **kwargs)
+
+
+def _get_env_var(prefix, keys):
+    """Retrieve environment variables with a given prefix."""
+    return {k: os.environ.get(f"{prefix}_{k.upper()}") for k in keys}
+
+
+def build_db_uri():
+    """
+    Build database URI from environment variables or use default.
+
+    Priority order:
+    1. INVENIO_SQLALCHEMY_DATABASE_URI
+    2. SQLALCHEMY_DATABASE_URI
+    3. INVENIO_DB_* specific environment variables
+    4. Default URI
+    """
+    default_uri = "postgresql+psycopg2://invenio-app-rdm:invenio-app-rdm@localhost/invenio-app-rdm"
+
+    uri = os.environ.get("INVENIO_SQLALCHEMY_DATABASE_URI") or os.environ.get(
+        "SQLALCHEMY_DATABASE_URI"
+    )
+    if uri:
+        return uri
+
+    db_params = _get_env_var(
+        "INVENIO_DB", ["user", "password", "host", "port", "name", "protocol"]
+    )
+    if all(db_params.values()):
+        uri = f"{db_params['protocol']}://{db_params['user']}:{db_params['password']}@{db_params['host']}:{db_params['port']}/{db_params['name']}"
+        return uri
+
+    return default_uri
+
+
+def build_broker_url():
+    """
+    Build broker URL from environment variables or use default.
+
+    Priority order:
+    1. INVENIO_BROKER_URL
+    2. BROKER_URL
+    3. INVENIO_BROKER_* specific environment variables
+    4. Default URL
+    """
+    default_url = "amqp://guest:guest@localhost:5672/"
+
+    uri = os.environ.get("INVENIO_BROKER_URL") or os.environ.get("BROKER_URL")
+    if uri:
+        return uri
+
+    broker_params = _get_env_var(
+        "INVENIO_BROKER", ["user", "password", "host", "port", "protocol"]
+    )
+    if all(broker_params.values()):
+        vhost = os.environ.get("INVENIO_BROKER_VHOST", "").lstrip("/") or ""
+        return f"{broker_params['protocol']}://{broker_params['user']}:{broker_params['password']}@{broker_params['host']}:{broker_params['port']}/{vhost}"
+
+    return default_url
+
+
+def build_redis_url(db=None):
+    """
+    Build Redis URL from environment variables or use default.
+
+    Priority order:
+    1. BROKER_URL (Redis-based)
+    2. INVENIO_REDIS_URL
+    3. INVENIO_REDIS_* specific environment variables
+    4. Default URL
+    """
+    db = db if db is not None else 0
+    default_url = f"redis://localhost:6379/{db}"
+
+    uri = os.environ.get("BROKER_URL")
+    if uri and uri.startswith(("redis://", "rediss://", "unix://")):
+        return uri
+
+    uri = os.environ.get("INVENIO_REDIS_URL")
+    if uri:
+        return uri
+
+    redis_params = _get_env_var(
+        "INVENIO_REDIS", ["host", "port", "password", "protocol"]
+    )
+    redis_params["protocol"] = redis_params.get("protocol") or "redis"
+
+    if redis_params["host"] and redis_params["port"]:
+        password = (
+            f":{redis_params['password']}@" if redis_params.get("password") else ""
+        )
+        return f"{redis_params['protocol']}://{password}{redis_params['host']}:{redis_params['port']}/{db}"
+
+    return default_url

tests/test_invenio_config.py

@@ -2,6 +2,7 @@
 #
 # This file is part of Invenio.
 # Copyright (C) 2015-2018 CERN.
+# Copyright (C) 2024 KTH Royal Institute of Technology.
 #
 # Invenio is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -16,6 +17,7 @@
 import warnings
 from os.path import join
 
+import pytest
 from flask import Flask
 from mock import patch
 from pkg_resources import EntryPoint
@@ -29,6 +31,7 @@
     create_config_loader,
 )
 from invenio_config.default import ALLOWED_HTML_ATTRS, ALLOWED_HTML_TAGS
+from invenio_config.utils import build_broker_url, build_db_uri, build_redis_url
 
 
 class ConfigEP(EntryPoint):
@@ -231,3 +234,205 @@ class Config(object):
         assert app.config["ENV"] == "env"
     finally:
         shutil.rmtree(tmppath)
+
+
+@pytest.mark.parametrize(
+    "env_vars, expected_uri",
+    [
+        (
+            {
+                "INVENIO_DB_USER": "testuser",
+                "INVENIO_DB_PASSWORD": "testpassword",
+                "INVENIO_DB_HOST": "testhost",
+                "INVENIO_DB_PORT": "5432",
+                "INVENIO_DB_NAME": "testdb",
+                "INVENIO_DB_PROTOCOL": "postgresql+psycopg2",
+            },
+            "postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb",
+        ),
+        (
+            {"INVENIO_SQLALCHEMY_DATABASE_URI": "sqlite:///test.db"},
+            "sqlite:///test.db",
+        ),
+        (
+            {"SQLALCHEMY_DATABASE_URI": "sqlite:///test.db"},
+            "sqlite:///test.db",
+        ),
+        (
+            {},
+            "postgresql+psycopg2://invenio-app-rdm:invenio-app-rdm@localhost/invenio-app-rdm",
+        ),
+    ],
+)
+def test_build_db_uri(monkeypatch, env_vars, expected_uri):
+    """Test building database URI."""
+    for key in [
+        "INVENIO_DB_USER",
+        "INVENIO_DB_PASSWORD",
+        "INVENIO_DB_HOST",
+        "INVENIO_DB_PORT",
+        "INVENIO_DB_NAME",
+        "INVENIO_DB_PROTOCOL",
+        "INVENIO_SQLALCHEMY_DATABASE_URI",
+        "SQLALCHEMY_DATABASE_URI",
+    ]:
+        monkeypatch.delenv(key, raising=False)
+    for key, value in env_vars.items():
+        monkeypatch.setenv(key, value)
+
+    assert build_db_uri() == expected_uri
+
+
+@pytest.mark.parametrize(
+    "env_vars, expected_url",
+    [
+        (
+            {
+                "INVENIO_BROKER_USER": "testuser",
+                "INVENIO_BROKER_PASSWORD": "testpassword",
+                "INVENIO_BROKER_HOST": "testhost",
+                "INVENIO_BROKER_PORT": "5672",
+                "INVENIO_BROKER_PROTOCOL": "amqp",
+            },
+            "amqp://testuser:testpassword@testhost:5672/",
+        ),
+        (
+            {"INVENIO_BROKER_URL": "amqp://guest:guest@localhost:5672/"},
+            "amqp://guest:guest@localhost:5672/",
+        ),
+        (
+            {"BROKER_URL": "amqp://guest:guest@localhost:5672/"},
+            "amqp://guest:guest@localhost:5672/",
+        ),
+        (
+            {},
+            "amqp://guest:guest@localhost:5672/",
+        ),
+    ],
+)
+def test_build_broker_url(monkeypatch, env_vars, expected_url):
+    """Test building broker URL."""
+    for key in [
+        "INVENIO_BROKER_USER",
+        "INVENIO_BROKER_PASSWORD",
+        "INVENIO_BROKER_HOST",
+        "INVENIO_BROKER_PORT",
+        "INVENIO_BROKER_PROTOCOL",
+        "INVENIO_BROKER_URL",
+        "BROKER_URL",
+    ]:
+        monkeypatch.delenv(key, raising=False)
+    for key, value in env_vars.items():
+        monkeypatch.setenv(key, value)
+
+    assert build_broker_url() == expected_url
+
+
+@pytest.mark.parametrize(
+    "env_vars, expected_url",
+    [
+        (
+            {
+                "INVENIO_BROKER_USER": "testuser",
+                "INVENIO_BROKER_PASSWORD": "testpassword",
+                "INVENIO_BROKER_HOST": "testhost",
+                "INVENIO_BROKER_PORT": "5672",
+                "INVENIO_BROKER_PROTOCOL": "amqp",
+                "INVENIO_BROKER_VHOST": "/testvhost",
+            },
+            "amqp://testuser:testpassword@testhost:5672/testvhost",
+        ),
+        (
+            {
+                "INVENIO_BROKER_USER": "testuser",
+                "INVENIO_BROKER_PASSWORD": "testpassword",
+                "INVENIO_BROKER_HOST": "testhost",
+                "INVENIO_BROKER_PORT": "5672",
+                "INVENIO_BROKER_PROTOCOL": "amqp",
+                "INVENIO_BROKER_VHOST": "testvhost",
+            },
+            "amqp://testuser:testpassword@testhost:5672/testvhost",
+        ),
+        (
+            {"INVENIO_BROKER_URL": "amqp://guest:guest@localhost:5672/"},
+            "amqp://guest:guest@localhost:5672/",
+        ),
+        (
+            {},
+            "amqp://guest:guest@localhost:5672/",
+        ),
+    ],
+)
+def test_build_broker_url_with_vhost(monkeypatch, env_vars, expected_url):
+    """Test building broker URL with vhost."""
+    for key in [
+        "INVENIO_BROKER_USER",
+        "INVENIO_BROKER_PASSWORD",
+        "INVENIO_BROKER_HOST",
+        "INVENIO_BROKER_PORT",
+        "INVENIO_BROKER_PROTOCOL",
+        "INVENIO_BROKER_URL",
+        "INVENIO_BROKER_VHOST",
+        "BROKER_URL",
+    ]:
+        monkeypatch.delenv(key, raising=False)
+    for key, value in env_vars.items():
+        monkeypatch.setenv(key, value)
+
+    assert build_broker_url() == expected_url
+
+
+@pytest.mark.parametrize(
+    "env_vars, db, expected_url",
+    [
+        (
+            {
+                "INVENIO_REDIS_HOST": "testhost",
+                "INVENIO_REDIS_PORT": "6379",
+                "INVENIO_REDIS_PASSWORD": "testpassword",
+                "INVENIO_REDIS_PROTOCOL": "redis",
+            },
+            2,
+            "redis://:testpassword@testhost:6379/2",
+        ),
+        (
+            {
+                "INVENIO_REDIS_HOST": "testhost",
+                "INVENIO_REDIS_PORT": "6379",
+                "INVENIO_REDIS_PROTOCOL": "redis",
+            },
+            1,
+            "redis://testhost:6379/1",
+        ),
+        (
+            {"BROKER_URL": "redis://localhost:6379/0"},
+            None,
+            "redis://localhost:6379/0",
+        ),
+        (
+            {"INVENIO_REDIS_URL": "redis://localhost:6379/3"},
+            3,
+            "redis://localhost:6379/3",
+        ),
+        (
+            {},
+            4,
+            "redis://localhost:6379/4",
+        ),
+    ],
+)
+def test_build_redis_url(monkeypatch, env_vars, db, expected_url):
+    """Test building Redis URL."""
+    for key in [
+        "INVENIO_REDIS_HOST",
+        "INVENIO_REDIS_PORT",
+        "INVENIO_REDIS_PASSWORD",
+        "INVENIO_REDIS_PROTOCOL",
+        "INVENIO_REDIS_URL",
+        "BROKER_URL",
+    ]:
+        monkeypatch.delenv(key, raising=False)
+    for key, value in env_vars.items():
+        monkeypatch.setenv(key, value)
+
+    assert build_redis_url(db=db) == expected_url