diff --git a/admin/local/docker/compose/docker-compose.yml b/admin/local/docker/compose/docker-compose.yml index d4dba8bfd0..8eb7cc4a3b 100644 --- a/admin/local/docker/compose/docker-compose.yml +++ b/admin/local/docker/compose/docker-compose.yml @@ -92,7 +92,7 @@ services: entrypoint worker-xrootd --db-uri mysql://qsmaster@127.0.0.1:3306 --db-admin-uri mysql://root:CHANGEME@127.0.0.1:3306 - --vnid-config "@/usr/local/lib64/libreplica.so mysql://qsmaster@127.0.0.1:3306/qservw_worker 0 0" + --vnid-config "@/usr/local/lib64/libreplica.so {{db_uri}}/qservw_worker 0 0" --cmsd-manager-name manager-xrootd --cmsd-manager-count 1 --mysql-monitor-password CHANGEME_MONITOR @@ -171,12 +171,13 @@ services: << : *worker-xrootd command: > entrypoint --log-level DEBUG worker-xrootd - --db-uri mysql://qsmaster@127.0.0.1:3306?socket=/qserv/mariadb/run/mysqld.sock - --db-admin-uri mysql://root:CHANGEME@127.0.0.1:3306?socket=/qserv/mariadb/run/mysqld.sock + --db-uri mysql://qsmaster@127.0.0.1:3306?socket={{db_socket}} + --db-admin-uri mysql://root:CHANGEME@127.0.0.1:3306?socket={{db_socket}} --vnid-config "@/usr/local/lib64/libreplica.so mysql://root:CHANGEME@localhost:3306/qservw_worker 0 0" --cmsd-manager-name manager-xrootd --cmsd-manager-count 1 --mysql-monitor-password CHANGEME_MONITOR + --targs db_socket=/qserv/mariadb/run/mysqld.sock environment: << : *log-environment volumes: @@ -306,8 +307,9 @@ services: init: true command: > entrypoint --log-level DEBUG proxy - --db-uri mysql://qsmaster@127.0.0.1:3306?socket=/qserv/mariadb/run/mysqld.sock - --db-admin-uri mysql://root:CHANGEME@127.0.0.1:3306?socket=/qserv/mariadb/run/mysqld.sock + --db-uri mysql://qsmaster@127.0.0.1:3306?socket={{db_socket}} + --db-admin-uri mysql://root:CHANGEME@127.0.0.1:3306?socket={{db_socket}} + --targs db_socket=/qserv/mariadb/run/mysqld.sock --xrootd-manager manager-xrootd environment: << : *log-environment diff --git a/src/admin/python/lsst/qserv/admin/cli/entrypoint.py b/src/admin/python/lsst/qserv/admin/cli/entrypoint.py index 6d098f903c..1e31a0ed0f 100644 --- a/src/admin/python/lsst/qserv/admin/cli/entrypoint.py +++ b/src/admin/python/lsst/qserv/admin/cli/entrypoint.py @@ -30,7 +30,7 @@ import logging import os import sys -from typing import Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional from click.decorators import pass_context @@ -65,6 +65,7 @@ xrootd_manager_option, ) from . import utils +from .render_targs import render_targs from . import script from ..watcher import watch @@ -426,35 +427,22 @@ def delete_database(repl_ctrl_uri: str, database: str, admin: bool) -> None: @targs_options() @cmd_options() @options_file_option() -def proxy( - ctx: click.Context, - db_uri: str, - db_admin_uri: str, - mysql_monitor_password: str, - repl_ctl_dn: str, - xrootd_manager: str, - proxy_backend_address: str, - proxy_cfg_file: str, - proxy_cfg_path: str, - czar_cfg_file: str, - czar_cfg_path: str, - targs: Dict[str, str], - targs_file: str, - cmd: str, -) -> None: +def proxy(ctx: click.Context, **kwargs: Any) -> None: """Start as a qserv-proxy node. """ + targs = utils.targs(ctx) + targs = render_targs(targs) script.enter_proxy( - targs=utils.targs(ctx), - db_uri=db_uri, - db_admin_uri=db_admin_uri, - repl_ctl_dn=repl_ctl_dn, - proxy_backend_address=proxy_backend_address, - proxy_cfg_file=proxy_cfg_file, - proxy_cfg_path=proxy_cfg_path, - czar_cfg_file=czar_cfg_file, - czar_cfg_path=czar_cfg_path, - cmd=cmd, + targs=targs, + db_uri=targs["db_uri"], + db_admin_uri=targs["db_admin_uri"], + repl_ctl_dn=targs["repl_ctl_dn"], + proxy_backend_address=targs["proxy_backend_address"], + proxy_cfg_file=targs["proxy_cfg_file"], + proxy_cfg_path=targs["proxy_cfg_path"], + czar_cfg_file=targs["czar_cfg_file"], + czar_cfg_path=targs["czar_cfg_path"], + cmd=targs["cmd"], ) @@ -479,22 +467,16 @@ def proxy( @targs_options() @cmd_options() @options_file_option() -def cmsd_manager( - ctx: click.Context, - cms_delay_servers: int, - cmsd_manager_cfg_file: str, - cmsd_manager_cfg_path: str, - targs: Dict[str, str], - targs_file: str, - cmd: str, -) -> None: +def cmsd_manager(ctx: click.Context, **kwargs: Any) -> None: """Start as a cmsd manager node. """ + targs = utils.targs(ctx) + targs = render_targs(targs) script.enter_manager_cmsd( - targs=utils.targs(ctx), - cmsd_manager_cfg_file=cmsd_manager_cfg_file, - cmsd_manager_cfg_path=cmsd_manager_cfg_path, - cmd=cmd, + targs=targs, + cmsd_manager_cfg_file=targs["cmsd_manager_cfg_file"], + cmsd_manager_cfg_path=targs["cmsd_manager_cfg_path"], + cmd=targs["cmd"], ) @@ -517,23 +499,16 @@ def cmsd_manager( @targs_options() @cmd_options() @options_file_option() -def xrootd_manager( - ctx: click.Context, - cmsd_manager_name: str, - cmsd_manager_count: str, - xrootd_manager_cfg_file: str, - xrootd_manager_cfg_path: str, - targs: Dict[str, str], - targs_file: str, - cmd: str, -) -> None: +def xrootd_manager(ctx: click.Context, **kwargs: Any) -> None: """Start as an xrootd manager node. """ + targs = utils.targs(ctx) + targs = render_targs(targs) script.enter_xrootd_manager( - targs=utils.targs(ctx), - xrootd_manager_cfg_file=xrootd_manager_cfg_file, - xrootd_manager_cfg_path=xrootd_manager_cfg_path, - cmd=cmd, + targs=targs, + xrootd_manager_cfg_file=targs["xrootd_manager_cfg_file"], + xrootd_manager_cfg_path=targs["xrootd_manager_cfg_path"], + cmd=targs["cmd"], ) @@ -551,30 +526,18 @@ def xrootd_manager( @targs_options() @cmd_options() @options_file_option() -def worker_cmsd( - ctx: click.Context, - cmsd_manager_name: str, - cmsd_manager_count: str, - vnid_config: str, - debug_port: Optional[int], - db_uri: str, - cmsd_worker_cfg_file: str, - cmsd_worker_cfg_path: str, - xrdssi_cfg_file: str, - xrdssi_cfg_path: str, - targs: Dict[str, str], - targs_file: str, - cmd: str, -) -> None: +def worker_cmsd(ctx: click.Context, **kwargs: Any) -> None: + targs = utils.targs(ctx) + targs = render_targs(targs) script.enter_worker_cmsd( - targs=utils.targs(ctx), - debug_port=debug_port, - db_uri=db_uri, - cmsd_worker_cfg_file=cmsd_worker_cfg_file, - cmsd_worker_cfg_path=cmsd_worker_cfg_path, - xrdssi_cfg_file=xrdssi_cfg_file, - xrdssi_cfg_path=xrdssi_cfg_path, - cmd=cmd, + targs=targs, + debug_port=targs["debug_port"], + db_uri=targs["db_uri"], + cmsd_worker_cfg_file=targs["cmsd_worker_cfg_file"], + cmsd_worker_cfg_path=targs["cmsd_worker_cfg_path"], + xrdssi_cfg_file=targs["xrdssi_cfg_file"], + xrdssi_cfg_path=targs["xrdssi_cfg_path"], + cmd=targs["cmd"], ) @@ -596,39 +559,23 @@ def worker_cmsd( @targs_options() @cmd_options() @options_file_option() -def worker_xrootd( - ctx: click.Context, - debug_port: Optional[int], - db_uri: str, - db_admin_uri: str, - vnid_config: str, - cmsd_manager_name: str, - cmsd_manager_count: int, - repl_ctl_dn: str, - mysql_monitor_password: str, - db_qserv_user: str, - cmsd_worker_cfg_file: str, - cmsd_worker_cfg_path: str, - xrdssi_cfg_file: str, - xrdssi_cfg_path: str, - targs: Dict[str, str], - targs_file: str, - cmd: str, -) -> None: +def worker_xrootd(ctx: click.Context, **kwargs: Any) -> None: + targs = utils.targs(ctx) + targs = render_targs(targs) script.enter_worker_xrootd( - targs=utils.targs(ctx), - debug_port=debug_port, - db_uri=db_uri, - db_admin_uri=db_admin_uri, - vnid_config=vnid_config, - repl_ctl_dn=repl_ctl_dn, - mysql_monitor_password=mysql_monitor_password, - db_qserv_user=db_qserv_user, - cmsd_worker_cfg_file=cmsd_worker_cfg_file, - cmsd_worker_cfg_path=cmsd_worker_cfg_path, - xrdssi_cfg_file=xrdssi_cfg_file, - xrdssi_cfg_path=xrdssi_cfg_path, - cmd=cmd, + targs=targs, + debug_port=targs["debug_port"], + db_uri=targs["db_uri"], + db_admin_uri=targs["db_admin_uri"], + vnid_config=targs["vnid_config"], + repl_ctl_dn=targs["repl_ctl_dn"], + mysql_monitor_password=targs["mysql_monitor_password"], + db_qserv_user=targs["db_qserv_user"], + cmsd_worker_cfg_file=targs["cmsd_worker_cfg_file"], + cmsd_worker_cfg_path=targs["cmsd_worker_cfg_path"], + xrdssi_cfg_file=targs["xrdssi_cfg_file"], + xrdssi_cfg_path=targs["xrdssi_cfg_path"], + cmd=targs["cmd"], ) @@ -654,24 +601,15 @@ def worker_xrootd( @targs_options() @run_option() @options_file_option() -def worker_repl( - ctx: click.Context, - db_admin_uri: str, - repl_connection: str, - debug_port: Optional[int], - cmd: str, - config: str, - targs: Dict[str, str], - targs_file: str, - run: bool, -) -> None: +def worker_repl(ctx: click.Context, **kwargs: Any) -> None: + targs = utils.targs(ctx) + targs = render_targs(targs) script.enter_worker_repl( - targs=utils.targs(ctx), - db_admin_uri=db_admin_uri, - repl_connection=repl_connection, - debug_port=debug_port, - cmd=cmd, - run=run, + db_admin_uri=targs["db_admin_uri"], + repl_connection=targs["repl_connection"], + debug_port=targs["debug_port"], + cmd=targs["cmd"], + run=targs["run"], ) @@ -718,29 +656,17 @@ def worker_repl( @targs_options() @run_option() @options_file_option() -def replication_controller( - ctx: click.Context, - db_uri: str, - db_admin_uri: str, - workers: List[str], - xrootd_manager: str, - log_cfg_file: str, - cmd: str, - http_root: str, - qserv_czar_db: str, - targs: Dict[str, str], - targs_file: str, - run: bool, -) -> None: +def replication_controller(ctx: click.Context, **kwargs: Any) -> None: """Start as a replication controller node.""" + targs = utils.targs(ctx) + targs = render_targs(targs) script.enter_replication_controller( - targs=utils.targs(ctx), - db_uri=db_uri, - db_admin_uri=db_admin_uri, - workers=workers, - log_cfg_file=log_cfg_file, - cmd=cmd, - run=run, + db_uri=targs["db_uri"], + db_admin_uri=targs["db_admin_uri"], + workers=targs["workers"], + log_cfg_file=targs["log_cfg_file"], + cmd=targs["cmd"], + run=targs["run"], ) diff --git a/src/admin/python/lsst/qserv/admin/cli/render_targs.py b/src/admin/python/lsst/qserv/admin/cli/render_targs.py new file mode 100644 index 0000000000..f1d710f79c --- /dev/null +++ b/src/admin/python/lsst/qserv/admin/cli/render_targs.py @@ -0,0 +1,81 @@ +# This file is part of qserv. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + + +"""This module contains functions for rendering jinja template +values in values passed into the `entrypoint` command line.""" + + +from copy import copy +import jinja2 + +from .utils import Targs + + +class UnresolvableTemplate(RuntimeError): + """Exception class used by `render` when a template value can not be + resolved.""" + pass + + +def render_targs(targs: Targs) -> Targs: + """Go through a dict whose values may contain jinja templates that are + other keys in the dict, and resolve the values. + + Will raise if any template value(s) can not be resolved. Causes include: + * the key is not present + * there is a circular reference where 2 or more keys refer to eachother. + + Parameters + ---------- + targs : + The dict to resolve. + + Returns + ------- + resolved_targs : + The dict, resolved. + + Raises + ------ + UnresolvableTemplate + If a value in the template can not be resolved. + """ + rendered = copy(targs) + while True: + changed = False + for k, v in rendered.items(): + if not isinstance(v, str): + continue + if "{{" in v: + t = jinja2.Template(v, undefined=jinja2.StrictUndefined) + try: + rendered[k] = t.render(rendered) + changed = True + except jinja2.exceptions.UndefinedError as e: + raise UnresolvableTemplate(f"Missing template value: {str(e)}") + if not changed: + break + if any([isinstance(v, str) and "{{" in v for v in rendered.values()]): + raise UnresolvableTemplate( + "Could not resolve inputs: " + f"{', '.join([f'{k}={targs[k]}' for k, v in rendered.items() if '{{' in v])}" + ) + return rendered diff --git a/src/admin/python/lsst/qserv/admin/cli/script.py b/src/admin/python/lsst/qserv/admin/cli/script.py index b4f0ca963d..dd734fff72 100644 --- a/src/admin/python/lsst/qserv/admin/cli/script.py +++ b/src/admin/python/lsst/qserv/admin/cli/script.py @@ -308,7 +308,7 @@ def enter_manager_cmsd( The jinja2 template for the command for this function to execute. """ apply_template_cfg_file(cmsd_manager_cfg_file, cmsd_manager_cfg_path, targs) - sys.exit(_run(args=None, template=cmd, targs=targs)) + sys.exit(_run(args=None, cmd=cmd)) def enter_xrootd_manager( @@ -331,7 +331,7 @@ def enter_xrootd_manager( The jinja2 template for the command for this function to execute. """ apply_template_cfg_file(xrootd_manager_cfg_file, xrootd_manager_cfg_path, targs) - sys.exit(_run(args=None, template=cmd, targs=targs)) + sys.exit(_run(args=None, cmd=cmd)) def enter_worker_cmsd( @@ -386,7 +386,7 @@ def enter_worker_cmsd( # for the vnid plugin to function correctly _do_smig_block(worker_smig_dir, "worker", db_uri) - sys.exit(_run(args=None, template=cmd, targs=targs)) + sys.exit(_run(args=None, cmd=cmd)) def enter_worker_xrootd( @@ -475,11 +475,10 @@ def enter_worker_xrootd( apply_template_cfg_file(cmsd_worker_cfg_file, cmsd_worker_cfg_path) apply_template_cfg_file(xrdssi_cfg_file, xrdssi_cfg_path) - sys.exit(_run(args=None, template=cmd, targs=targs)) + sys.exit(_run(args=None, cmd=cmd)) def enter_worker_repl( - targs: Targs, db_admin_uri: str, repl_connection: str, debug_port: Optional[int], @@ -490,8 +489,6 @@ def enter_worker_repl( Parameters ---------- - targs : Targs - The arguments for template expansion. replic_worker_args : `list` [ `str` ] A list of options and arguments that will be passed directly to the replica worker app. @@ -549,7 +546,7 @@ def enter_worker_repl( # qserv-replica-worker returned then by definition it failed, and we # just wait a moment and restart it. # This is recorded in DM-31252 - _run(args=None, template=cmd, targs=targs, run=run) + _run(args=None, cmd=cmd, run=run) _log.info("qserv-replica-worker exited. waiting 5 seconds and restarting.") time.sleep(5) @@ -633,11 +630,10 @@ def enter_proxy( env = dict(os.environ, QSERV_CONFIG=czar_cfg_path) - sys.exit(_run(args=None, template=cmd, env=env, targs=targs)) + sys.exit(_run(args=None, cmd=cmd, env=env)) def enter_replication_controller( - targs: Targs, db_uri: str, db_admin_uri: str, workers: List[str], @@ -649,8 +645,6 @@ def enter_replication_controller( Parameters ---------- - targs : Targs - The arguments for template expansion. db_uri : `str` The connection string for the replication manager database for the non-admin user (created using the `connection`), the user is typically @@ -754,7 +748,7 @@ def set_initial_configuration(workers: Sequence[str]) -> None: ) env = dict(os.environ, LSST_LOG_CONFIG=log_cfg_file) - sys.exit(_run(args=None, template=cmd, targs=targs, env=env, run=run)) + sys.exit(_run(args=None, cmd=cmd, env=env, run=run)) def smig_update(czar_connection: str, worker_connections: List[str], repl_connection: str) -> None: @@ -782,8 +776,7 @@ def smig_update(czar_connection: str, worker_connections: List[str], repl_connec def _run( args: Optional[Sequence[Union[str, int]]], - template: str = None, - targs: Targs = None, + cmd: str = None, env: Dict[str, str] = None, debug_port: Optional[int] = None, run: bool = True, @@ -794,12 +787,9 @@ def _run( Parameters ---------- args : List[Union[str, int]] - The command and arguments to the command. Mutually exclusive with `template`. - template : str, optional - The command and arguments in jinja template form. Mutually exclusive with `args`. - tvars: Dict[str, str], optional - The values for the `template`. If `template` is passed in, must not be `None`. - Mutually exclusive with `args`. + The command and arguments to the command. Mutually exclusive with `cmd`. + cmd : str, optional + The command and arguments to run, in the form of a string. env : Dict[str, str], optional The environment variables to run the command with, by default None which uses the same environment as the current shell. @@ -823,10 +813,10 @@ def _run( exit_code : `int` The exit code of the command that was run. """ - if args and (template is not None or targs is not None): - raise RuntimeError("Invalid use of `args` and `template` or `targs`.") - if template is not None and args is not None: - raise RuntimeError("If `template` is not `None`, `args` must not be `None`.") + if args and cmd: + raise RuntimeError("Invalid use of `args` and `cmd`.") + if cmd is not None and args is not None: + raise RuntimeError("If `cmd` is not `None`, `args` must not be `None`.") if args: str_args = [str(a) for a in args] if debug_port: @@ -835,17 +825,8 @@ def _run( print(" ".join(str_args)) return 0 result = subprocess.run(str_args, env=env, cwd="/home/qserv") - if template: - rendered = template - while "{{" in rendered: - t = jinja2.Template(rendered, undefined=jinja2.StrictUndefined) - try: - rendered = t.render(targs) - except jinja2.exceptions.UndefinedError as e: - _log.error(f"Missing template value: {str(e)}") - raise - args = shlex.split(rendered) - _log.debug("calling subprocess with args: %s", args) # TEMP security don't check in + if cmd: + args = shlex.split(cmd) result = subprocess.run(args, env=env, cwd="/home/qserv") if check_returncode: result.check_returncode @@ -972,5 +953,5 @@ def spawned_app_help( """ app = cmd.split()[0] print(f"Help for '{app}':\n", flush=True) - _run(template=f"{app} --help", args=None) + _run(cmd=f"{app} --help", args=None) sys.exit(0) diff --git a/src/admin/python/lsst/qserv/admin/cli/utils.py b/src/admin/python/lsst/qserv/admin/cli/utils.py index 2825148fd3..4505c17398 100644 --- a/src/admin/python/lsst/qserv/admin/cli/utils.py +++ b/src/admin/python/lsst/qserv/admin/cli/utils.py @@ -27,14 +27,14 @@ import logging import os import traceback -from typing import cast, Dict, List, Sequence, Tuple, Union +from typing import Any, cast, Dict, List, Sequence, Tuple, Union import yaml _log = logging.getLogger(__name__) -Targs = Dict[str, Union[str, Sequence[str]]] +Targs = Dict[str, Any] def split_kv(values: Sequence[str]) -> Dict[str, str]: diff --git a/src/admin/tests/test_render_targs.py b/src/admin/tests/test_render_targs.py new file mode 100644 index 0000000000..b6b5117e87 --- /dev/null +++ b/src/admin/tests/test_render_targs.py @@ -0,0 +1,83 @@ +# This file is part of qserv. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + + +import unittest + +from lsst.qserv.admin.cli.render_targs import render_targs, UnresolvableTemplate + + +class RenderTargsTestCase(unittest.TestCase): + + def testMutualReference(self): + """Test for failure when targs refer directly to each other.""" + with self.assertRaises(UnresolvableTemplate) as r: + render_targs({"a": "{{b}}", "b": "{{a}}"}) + self.assertIn("a={{b}}, b={{a}}", str(r.exception)) + + def testCircularReference(self): + """Test for failure when there is a circular reference in targs.""" + with self.assertRaises(UnresolvableTemplate) as r: + render_targs({"a": "{{b}}", "b": "{{c}}", "c": "{{a}}"}) + self.assertIn("a={{b}}, b={{c}}, c={{a}}", str(r.exception)) + + def testSelfReference(self): + """Test for failure when a targ refers to itself.""" + with self.assertRaises(UnresolvableTemplate) as r: + render_targs({"a": "{{a}}"}) + self.assertIn("a={{a}}", str(r.exception)) + + def testMissingValue(self): + """Test for failure when a template value is not provided.""" + with self.assertRaises(UnresolvableTemplate) as r: + render_targs({"a": "{{b}}"}) + self.assertIn("Missing template value:", str(r.exception)) + + def testList(self): + """Verify that lists can be used as values and manipulated by the template.""" + self.assertEqual( + render_targs({"a": "{{b|join(' ')}}", "b": ["foo", "bar"]}), + {"a": "foo bar", "b": ["foo", "bar"]} + ) + + def testResolves(self): + """Verify that a dict with legal values resolves correctly.""" + self.assertEqual( + render_targs({"a": "{{b}}", "b": "{{c}}", "c": "d"}), + {"a": "d", "b": "d", "c": "d"} + ) + + def testNone(self): + """Test that a dict with None as a value resolves correctly.""" + self.assertEqual( + render_targs({"a": None}), + {"a": None} + ) + + def testBool(self): + """Test that a dict with a bool as a value resolves correctly.""" + self.assertEqual( + render_targs({"a": True}), + {"a": True} + ) + + +if __name__ == "__main__": + unittest.main()