From 85bd30dfa7a5ec32f1cd39b3cc27a546ace7f542 Mon Sep 17 00:00:00 2001 From: Ara Ghukasyan <38226926+araghukas@users.noreply.github.com> Date: Fri, 24 Nov 2023 15:07:56 -0500 Subject: [PATCH] Expose programmatic equivalent of CLI commands (#1854) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * export `covalent_start` and `covalent_stop` * check server stopped * update changelog * move commands to main namespace * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * improve docstrings * fix `covalent_is_running` to return bool * reorder `covalent_is_running` conditions * `quiet` mode to suppress stdout; more docstrings * use poll function instead of while loop * explain package * add api docs entry * update api docs * restore import from `._programmatic` * update api docs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add test for new functions * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add test for `covalent_is_running` * removing covalent's dependency on dispatcher * ignore pip reqs in new package * refactor docstrings * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update docs * revert api docs * include 'Returns' in docstrings so maybe docs will render 🤷 pls * remove useless 'Returns' from docstrings 🤦‍♂️ * try autofunction refs to main namespace instead * revert using main namespace refs * add more logging and edit messages * refactor hanging tests * refactor tests into functional tests * Revert "refactor tests into functional tests" This reverts commit f308f8e737d0a43b20eccc848338c2b25e164ff9. * create global var for timeout * use mock start and stop commands * renamed server check function and added import error check tests * None wasn't an acceptable value to redirect_stdout * refactor to use subprocess * refactor as multiple tests w/ patched start/stop * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add nopycln inside new tests * renaming things a bit --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: sankalp --- .github/workflows/requirements.yml | 1 + CHANGELOG.md | 5 + covalent/__init__.py | 5 + covalent/_programmatic/__init__.py | 20 +++ covalent/_programmatic/commands.py | 161 ++++++++++++++++++ doc/source/api/api.rst | 24 +++ tests/covalent_tests/programmatic/__init__.py | 15 ++ .../programmatic/commands_test.py | 139 +++++++++++++++ 8 files changed, 370 insertions(+) create mode 100644 covalent/_programmatic/__init__.py create mode 100644 covalent/_programmatic/commands.py create mode 100644 tests/covalent_tests/programmatic/__init__.py create mode 100644 tests/covalent_tests/programmatic/commands_test.py diff --git a/.github/workflows/requirements.yml b/.github/workflows/requirements.yml index 8d692b0fe..30198b6be 100644 --- a/.github/workflows/requirements.yml +++ b/.github/workflows/requirements.yml @@ -54,6 +54,7 @@ jobs: --ignore-file=covalent/triggers/** --ignore-file=covalent/cloud_resource_manager/** --ignore-file=covalent/quantum/qserver/** + --ignore-file=covalent/_programmatic/** covalent - name: Check missing dispatcher requirements diff --git a/CHANGELOG.md b/CHANGELOG.md index f3710d7e9..7b80bd16a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] + +### Added + +- Programmatic equivalents of CLI commands `covalent start` and `covalent stop` + ### Changed - Changed the azurebatch.rst banner from default covalent jpg to azure batch's svg file diff --git a/covalent/__init__.py b/covalent/__init__.py index aed63a724..524952776 100644 --- a/covalent/__init__.py +++ b/covalent/__init__.py @@ -25,6 +25,11 @@ from ._dispatcher_plugins import local_redispatch as redispatch # nopycln: import from ._dispatcher_plugins import stop_triggers # nopycln: import from ._file_transfer import strategies as fs_strategies # nopycln: import +from ._programmatic.commands import ( # nopycln: import + covalent_start, + covalent_stop, + is_covalent_running, +) from ._results_manager.results_manager import ( # nopycln: import cancel, get_result, diff --git a/covalent/_programmatic/__init__.py b/covalent/_programmatic/__init__.py new file mode 100644 index 000000000..fa08678aa --- /dev/null +++ b/covalent/_programmatic/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +NOTE: This package exists to avoid circular imports that would be encountered if +`covalent` imports from `covalent_dispatcher._cli`. +""" diff --git a/covalent/_programmatic/commands.py b/covalent/_programmatic/commands.py new file mode 100644 index 000000000..2ea5f68d3 --- /dev/null +++ b/covalent/_programmatic/commands.py @@ -0,0 +1,161 @@ +# Copyright 2021 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Functions providing programmatic access to Covalent CLI commands.""" +import subprocess +from typing import List, Optional + +import psutil + +from .._shared_files import logger +from .._shared_files.config import get_config + +__all__ = ["is_covalent_running", "covalent_start", "covalent_stop"] + + +app_log = logger.app_log + +_MISSING_SERVER_WARNING = "Covalent has not been installed with the server component." + + +def _call_cli_command(cmd: List[str], *, quiet: bool = False) -> subprocess.CompletedProcess: + """ + Call a CLI command with the specified kwargs. + + Args: + func: The CLI command to call. + quiet: Suppress stdout. Defaults to :code:`False`. + """ + + if quiet: + return subprocess.run( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + + return subprocess.run(cmd, check=True) + + +def is_covalent_running() -> bool: + """ + Check if the Covalent server is running. + + Returns: + :code:`True` if the Covalent server is in a ready state, :code:`False` otherwise. + """ + try: + from covalent_dispatcher._cli.service import _read_pid + + pid = _read_pid(get_config("dispatcher.cache_dir") + "/ui.pid") + return ( + pid != -1 + and psutil.pid_exists(pid) + and get_config("dispatcher.address") != "" + and get_config("dispatcher.port") != "" + ) + + except ModuleNotFoundError: + # If the covalent_dispatcher is not installed, assume Covalent is not running. + app_log.warning(_MISSING_SERVER_WARNING) + return False + + +def covalent_start( + develop: bool = False, + port: Optional[str] = None, + mem_per_worker: Optional[str] = None, + workers: Optional[int] = None, + threads_per_worker: Optional[int] = None, + ignore_migrations: bool = False, + no_cluster: bool = False, + no_triggers: bool = False, + triggers_only: bool = False, + *, + quiet: bool = False, +) -> None: + """ + Start the Covalent server. Wrapper for the :code:`covalent start` CLI command. + This function returns immediately if the local Covalent server is already running. + + Args: + develop: Start local server in develop mode. Defaults to :code:`False`. + port: Local server port number. Defaults to :code:`"48008"`. + mem_per_worker: Memory limit per worker in GB. Defaults to auto. + workers: Number of Dask workers. Defaults to 8. + threads_per_worker: Number of threads per Dask worker. Defaults to 1. + ignore_migrations: Start server without database migrations. Defaults to :code:`False`. + no_cluster: Start server without Dask cluster. Defaults to :code:`False`. + no_triggers: Start server without a triggers server. Defaults to :code:`False`. + triggers_only: Start only the triggers server. Defaults to :code:`False`. + quiet: Suppress stdout. Defaults to :code:`False`. + """ + + if is_covalent_running(): + msg = "Covalent server is already running." + if not quiet: + print(msg) + + app_log.debug(msg) + return + + flags = { + "--develop": develop, + "--ignore-migrations": ignore_migrations, + "--no-cluster": no_cluster, + "--no-triggers": no_triggers, + "--triggers-only": triggers_only, + } + + args = { + "--port": port or get_config("dispatcher.port"), + "--mem-per-worker": mem_per_worker or get_config("dask.mem_per_worker"), + "--workers": workers or get_config("dask.num_workers"), + "--threads-per-worker": threads_per_worker or get_config("dask.threads_per_worker"), + } + + cmd = ["covalent", "start"] + cmd.extend(flag for flag, value in flags.items() if value) + + for arg, value in args.items(): + cmd.extend((arg, str(value))) + + # Run the `covalent start [OPTIONS]` command. + app_log.debug("Starting Covalent server programmatically...") + _call_cli_command(cmd, quiet=quiet) + + +def covalent_stop(*, quiet: bool = False) -> None: + """ + Stop the Covalent server. Wrapper for the :code:`covalent stop` CLI command. + This function returns immediately if the local Covalent server is not running. + + Args: + quiet: Suppress stdout. Defaults to :code:`False`. + """ + + if not is_covalent_running(): + msg = "Covalent server is not running." + if not quiet: + print(msg) + + app_log.debug(msg) + return + + # Run the `covalent stop` command. + app_log.debug("Stopping Covalent server programmatically...") + _call_cli_command(["covalent", "stop"], quiet=quiet) diff --git a/doc/source/api/api.rst b/doc/source/api/api.rst index 2a603c728..01b694676 100644 --- a/doc/source/api/api.rst +++ b/doc/source/api/api.rst @@ -6,6 +6,7 @@ Covalent API The following API documentation describes how to use Covalent. +- The :ref:`covalent_server` manages workflow dispatch, orchestration, and metadata - :ref:`electrons_api` and :ref:`lattices_api` are used for constructing workflows - :ref:`qelectrons_api` are used to customize and track quantum circuit execution - :ref:`qclusters_api` are used to distribute Quantum Electrons across multiple quantum backends. @@ -22,6 +23,29 @@ The following API documentation describes how to use Covalent. - :ref:`dispatcher_interface` is used for dispatching workflows and stopping triggered dispatches - The :ref:`dispatcher_server_api` is used for interfacing with the Covalent server +.. _covalent_server: + +Covalent Server +""""""""""""""""""""""""""" +A Covalent server must be running in order to dispatch workflows. The Covalent CLI provides various utilities for starting, stopping, and managing a Covalent server. For more information, see: + +.. code-block:: bash + + covalent --help + +The Covalent SDK also includes a Python interface for starting and stopping the Covalent server. + +.. autofunction:: covalent._programmatic.commands.is_covalent_running + + +.. autofunction:: covalent._programmatic.commands.covalent_start + + +.. autofunction:: covalent._programmatic.commands.covalent_stop + + +---------------------------------------------------------------- + .. _electrons_api: Electron diff --git a/tests/covalent_tests/programmatic/__init__.py b/tests/covalent_tests/programmatic/__init__.py new file mode 100644 index 000000000..cfc23bfdf --- /dev/null +++ b/tests/covalent_tests/programmatic/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/covalent_tests/programmatic/commands_test.py b/tests/covalent_tests/programmatic/commands_test.py new file mode 100644 index 000000000..324984113 --- /dev/null +++ b/tests/covalent_tests/programmatic/commands_test.py @@ -0,0 +1,139 @@ +# Copyright 2021 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import covalent as ct +from covalent._programmatic.commands import _MISSING_SERVER_WARNING + + +def test_is_covalent_running(mocker): + """Check that `is_covalent_running` returns True when the server is running.""" + try: + from covalent_dispatcher._cli.service import _read_pid # nopycln: import + except ModuleNotFoundError: + pytest.xfail("`covalent_dispatcher` not installed") + + # Simulate server running + mocker.patch("covalent_dispatcher._cli.service._read_pid", return_value=1) + mocker.patch("psutil.pid_exists", return_value=True) + mocker.patch( + "covalent._shared_files.config.get_config", + return_value={"port": 48008, "host": "localhost"}, + ) + assert ct.is_covalent_running() + + # Simulate server stopped + mocker.patch("covalent_dispatcher._cli.service._read_pid", return_value=-1) + assert not ct.is_covalent_running() + + +def test_is_covalent_running_import_error(mocker): + """Check that `is_covalent_running` catches the `ModuleNotFoundError`.""" + try: + from covalent_dispatcher._cli.service import _read_pid # nopycln: import + except ModuleNotFoundError: + pytest.xfail("`covalent_dispatcher` not installed") + + mocker.patch( + "covalent_dispatcher._cli.service._read_pid", + side_effect=ModuleNotFoundError(), + ) + + mock_app_log = mocker.patch("covalent._programmatic.commands.app_log") + + assert not ct.is_covalent_running() + mock_app_log.warning.assert_called_once_with(_MISSING_SERVER_WARNING) + + +def test_covalent_start(mocker): + """Test the `covalent_start` function without actually starting server.""" + + mocker.patch("subprocess.run") + + # Simulate server running + mocker.patch( + "covalent._programmatic.commands.is_covalent_running", + return_value=True, + ) + ct.covalent_start() + + # Simulate server stopped + mocker.patch( + "covalent._programmatic.commands.is_covalent_running", + return_value=False, + ) + ct.covalent_start() + + +def test_covalent_start_quiet(mocker): + """Test the `covalent_start` function without actually starting server.""" + + mocker.patch("subprocess.run") + + # Simulate server running + mocker.patch( + "covalent._programmatic.commands.is_covalent_running", + return_value=True, + ) + ct.covalent_start(quiet=True) + + # Simulate server stopped + mocker.patch( + "covalent._programmatic.commands.is_covalent_running", + return_value=False, + ) + ct.covalent_start(quiet=True) + + +def test_covalent_stop(mocker): + """Test the `covalent_start` function without actually starting server.""" + + mocker.patch("subprocess.run") + + # Simulate server running + mocker.patch( + "covalent._programmatic.commands.is_covalent_running", + return_value=True, + ) + ct.covalent_stop() + + # Simulate server stopped + mocker.patch( + "covalent._programmatic.commands.is_covalent_running", + return_value=False, + ) + ct.covalent_stop() + + +def test_covalent_stop_quiet(mocker): + """Test the `covalent_start` function without actually starting server.""" + + mocker.patch("subprocess.run") + + # Simulate server running + mocker.patch( + "covalent._programmatic.commands.is_covalent_running", + return_value=True, + ) + ct.covalent_stop(quiet=True) + + # Simulate server stopped + mocker.patch( + "covalent._programmatic.commands.is_covalent_running", + return_value=False, + ) + ct.covalent_stop(quiet=True)