From 8d22e7d1b3c6078fb7fe1d63939ad064f7dcc04c Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Thu, 25 Jan 2024 20:09:54 -0500 Subject: [PATCH 01/12] Enabling qelectron tests and making qelectron opt-in only (#1916) * Adding qelectron tests to the suite * disabling qelectron tests as they are not working currently * making qelectrons a feature users can opt out of * fixed basic dispatching * fixing boilerplate and requirements workflows * fixing boilerplate and requirements workflows * fixing tests * fixing tests and workflows * fixing requirements * fixing requirements * Made qelectron an opt-in feature * minor import fix * fixing tests * fixing tests * added executor back to init file --- .github/workflows/boilerplate.yml | 19 +++++- .github/workflows/requirements.yml | 6 +- .github/workflows/tests.yml | 19 ++++++ .gitignore | 1 + CHANGELOG.md | 11 ++++ MANIFEST.in | 1 + covalent/__init__.py | 7 ++- covalent/_shared_files/qelectron_utils.py | 58 +++++++++++++++++- covalent/_shared_files/qresult_utils.py | 2 +- covalent/_shared_files/utils.py | 59 +++---------------- covalent/_workflow/qelectron.py | 2 +- covalent/_workflow/qnode.py | 2 +- covalent/executor/__init__.py | 12 ++-- .../qiskit_plugin/qiskit_plugin.py | 2 +- covalent/executor/utils/wrappers.py | 2 +- covalent/quantum/qclient/local_client.py | 2 +- covalent/quantum/qcluster/clusters.py | 2 +- covalent/quantum/qserver/core.py | 4 +- covalent_dispatcher/_cli/service.py | 5 ++ covalent_ui/api/v1/data_layer/electron_dal.py | 19 +++--- requirements-qelectron.txt | 4 ++ requirements.txt | 4 -- setup.py | 6 ++ .../shared_files/qelectron_utils_test.py | 2 +- tests/qelectron_tests/core_tests/__init__.py | 15 +++++ .../{ => core_tests}/test_decorator.py | 2 +- .../{ => core_tests}/test_qelectron_db.py | 2 +- .../{ => core_tests}/test_run_later.py | 1 - .../pennylane_tests/conftest.py | 2 +- .../qelectron_tests/plugin_tests/__init__.py | 15 +++++ .../{ => plugin_tests}/test_braket_plugin.py | 0 .../{ => plugin_tests}/test_qiskit_plugin.py | 2 +- .../test_qiskit_plugin_runtime.py | 2 +- tests/requirements.txt | 1 + 34 files changed, 206 insertions(+), 87 deletions(-) create mode 100644 requirements-qelectron.txt create mode 100644 tests/qelectron_tests/core_tests/__init__.py rename tests/qelectron_tests/{ => core_tests}/test_decorator.py (98%) rename tests/qelectron_tests/{ => core_tests}/test_qelectron_db.py (96%) rename tests/qelectron_tests/{ => core_tests}/test_run_later.py (97%) create mode 100644 tests/qelectron_tests/plugin_tests/__init__.py rename tests/qelectron_tests/{ => plugin_tests}/test_braket_plugin.py (100%) rename tests/qelectron_tests/{ => plugin_tests}/test_qiskit_plugin.py (98%) rename tests/qelectron_tests/{ => plugin_tests}/test_qiskit_plugin_runtime.py (98%) diff --git a/.github/workflows/boilerplate.yml b/.github/workflows/boilerplate.yml index 13a2f111a..862b33f87 100644 --- a/.github/workflows/boilerplate.yml +++ b/.github/workflows/boilerplate.yml @@ -85,9 +85,26 @@ jobs: # See the License for the specific language governing permissions and # limitations under the License. + boilerplate2024: |- + # Copyright 2024 Agnostiq Inc. + # + # This file is part of Covalent. + # + # Licensed under the Apache License 2.0 (the "License"). A copy of the + # License may be obtained with this software package or at + # + # https://www.apache.org/licenses/LICENSE-2.0 + # + # Use of this file is prohibited except in compliance with the License. + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + run: | for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - if [[ ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2021" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2022" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2023" ]] ; then + if [[ ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2021" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2022" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2023" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2024" ]] ; then printf "Boilerplate is missing from $file.\n" printf "The first 15 lines of $file are\n\n" cat $file | tr -d '\r' | cat -ET | head -n 15 diff --git a/.github/workflows/requirements.yml b/.github/workflows/requirements.yml index 30198b6be..4502b183f 100644 --- a/.github/workflows/requirements.yml +++ b/.github/workflows/requirements.yml @@ -50,10 +50,14 @@ jobs: --ignore-module=pkg_resources --ignore-module=covalent/_dispatcher_plugins --ignore-module=covalent/_shared_files + --ignore-file=covalent/quantum/** + --ignore-file=covalent/_workflow/q* + --ignore-file=covalent/_shared_files/q* + --ignore-file=covalent/_results_manager/q* + --ignore-file=covalent/_shared_files/pickling.py --ignore-file=covalent/executor/** --ignore-file=covalent/triggers/** --ignore-file=covalent/cloud_resource_manager/** - --ignore-file=covalent/quantum/qserver/** --ignore-file=covalent/_programmatic/** covalent diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 946e343f5..710655301 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -90,6 +90,11 @@ jobs: sdk: - 'covalent/**' - 'tests/covalent_tests/**' + qelectron: + - 'covalent/executor/quantum_plugins/**' + - 'covalent/executor/qbase.py' + - 'covalent/quantum/**' + - 'tests/qelectron_tests/**' dispatcher: - 'covalent_dispatcher/**' - 'tests/covalent_dispatcher_tests/**' @@ -134,6 +139,7 @@ jobs: echo "NEED_PYTHON=$NEED_PYTHON" >> $GITHUB_ENV echo "NEED_FRONTEND=$NEED_FRONTEND" >> $GITHUB_ENV echo "BUILD_AND_RUN_ALL=$BUILD_AND_RUN_ALL" >> $GITHUB_ENV + echo "COVALENT_DISABLE_QELECTRON_TESTS=true" >> $GITHUB_ENV - name: Set up Python if: > @@ -159,6 +165,7 @@ jobs: run: | pip install --no-cache-dir -r ./requirements.txt pip install --no-cache-dir -r ./tests/requirements.txt + pip install --no-cache-dir -r ./requirements-qelectron.txt - name: Set up Node if: env.NEED_FRONTEND || env.BUILD_AND_RUN_ALL @@ -252,6 +259,18 @@ jobs: if: steps.sdk-tests.outcome == 'success' run: coverage xml -o sdk_coverage.xml + - name: Run Qelectron tests and measure coverage + id: qelectron-tests + if: > + (steps.modified-files.outputs.qelectron == 'true' + || env.BUILD_AND_RUN_ALL) && env.COVALENT_DISABLE_QELECTRON_TESTS != 'true' + run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/qelectron_tests/core_tests --cov=covalent_qelectron --cov-config=.coveragerc + + - name: Generate Qelectron coverage report + id: qelectron-coverage + if: steps.qelectron-tests.outcome == 'success' && env.COVALENT_DISABLE_QELECTRON_TESTS != 'true' + run: coverage xml -o qelectron_coverage.xml + - name: Run dispatcher tests and measure coverage id: dispatcher-tests if: > diff --git a/.gitignore b/.gitignore index 4e5d125f3..9e53722e0 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ !pyproject.toml !requirements.txt !requirements-client.txt +!requirements-qelectron.txt !setup.py # Allow markdown etc diff --git a/CHANGELOG.md b/CHANGELOG.md index 0df4c8bf1..474352138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Added + +- Added `pennylane` as a requirement in tests due to the tutorials using it + ### Changed - Updated RTD notebooks to fix their behavior - Changed the error being shown when drawing the transport graph of a lattice to a debug message instead - Revamped README +- Reorganized `qelectron` tests +- Made qelectron an opt-in feature using `covalent[quantum]` extra ### Removed @@ -26,8 +32,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed the scenario where any deploy commands would fail if the user had a non deploy compatible plugin installed +- Fixed the SQLAlchemy warning that used to show up at every fresh server start - Fixed deploy commands' default value of plugins not being propagated to the tfvars file +### Operations + +- Added qelectron tests to the `tests` workflow + ## [0.233.0-rc.0] - 2024-01-07 ### Authors diff --git a/MANIFEST.in b/MANIFEST.in index 8d2699c53..71deeab82 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include VERSION include requirements.txt include requirements-client.txt +include requirements-qelectron.txt include covalent/py.typed recursive-include covalent/executor/ * recursive-include covalent_dispatcher/_service/ * diff --git a/covalent/__init__.py b/covalent/__init__.py index 6d88af00d..0cfc7bbe7 100644 --- a/covalent/__init__.py +++ b/covalent/__init__.py @@ -16,6 +16,7 @@ """Main Covalent public functionality.""" +import contextlib from importlib import metadata from . import _file_transfer as fs # nopycln: import @@ -48,9 +49,11 @@ lattice, ) from ._workflow.electron import wait # nopycln: import -from ._workflow.qelectron import qelectron # nopycln: import from .executor.utils import get_context # nopycln: import -from .quantum import QCluster # nopycln: import + +with contextlib.suppress(ImportError): + from ._workflow.qelectron import qelectron # nopycln: import + from .quantum import QCluster # nopycln: import __all__ = [s for s in dir() if not s.startswith("_")] diff --git a/covalent/_shared_files/qelectron_utils.py b/covalent/_shared_files/qelectron_utils.py index 1c0a8c6f3..387722a2a 100644 --- a/covalent/_shared_files/qelectron_utils.py +++ b/covalent/_shared_files/qelectron_utils.py @@ -14,10 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import importlib +import inspect +from typing import Any, Tuple -from covalent.quantum.qserver.database import Database +import cloudpickle +from pennylane._device import Device from .logger import app_log +from .pickling import _qml_mods_pickle + +_IMPORT_PATH_SEPARATOR = ":" def get_qelectron_db_path(dispatch_id: str, task_id: int): @@ -28,6 +35,8 @@ def get_qelectron_db_path(dispatch_id: str, task_id: int): AS WHERE THE USER'S TASK FUNCTION IS BEING RUN. """ + from covalent.quantum.qserver.database import Database + database = Database() db_path = database.get_db_path(dispatch_id=dispatch_id, node_id=task_id) @@ -38,3 +47,50 @@ def get_qelectron_db_path(dispatch_id: str, task_id: int): else: app_log.debug(f"Qelectron database not found for task {task_id}") return None + + +@_qml_mods_pickle +def cloudpickle_serialize(obj): + return cloudpickle.dumps(obj) + + +def cloudpickle_deserialize(obj): + return cloudpickle.loads(obj) + + +def select_first_executor(qnode, executors): + """Selects the first executor to run the qnode""" + return executors[0] + + +def get_import_path(obj) -> Tuple[str, str]: + """ + Determine the import path of an object. + """ + if module := inspect.getmodule(obj): + module_path = module.__name__ + class_name = obj.__name__ + return f"{module_path}{_IMPORT_PATH_SEPARATOR}{class_name}" + raise RuntimeError(f"Unable to determine import path for {obj}.") + + +def import_from_path(path: str) -> Any: + """ + Import a class from a path. + """ + module_path, class_name = path.split(_IMPORT_PATH_SEPARATOR) + module = importlib.import_module(module_path) + return getattr(module, class_name) + + +def get_original_shots(dev: Device): + """ + Recreate vector of shots if device has a shot vector. + """ + if not dev.shot_vector: + return dev.shots + + shot_sequence = [] + for shots in dev.shot_vector: + shot_sequence.extend([shots.shots] * shots.copies) + return type(dev.shot_vector)(shot_sequence) diff --git a/covalent/_shared_files/qresult_utils.py b/covalent/_shared_files/qresult_utils.py index 9ffe3c182..efad64797 100644 --- a/covalent/_shared_files/qresult_utils.py +++ b/covalent/_shared_files/qresult_utils.py @@ -23,7 +23,7 @@ from pennylane.tape import QuantumTape from .._workflow.qdevice import QEDevice -from .utils import get_original_shots +from .qelectron_utils import get_original_shots def re_execute( diff --git a/covalent/_shared_files/utils.py b/covalent/_shared_files/utils.py index e7bd60368..f41899f24 100644 --- a/covalent/_shared_files/utils.py +++ b/covalent/_shared_files/utils.py @@ -16,19 +16,14 @@ """General utils for Covalent.""" -import importlib import inspect import shutil import socket from datetime import timedelta -from typing import Any, Callable, Dict, List, Tuple - -import cloudpickle -from pennylane._device import Device +from typing import Callable, Dict, List, Tuple from . import logger from .config import get_config -from .pickling import _qml_mods_pickle app_log = logger.app_log log_stack_info = logger.log_stack_info @@ -37,9 +32,6 @@ DEFAULT_UI_PORT = get_config("user_interface.port") -_IMPORT_PATH_SEPARATOR = ":" - - def get_ui_url(path): baseUrl = f"http://{DEFAULT_UI_ADDRESS}:{DEFAULT_UI_PORT}" return f"{baseUrl}{path}" @@ -264,49 +256,16 @@ def copy_file_locally(src_uri, dest_uri): shutil.copyfile(src_path, dest_path) -@_qml_mods_pickle -def cloudpickle_serialize(obj): - return cloudpickle.dumps(obj) - - -def cloudpickle_deserialize(obj): - return cloudpickle.loads(obj) - - -def select_first_executor(qnode, executors): - """Selects the first executor to run the qnode""" - return executors[0] - - -def get_import_path(obj) -> Tuple[str, str]: - """ - Determine the import path of an object. +def get_qelectron_db_path(dispatch_id: str, task_id: int): """ - module = inspect.getmodule(obj) - if module: - module_path = module.__name__ - class_name = obj.__name__ - return f"{module_path}{_IMPORT_PATH_SEPARATOR}{class_name}" - raise RuntimeError(f"Unable to determine import path for {obj}.") + Return the path to the Qelectron database for a given dispatch_id and task_id. - -def import_from_path(path: str) -> Any: - """ - Import a class from a path. + This is a proxy to qelectron_utils.get_qelectron_db_path() for removing qelectron dependency. """ - module_path, class_name = path.split(_IMPORT_PATH_SEPARATOR) - module = importlib.import_module(module_path) - return getattr(module, class_name) + try: + from .qelectron_utils import get_qelectron_db_path -def get_original_shots(dev: Device): - """ - Recreate vector of shots if device has a shot vector. - """ - if not dev.shot_vector: - return dev.shots - - shot_sequence = [] - for shots in dev.shot_vector: - shot_sequence.extend([shots.shots] * shots.copies) - return type(dev.shot_vector)(shot_sequence) + return get_qelectron_db_path(dispatch_id, task_id) + except ImportError: + return None diff --git a/covalent/_workflow/qelectron.py b/covalent/_workflow/qelectron.py index e0356723a..641e4a78e 100644 --- a/covalent/_workflow/qelectron.py +++ b/covalent/_workflow/qelectron.py @@ -19,7 +19,7 @@ import pennylane as qml -from .._shared_files.utils import get_import_path, get_original_shots +from .._shared_files.qelectron_utils import get_import_path, get_original_shots from ..quantum.qcluster import QCluster from ..quantum.qcluster.base import AsyncBaseQCluster, BaseQExecutor from ..quantum.qcluster.simulator import Simulator diff --git a/covalent/_workflow/qnode.py b/covalent/_workflow/qnode.py index 7e97fee34..f2019aaa0 100644 --- a/covalent/_workflow/qnode.py +++ b/covalent/_workflow/qnode.py @@ -25,9 +25,9 @@ from .._results_manager.qresult import QNodeFutureResult from .._shared_files import logger +from .._shared_files.qelectron_utils import get_original_shots from .._shared_files.qinfo import QElectronInfo, QNodeSpecs from .._shared_files.qresult_utils import re_execute -from .._shared_files.utils import get_original_shots from ..executor.qbase import BaseQExecutor from .qdevice import QEDevice diff --git a/covalent/executor/__init__.py b/covalent/executor/__init__.py index ffc139f96..be68b1182 100644 --- a/covalent/executor/__init__.py +++ b/covalent/executor/__init__.py @@ -31,7 +31,6 @@ from .._shared_files import logger from .._shared_files.config import get_config, update_config -from ..quantum import QCluster, Simulator from .base import BaseExecutor app_log = logger.app_log @@ -284,6 +283,8 @@ class _QExecutorManager: """ def __init__(self): + from ..quantum import QCluster, Simulator + # Dictionary mapping executor name to executor class self.executor_plugins_map: Dict[str, Any] = { "QCluster": QCluster, @@ -370,11 +371,12 @@ def validate_module(self, module_obj) -> None: _executor_manager = _ExecutorManager() -_qexecutor_manager = _QExecutorManager() - for name in _executor_manager.executor_plugins_map: plugin_class = _executor_manager.executor_plugins_map[name] globals()[plugin_class.__name__] = plugin_class -for qexecutor_cls in _qexecutor_manager.executor_plugins_map.values(): - globals()[qexecutor_cls.__name__] = qexecutor_cls +# Only creating the qexecutor manager if its requirements are installed +with contextlib.suppress(ImportError): + _qexecutor_manager = _QExecutorManager() + for qexecutor_cls in _qexecutor_manager.executor_plugins_map.values(): + globals()[qexecutor_cls.__name__] = qexecutor_cls diff --git a/covalent/executor/quantum_plugins/qiskit_plugin/qiskit_plugin.py b/covalent/executor/quantum_plugins/qiskit_plugin/qiskit_plugin.py index 43531ddef..a959122e3 100644 --- a/covalent/executor/quantum_plugins/qiskit_plugin/qiskit_plugin.py +++ b/covalent/executor/quantum_plugins/qiskit_plugin/qiskit_plugin.py @@ -25,7 +25,7 @@ from runtime_sampler import QiskitRuntimeSampler from covalent._shared_files.config import get_config -from covalent._shared_files.utils import import_from_path +from covalent._shared_files.qelectron_utils import import_from_path from covalent.executor.qbase import ( AsyncBaseQExecutor, BaseThreadPoolQExecutor, diff --git a/covalent/executor/utils/wrappers.py b/covalent/executor/utils/wrappers.py index 4cbd7fccb..8f7fd8256 100644 --- a/covalent/executor/utils/wrappers.py +++ b/covalent/executor/utils/wrappers.py @@ -29,7 +29,7 @@ import requests -from covalent._shared_files.qelectron_utils import get_qelectron_db_path +from covalent._shared_files.utils import get_qelectron_db_path from covalent._workflow.depsbash import DepsBash from covalent._workflow.depscall import RESERVED_RETVAL_KEY__FILES, DepsCall from covalent._workflow.depspip import DepsPip diff --git a/covalent/quantum/qclient/local_client.py b/covalent/quantum/qclient/local_client.py index 89bff7382..8ca46c99d 100644 --- a/covalent/quantum/qclient/local_client.py +++ b/covalent/quantum/qclient/local_client.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ..._shared_files.utils import cloudpickle_deserialize, cloudpickle_serialize +from ..._shared_files.qelectron_utils import cloudpickle_deserialize, cloudpickle_serialize from ..qserver import LocalQServer from .base_client import BaseQClient diff --git a/covalent/quantum/qcluster/clusters.py b/covalent/quantum/qcluster/clusters.py index fe56aecbf..7aa1de314 100644 --- a/covalent/quantum/qcluster/clusters.py +++ b/covalent/quantum/qcluster/clusters.py @@ -17,7 +17,7 @@ import base64 from typing import Callable, Union -from ..._shared_files.utils import cloudpickle_deserialize, cloudpickle_serialize +from ..._shared_files.qelectron_utils import cloudpickle_deserialize, cloudpickle_serialize from .base import AsyncBaseQCluster, BaseQExecutor from .default_selectors import selector_map diff --git a/covalent/quantum/qserver/core.py b/covalent/quantum/qserver/core.py index f8efe09d8..2bc31c913 100644 --- a/covalent/quantum/qserver/core.py +++ b/covalent/quantum/qserver/core.py @@ -25,12 +25,12 @@ from pennylane.tape import QuantumScript -from ..._shared_files.qinfo import QElectronInfo, QNodeSpecs -from ..._shared_files.utils import ( +from ..._shared_files.qelectron_utils import ( cloudpickle_deserialize, cloudpickle_serialize, select_first_executor, ) +from ..._shared_files.qinfo import QElectronInfo, QNodeSpecs from ...executor.utils import get_context from ..qcluster.base import AsyncBaseQCluster, BaseQExecutor from .database import Database diff --git a/covalent_dispatcher/_cli/service.py b/covalent_dispatcher/_cli/service.py index faa14617d..df2299a8f 100644 --- a/covalent_dispatcher/_cli/service.py +++ b/covalent_dispatcher/_cli/service.py @@ -27,6 +27,7 @@ import sys import time import traceback +import warnings from pathlib import Path from subprocess import DEVNULL, Popen from typing import Optional @@ -50,6 +51,7 @@ from rich.syntax import Syntax from rich.table import Table from rich.text import Text +from sqlalchemy import exc as sa_exc from covalent._shared_files.config import ConfigManager, get_config, reload_config, set_config @@ -67,6 +69,9 @@ ZOMBIE_PROCESS_STATUS_MSG = "Covalent server is unhealthy: Process is in zombie status" STOPPED_PROCESS_STATUS_MSG = "Covalent server is unhealthy: Process is in stopped status" +# Ignore SQLAlchemy warnings +warnings.simplefilter("ignore", category=sa_exc.SAWarning) + def print_header(console): branding_title = Text("Covalent", style="bold blue") diff --git a/covalent_ui/api/v1/data_layer/electron_dal.py b/covalent_ui/api/v1/data_layer/electron_dal.py index 5f85c965a..9e10ee154 100644 --- a/covalent_ui/api/v1/data_layer/electron_dal.py +++ b/covalent_ui/api/v1/data_layer/electron_dal.py @@ -28,7 +28,6 @@ from covalent._results_manager.results_manager import get_result from covalent._shared_files import logger from covalent._shared_files.config import get_config -from covalent.quantum.qserver.database import Database from covalent_dispatcher._core.execution import _get_task_inputs as get_task_inputs from covalent_ui.api.v1.data_layer.lattice_dal import Lattices from covalent_ui.api.v1.database.schema.electron import Electron @@ -322,11 +321,17 @@ def get_electron_inputs(self, dispatch_id: uuid.UUID, electron_id: int) -> str: def _get_qelectron_db_dict(self, dispatch_id: str, node_id: int) -> dict: """Return the QElectron DB for a given node.""" - electron = self.get_electrons_id(dispatch_id, node_id) + try: + from covalent.quantum.qserver.database import Database - database = Database(electron.storage_path) - qelectron_db_dict = database.get_db_dict( - dispatch_id=dispatch_id, node_id=node_id, direct_path=True - ) + electron = self.get_electrons_id(dispatch_id, node_id) + + database = Database(electron.storage_path) + qelectron_db_dict = database.get_db_dict( + dispatch_id=dispatch_id, node_id=node_id, direct_path=True + ) - return qelectron_db_dict + return qelectron_db_dict + except ImportError: + app_log.debug("QElectron not installed.") + return {} diff --git a/requirements-qelectron.txt b/requirements-qelectron.txt new file mode 100644 index 000000000..98b537b50 --- /dev/null +++ b/requirements-qelectron.txt @@ -0,0 +1,4 @@ +lmdbm>=0.0.5 +mpire>=2.7.1 +orjson>=3.8.10 +pennylane>=0.31.1,<0.33.0 diff --git a/requirements.txt b/requirements.txt index 886884819..f9d0fa34c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,12 +8,8 @@ dask[distributed]>=2022.6.0 fastapi>=0.100.0 filelock>=3.12.2 furl>=2.1.3 -lmdbm>=0.0.5 -mpire>=2.7.1 natsort>=8.4.0 networkx>=2.8.6 -orjson>=3.8.10 -pennylane>=0.31.1,<0.33.0 psutil>=5.9.0 pydantic>=2.1.1 python-multipart>=0.0.6 diff --git a/setup.py b/setup.py index 50e3d6b63..317a469c7 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,11 @@ with open(requirements_file) as f: required = f.read().splitlines() +# By default we don't install qelectron requirements +# and only install them as an extra +with open("requirements-qelectron.txt") as f: + qelectron_reqs = f.read().splitlines() + def recursively_append_files(directory: str): """ @@ -220,6 +225,7 @@ def find_sources(self): "qiskit-ibm-provider==0.6.1", "qiskit-ibm-runtime==0.10.0", ], + "quantum": qelectron_reqs, }, "classifiers": [ "Development Status :: 4 - Beta", diff --git a/tests/covalent_tests/shared_files/qelectron_utils_test.py b/tests/covalent_tests/shared_files/qelectron_utils_test.py index 6c213e217..2cb66dbb6 100644 --- a/tests/covalent_tests/shared_files/qelectron_utils_test.py +++ b/tests/covalent_tests/shared_files/qelectron_utils_test.py @@ -30,7 +30,7 @@ def test_get_qelectron_db_path(mocker, db_exists): """Test the get_qelectron_db_path function.""" - mock_database = mocker.patch("covalent._shared_files.qelectron_utils.Database") + mock_database = mocker.patch("covalent.quantum.qserver.database.Database") mock_database.return_value.get_db_path.return_value.exists.return_value = db_exists dispatch_id = "mock_dispatch_id" diff --git a/tests/qelectron_tests/core_tests/__init__.py b/tests/qelectron_tests/core_tests/__init__.py new file mode 100644 index 000000000..6b2415372 --- /dev/null +++ b/tests/qelectron_tests/core_tests/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2024 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/qelectron_tests/test_decorator.py b/tests/qelectron_tests/core_tests/test_decorator.py similarity index 98% rename from tests/qelectron_tests/test_decorator.py rename to tests/qelectron_tests/core_tests/test_decorator.py index f376d0099..3bd11da15 100644 --- a/tests/qelectron_tests/test_decorator.py +++ b/tests/qelectron_tests/core_tests/test_decorator.py @@ -25,7 +25,7 @@ import covalent as ct EXECUTORS = [ - ct.executor.QiskitExecutor(device="local_sampler", shots=10_000), + ct.executor.Simulator(), ] diff --git a/tests/qelectron_tests/test_qelectron_db.py b/tests/qelectron_tests/core_tests/test_qelectron_db.py similarity index 96% rename from tests/qelectron_tests/test_qelectron_db.py rename to tests/qelectron_tests/core_tests/test_qelectron_db.py index 4d75f296e..7bc771003 100644 --- a/tests/qelectron_tests/test_qelectron_db.py +++ b/tests/qelectron_tests/core_tests/test_qelectron_db.py @@ -25,7 +25,7 @@ def test_db_exposed_in_result(): """ # Define a QElectron circuit. - qexecutor = ct.executor.QiskitExecutor(device="local_sampler") # pylint: disable=no-member + qexecutor = ct.executor.Simulator() # pylint: disable=no-member @ct.qelectron(executors=qexecutor) @qml.qnode(qml.device("default.qubit", wires=1)) diff --git a/tests/qelectron_tests/test_run_later.py b/tests/qelectron_tests/core_tests/test_run_later.py similarity index 97% rename from tests/qelectron_tests/test_run_later.py rename to tests/qelectron_tests/core_tests/test_run_later.py index 169e96662..aec023182 100644 --- a/tests/qelectron_tests/test_run_later.py +++ b/tests/qelectron_tests/core_tests/test_run_later.py @@ -23,7 +23,6 @@ import covalent as ct EXECUTORS = [ - ct.executor.QiskitExecutor(device="local_sampler", shots=10_000), ct.executor.Simulator(), ] diff --git a/tests/qelectron_tests/pennylane_tests/conftest.py b/tests/qelectron_tests/pennylane_tests/conftest.py index 30b774570..773ba2edf 100644 --- a/tests/qelectron_tests/pennylane_tests/conftest.py +++ b/tests/qelectron_tests/pennylane_tests/conftest.py @@ -33,7 +33,7 @@ import pytest import covalent as ct -from covalent._shared_files.utils import get_original_shots +from covalent._shared_files.qelectron_utils import get_original_shots from covalent.quantum.qcluster.simulator import SIMULATOR_DEVICES SKIP_RETURN_TYPES = ["qml.apply", "qml.vn_entropy", "qml.mutual_info"] diff --git a/tests/qelectron_tests/plugin_tests/__init__.py b/tests/qelectron_tests/plugin_tests/__init__.py new file mode 100644 index 000000000..6b2415372 --- /dev/null +++ b/tests/qelectron_tests/plugin_tests/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2024 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/qelectron_tests/test_braket_plugin.py b/tests/qelectron_tests/plugin_tests/test_braket_plugin.py similarity index 100% rename from tests/qelectron_tests/test_braket_plugin.py rename to tests/qelectron_tests/plugin_tests/test_braket_plugin.py diff --git a/tests/qelectron_tests/test_qiskit_plugin.py b/tests/qelectron_tests/plugin_tests/test_qiskit_plugin.py similarity index 98% rename from tests/qelectron_tests/test_qiskit_plugin.py rename to tests/qelectron_tests/plugin_tests/test_qiskit_plugin.py index 438532424..b6046005f 100644 --- a/tests/qelectron_tests/test_qiskit_plugin.py +++ b/tests/qelectron_tests/plugin_tests/test_qiskit_plugin.py @@ -23,7 +23,7 @@ import covalent as ct from covalent._shared_files.config import get_config -from .utils import arg_vector, simple_circuit, weight_vector +from ..utils import arg_vector, simple_circuit, weight_vector EXECUTOR_CLASSES = [ ct.executor.QiskitExecutor, diff --git a/tests/qelectron_tests/test_qiskit_plugin_runtime.py b/tests/qelectron_tests/plugin_tests/test_qiskit_plugin_runtime.py similarity index 98% rename from tests/qelectron_tests/test_qiskit_plugin_runtime.py rename to tests/qelectron_tests/plugin_tests/test_qiskit_plugin_runtime.py index 913d54388..e21724d23 100644 --- a/tests/qelectron_tests/test_qiskit_plugin_runtime.py +++ b/tests/qelectron_tests/plugin_tests/test_qiskit_plugin_runtime.py @@ -25,7 +25,7 @@ import covalent as ct from covalent._shared_files.config import get_config -from .utils import arg_vector, cyclic_selector, get_hamiltonian_circuit, weight_vector +from ..utils import arg_vector, cyclic_selector, get_hamiltonian_circuit, weight_vector EXECUTOR_CLASSES = [ ct.executor.QiskitExecutor, diff --git a/tests/requirements.txt b/tests/requirements.txt index 99f936c73..38c17dc3c 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,6 +5,7 @@ isort>=5.10.1 locust>=2.11.0 mock>=4.0.3 nbconvert>=6.5.1 +pennylane>=0.31.1 pre-commit>=2.20.0 pytest>=7.1.3 pytest-asyncio>=0.21.0 From 859858964529b21345d9854efe2fa1a9aacb7780 Mon Sep 17 00:00:00 2001 From: Ara Ghukasyan <38226926+araghukas@users.noreply.github.com> Date: Wed, 31 Jan 2024 20:29:38 -0500 Subject: [PATCH 02/12] Allow non-string infra defaults for plugins (#1921) * method to handle various py-to-tf conversions * update changelog * make staticmethod * add convert to tfvar test * fix changelog typo * remove getattr patch; assert new method called * fake infra defaults to test conversion in `up` --- CHANGELOG.md | 1 + covalent/cloud_resource_manager/core.py | 32 ++++++++++++-- .../cloud_resource_manager/core_test.py | 44 ++++++++++++++++--- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 474352138..17cf7a24d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added CRM method to handle Python to TF value conversion (e.g. None->null, True->true, False->false). - Added `pennylane` as a requirement in tests due to the tutorials using it ### Changed diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 974e2f61c..eb63bb604 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -24,7 +24,7 @@ from configparser import ConfigParser from pathlib import Path from types import ModuleType -from typing import Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Union from covalent._shared_files.config import set_config from covalent.executor import _executor_manager @@ -432,8 +432,8 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None: if "default" in value: tf_vars_env_dict[f"TF_VAR_{key}"] = value["default"] - if value["default"] != "": - f.write(f'{key}="{value["default"]}"\n') + if value["default"]: + f.write(f'{key}={self._convert_to_tfvar(value["default"])}\n') # Overwrite the default values with the user passed values if self.executor_options: @@ -537,3 +537,29 @@ def status(self) -> None: # Run `terraform state list` return self._run_in_subprocess(cmd=tf_state, env_vars=self._terraform_log_env_vars) + + @staticmethod + def _convert_to_tfvar(value: Any) -> Any: + """ + Convert the value to a string that can be parsed as a terraform variable. + + Args: + value: Value to convert + + Returns: + Converted value + + """ + if value is True: + return "true" + if value is False: + return "false" + if value is None: + return "null" + if isinstance(value, str): + return f'"{value}"' + if isinstance(value, Sequence): + values = [CloudResourceManager._convert_to_tfvar(v) for v in value] + return f"[{', '.join(values)}]" + + return str(value) diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py index 7fdca350d..b9b0f2ab0 100644 --- a/tests/covalent_tests/cloud_resource_manager/core_test.py +++ b/tests/covalent_tests/cloud_resource_manager/core_test.py @@ -41,6 +41,18 @@ def executor_module_path(): return "test_executor_module_path" +@pytest.fixture +def executor_infra_defaults(): + from pydantic import BaseModel + + class FakeExecutorInfraDefaults(BaseModel): + string_param: str = "fake_address_123" + number_param: int = 123 + sequence_param: tuple = (1, 2, 3) + + return FakeExecutorInfraDefaults + + @pytest.fixture def crm(mocker, executor_name, executor_module_path): mocker.patch( @@ -377,7 +389,9 @@ def test_get_tf_statefile_path(mocker, crm, executor_name): (False, {"test_key": "test_value"}), ], ) -def test_up(mocker, dry_run, executor_options, executor_name, executor_module_path): +def test_up( + mocker, dry_run, executor_options, executor_name, executor_module_path, executor_infra_defaults +): """ Unit test for CloudResourceManager.up() method """ @@ -401,10 +415,6 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa "covalent.cloud_resource_manager.core.get_executor_module", ) - mocker.patch( - "covalent.cloud_resource_manager.core.getattr", - ) - # Mocking as we are instantiating with executor options mocker.patch( "covalent.cloud_resource_manager.core.validate_options", @@ -438,6 +448,9 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa options=executor_options, ) + # Override infra defaults with dummy values. + crm.ExecutorInfraDefaults = executor_infra_defaults + with mock.patch( "covalent.cloud_resource_manager.core.open", mock.mock_open(), @@ -652,6 +665,27 @@ def test_crm_get_resource_status(mocker, crm): mock_terraform_error_validator.assert_called_once() +def test_crm_convert_to_tfvar(mocker, crm): + """ + Unit test for CloudResourceManager._convert_to_tfvar() method. + + Test conversion outcomes. + """ + _values_map = { + # Convenient test case (not valid Terraform): + (1, False, None, "covalent321"): '[1, false, null, "covalent321"]', + # Usual test cases: + True: "true", + False: "false", + None: "null", + "covalent123": '"covalent123"', # must include quotes + 16: "16", + } + + for _value, _expected in _values_map.items(): + assert crm._convert_to_tfvar(_value) == _expected + + def test_no_git(crm, mocker): """ Test for exit with status 1 if `git` is not available. From 7c9b2cb72638f69d7bd452bb4e9866a7c87533fc Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Mon, 5 Feb 2024 07:47:49 -0500 Subject: [PATCH 03/12] Modifying nightly CI to be more interactive and streamlined (#1869) * Modifying CI to be more interactive * removed conda release * added on trigger and warning description * added the create release workflow as well * updated changelog * updated workflows * added retry mechanism to uploading to codecov * using pre-release version from workflow dispatch * added version assigner to release * fixed some stuff * fixed some stuff * fixed some stuff * fixed some stuff * minor fixes to result attributes * minor fixes * fixed serialization of output * updated changelog * trying to limit cloudpickle * removed minor changes * fixing tests.yml * fixing tests.yml * fixing tests.yml * using local workflows instead of from the develop branch * fixed release.yml * fixed push_to_master workflow * added permissions to nightly-tests.yml * updated changelog * Update nightly-tests.yml --- .github/workflows/changelog.yml | 4 + .github/workflows/man_0_assign_version.yml | 45 ++++ .github/workflows/man_1_push_to_master.yml | 80 ++++++ .github/workflows/man_2_create_prerelease.yml | 66 +++++ .github/workflows/nightly-tests.yml | 40 +++ .github/workflows/nightly.yml | 144 ---------- .github/workflows/release.yml | 249 ++++-------------- .github/workflows/tests.yml | 120 +++++---- CHANGELOG.md | 13 +- 9 files changed, 354 insertions(+), 407 deletions(-) create mode 100644 .github/workflows/man_0_assign_version.yml create mode 100644 .github/workflows/man_1_push_to_master.yml create mode 100644 .github/workflows/man_2_create_prerelease.yml create mode 100644 .github/workflows/nightly-tests.yml delete mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 9a3589671..046d3318a 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -32,6 +32,7 @@ jobs: uses: actions/checkout@v4 with: token: ${{ secrets.COVALENT_OPS_BOT_TOKEN }} + - uses: dorny/paths-filter@v2 id: filter with: @@ -42,6 +43,7 @@ jobs: - '.github/actions/changelog/action.yml' dist: - '.github/actions/changelog/dist/**' + - name: Latest tag id: get-latest-tag uses: ./.github/actions/describe @@ -49,6 +51,7 @@ jobs: token: ${{ secrets.COVALENT_OPS_BOT_TOKEN }} branch: develop stable: false + - name: Update version number id: changelog uses: ./.github/actions/changelog @@ -57,6 +60,7 @@ jobs: version-path: VERSION token: ${{ secrets.COVALENT_OPS_BOT_TOKEN }} basehead: ${{ steps.get-latest-tag.outputs.tag }}...${{ github.sha }} + - name: Commit if: ${{ steps.changelog.outputs.message != 'noop' }} uses: EndBug/add-and-commit@v9 diff --git a/.github/workflows/man_0_assign_version.yml b/.github/workflows/man_0_assign_version.yml new file mode 100644 index 000000000..b9ce3d410 --- /dev/null +++ b/.github/workflows/man_0_assign_version.yml @@ -0,0 +1,45 @@ +# Copyright 2024 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: man_0_assign_version + +on: + workflow_dispatch: + inputs: + nightly_tests_failed: + type: boolean + required: true + default: true + description: "WARNING: Make sure the `nightly-tests` or the most recent `tests` workflow has passed successfully in develop before running this workflow. + Uncheck this box if it has." + +permissions: + id-token: write + contents: read + +jobs: + license: + name: License Scanner + uses: ./.github/workflows/license.yml + + version_assigner: + name: Assign Version + needs: + - license + if: > + !github.event.inputs.nightly_tests_failed + uses: ./.github/workflows/changelog.yml + secrets: inherit # pragma: allowlist secret diff --git a/.github/workflows/man_1_push_to_master.yml b/.github/workflows/man_1_push_to_master.yml new file mode 100644 index 000000000..efe6d4db4 --- /dev/null +++ b/.github/workflows/man_1_push_to_master.yml @@ -0,0 +1,80 @@ +# Copyright 2024 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: man_1_push_to_master + +on: + workflow_dispatch: + inputs: + assign_version_failed: + type: boolean + required: true + default: true + description: "WARNING: Make sure the `man_0_assign_version` workflow has passed successfully before running this workflow. + Uncheck this box if it has." + +permissions: + id-token: write + contents: read + +jobs: + push_to_master: + name: Push develop to master + runs-on: ubuntu-latest + outputs: + release: ${{ steps.push.outputs.release }} + + steps: + - name: Checkout develop + uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Get versions of develop and master + id: get-versions + run: | + develop_version="$(cat ./VERSION)" + master_version="$(git show origin/master:VERSION)" + echo "::set-output name=develop_version::${develop_version}" + echo "::set-output name=master_version::${master_version}" + + - name: Perform the push to master if develop is ahead + id: push + if: > + !github.event.inputs.assign_version_failed + run: | + DEVELOP_VERSION="${{ steps.get-versions.outputs.develop_version }}" + MASTER_VERSION="${{ steps.get-versions.outputs.master_version }}" + release=false + echo "Develop version: ${DEVELOP_VERSION}" + echo "Master version: ${MASTER_VERSION}" + if [[ "${DEVELOP_VERSION}" == "${MASTER_VERSION}" ]]; then + echo "No new version detected. Exiting." + exit 1 + elif dpkg --compare-versions $DEVELOP_VERSION 'gt' $MASTER_VERSION ; then + echo "Pushing to master." + git config user.name "CovalentOpsBot" + git config user.email "covalentopsbot@users.noreply.github.com" + git remote set-url origin https://${{ secrets.COVALENT_OPS_BOT_TOKEN }}@github.com/AgnostiqHQ/covalent.git + git push origin HEAD:master + release=true + else + echo "This means the version on develop is lower than the version on master or something is wrong." + exit 1 + fi + echo "Ready to release: ${release}" + echo "::set-output name=release::$release" diff --git a/.github/workflows/man_2_create_prerelease.yml b/.github/workflows/man_2_create_prerelease.yml new file mode 100644 index 000000000..e982b8b37 --- /dev/null +++ b/.github/workflows/man_2_create_prerelease.yml @@ -0,0 +1,66 @@ +# Copyright 2024 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: man_2_create_prerelease + +on: + workflow_dispatch: + inputs: + push_to_master_failed: + type: boolean + required: true + default: true + description: "WARNING: Make sure the `man_1_push_to_master` workflow has passed successfully before running this workflow. + Uncheck this box if it has." + +permissions: + id-token: write + contents: read + +jobs: + create_release: + name: Create a Prerelease + uses: ./.github/workflows/release.yml + if: > + !github.event.inputs.push_to_master_failed + with: + prerelease: true + secrets: inherit # pragma: allowlist secret + + notify_slack: + name: Notify on Slack + needs: + - create_release + runs-on: ubuntu-latest + steps: + - name: Checkout master + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.branch_name }} + + - name: Format Slack message + run: | + VERSION="$(cat ./VERSION)" + SLACK_MSG=":rocket: Version $VERSION is now available." + echo "SLACK_MSG=$SLACK_MSG" >> $GITHUB_ENV + + - name: Notify Slack + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_CHANNEL: "covalent-ci" + SLACK_USERNAME: "CovalentOpsBot" + SLACK_MESSAGE: ${{ env.SLACK_MSG }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml new file mode 100644 index 000000000..da7becdf1 --- /dev/null +++ b/.github/workflows/nightly-tests.yml @@ -0,0 +1,40 @@ +# Copyright 2024 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: nightly-tests + +on: + push: + branches: + - develop + schedule: + - cron: "0 0 * * *" + + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + license: + name: License Scanner + uses: ./.github/workflows/license.yml + + tests: + name: Unit and Functional Tests + uses: ./.github/workflows/tests.yml + secrets: inherit # pragma: allowlist secret diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index 539c4c581..000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2021 Agnostiq Inc. -# -# This file is part of Covalent. -# -# Licensed under the Apache License 2.0 (the "License"). A copy of the -# License may be obtained with this software package or at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Use of this file is prohibited except in compliance with the License. -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: nightly - -on: - schedule: - - cron: "*/10 * * * *" - -permissions: - id-token: write - contents: read - -jobs: - license: - name: License Scanner - uses: AgnostiqHQ/covalent/.github/workflows/license.yml@develop - - tests: - name: Unit and Functional Tests - uses: AgnostiqHQ/covalent/.github/workflows/tests.yml@develop - secrets: inherit # pragma: allowlist secret - - changelog: - name: Assign Version - needs: - - license - - tests - uses: AgnostiqHQ/covalent/.github/workflows/changelog.yml@develop - secrets: inherit # pragma: allowlist secret - - push_to_master: - name: Push to Master - runs-on: ubuntu-latest - needs: changelog - outputs: - release: ${{ steps.push.outputs.release }} - steps: - - name: Get latest release - id: query-tags - uses: octokit/request-action@v2.x - with: - route: GET /repos/AgnostiqHQ/covalent/tags - env: - GITHUB_TOKEN: ${{ secrets.COVALENT_OPS_BOT_TOKEN }} - - - name: Parse latest tag - id: get-latest-tag - run: | - # This assumes that the response from the API is sorted in decreasing order (thus the first element is the latest tag) - latest_tag=${{ fromJSON(steps.query-tags.outputs.data)[0].name }} - echo "::set-output name=tag::${latest_tag}" - - - name: Checkout default branch - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Push to master - id: push - run: | - CHANGELOG_VERSION="${{ needs.changelog.outputs.version }}" - MASTER_VERSION="$(echo ${{ steps.get-latest-tag.outputs.tag }} | cut -c2- )" - VERSION="$(cat ./VERSION)" - release=false - if [ "$MASTER_VERSION" = "$VERSION" ] ; then - echo "$VERSION has been previously released." - elif dpkg --compare-versions $VERSION 'gt' '0.177.0' ; then - git config user.name "CovalentOpsBot" - git config user.email "covalentopsbot@users.noreply.github.com" - git remote set-url origin https://${{ secrets.COVALENT_OPS_BOT_TOKEN }}@github.com/AgnostiqHQ/covalent.git - git push origin HEAD:master - release=true - else - echo "We cannot release versions less than 0.177.0." - fi - echo "::set-output name=release::$release" - - release: - name: Create Release - needs: push_to_master - if: needs.push_to_master.outputs.release == 'true' - uses: AgnostiqHQ/covalent/.github/workflows/release.yml@develop - with: - prerelease: true - secrets: inherit # pragma: allowlist secret - - notify: - name: Notify Slack - needs: release - runs-on: ubuntu-latest - steps: - - name: Checkout master - uses: actions/checkout@v4 - with: - ref: "master" - - - name: Format Slack message - run: | - VERSION="$(cat ./VERSION)" - SLACK_MSG=":rocket: Version $VERSION is now available." - echo "SLACK_MSG=$SLACK_MSG" >> $GITHUB_ENV - - - name: Notify Slack - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_CHANNEL: "covalent-ci" - SLACK_USERNAME: "CovalentOpsBot" - SLACK_MESSAGE: ${{ env.SLACK_MSG }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - executor_base_images: - name: Build Executor base images - runs-on: ubuntu-latest - needs: release - strategy: - matrix: - repo: - [ - "AgnostiqHQ/covalent-aws-plugins", - "AgnostiqHQ/covalent-awslambda-plugin", - "AgnostiqHQ/covalent-braket-plugin", - ] - steps: - - name: Build executor_base_images - uses: peter-evans/repository-dispatch@v2 - with: - token: ${{ secrets.COVALENT_OPS_BOT_TOKEN }} - repo: ${{ matrix.repo }} - event-type: "prerelease" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2e5b1da83..a928b35d5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ on: workflow_dispatch: inputs: stable_version: - description: "Stable version number, e.g. 0.32.3" + description: "Stable version number, e.g. 0.32.3. Mandatory if a stable release is being created." type: string test_release: description: "Test the workflow but don't create the release. Uncheck this box to create a release." @@ -30,15 +30,10 @@ on: workflow_call: inputs: prerelease: - description: "true: Create a prerelease. false: Create a stable release" + description: "true: Create a prerelease. false: Create a stable release." required: true type: boolean default: true - prerelease_version: - description: "The prerelease version to release" - required: false - type: string - default: "master" permissions: id-token: write @@ -50,9 +45,8 @@ env: '"AlejandroEsquivel",' '"FyzHsn",' '"wjcunningham7",' + '"kessler-frost",' '"santoshkumarradha"]' - EXECUTOR_BASE_DOCKERFILE_URL: "https://raw.githubusercontent.com/AgnostiqHQ/covalent-aws-plugins/develop/Dockerfile?token=${{ secrets.COVALENT_OPS_BOT_TOKEN }}" - AWS_PLUGINS_VERSION_URL: "https://raw.githubusercontent.com/AgnostiqHQ/covalent-aws-plugins/develop/VERSION?token=${{ secrets.COVALENT_OPS_BOT_TOKEN }}" jobs: github: @@ -61,38 +55,20 @@ jobs: release: ${{ env.RELEASE }} steps: - name: Check out stable release tag - uses: actions/checkout@v2 + uses: actions/checkout@v4 if: github.event.inputs.stable_version with: persist-credentials: false fetch-depth: 0 ref: "v${{ github.event.inputs.stable_version }}" - - name: Format prerelease ref - if: inputs.prerelease - run: | - re='^[0-9]+$' - IFS='.' read -ra version <<< "${{ inputs.prerelease_version }}" - if [[ ${version[0]} =~ $re ]] ; then - echo "PRERELEASE=v$version" >> $GITHUB_ENV - else - echo "PRERELEASE=$version" >> $GITHUB_ENV - fi - - name: Check out prerelease tag - uses: actions/checkout@v2 + + - name: Check out master branch for prerelease + uses: actions/checkout@v4 if: inputs.prerelease with: persist-credentials: false fetch-depth: 0 - ref: "${{ env.PRERELEASE }}" - - name: Generate stable release message - if: > - github.event.inputs.stable_version - && contains(env.PAUL_BLART, github.actor) - id: stable-changelog - uses: AgnostiqHQ/covalent/.github/actions/stable-changelog@develop - with: - changelog-path: CHANGELOG.md - version-path: VERSION + ref: "master" - name: Read version run: | @@ -104,6 +80,27 @@ jobs: VERSION="$(cat ./VERSION)" echo "VERSION=$VERSION" >> $GITHUB_ENV echo "RELEASE=v$VERSION" >> $GITHUB_ENV + + - name: Tag commit for prerelease + if: inputs.prerelease + id: push + run: | + git config user.name "CovalentOpsBot" + git config user.email "covalentopsbot@users.noreply.github.com" + git tag -a $RELEASE -m "Release $RELEASE" + git remote set-url origin https://${{ secrets.COVALENT_OPS_BOT_TOKEN }}@github.com/AgnostiqHQ/covalent.git + git push origin $RELEASE + + - name: Generate stable release message + if: > + github.event.inputs.stable_version + && contains(env.PAUL_BLART, github.actor) + id: stable-changelog + uses: AgnostiqHQ/covalent/.github/actions/stable-changelog@develop + with: + changelog-path: CHANGELOG.md + version-path: VERSION + - name: Generate prerelease message if: inputs.prerelease id: message @@ -114,26 +111,20 @@ jobs: echo 'MESSAGE<> $GITHUB_ENV tail +$begin ./CHANGELOG.md | head -$end >> $GITHUB_ENV echo 'EOF' >> $GITHUB_ENV - - name: Tag commit - id: push - continue-on-error: true - run: | - git config user.name "CovalentOpsBot" - git config user.email "covalentopsbot@users.noreply.github.com" - git tag -a $RELEASE -m "Release $RELEASE" - git remote set-url origin https://${{ secrets.COVALENT_OPS_BOT_TOKEN }}@github.com/AgnostiqHQ/covalent.git - git push origin $RELEASE + - name: Create prerelease if: >- inputs.prerelease + && ${{ steps.push.outcome == 'success' && steps.message.outcome == 'success' - && (!github.event.inputs.test_release || github.event.inputs.test_release == 'false') + && (!github.event.inputs.test_release || github.event.inputs.test_release == 'false') }} uses: ncipollo/release-action@v1 with: body: ${{ env.MESSAGE }} token: ${{ secrets.COVALENT_OPS_BOT_TOKEN }} tag: ${{ env.RELEASE }} - prerelease: true + prerelease: ${{ inputs.prerelease }} + - name: Create stable release if: >- github.event.inputs.stable_version @@ -145,6 +136,7 @@ jobs: body: ${{ steps.stable-changelog.outputs.message }} token: ${{ secrets.COVALENT_OPS_BOT_TOKEN }} tag: ${{ env.RELEASE }} + - name: Alert Slack if: failure() uses: rtCamp/action-slack-notify@v2 @@ -163,36 +155,43 @@ jobs: version: ${{ steps.validate.outputs.version }} steps: - name: Check out release tag - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 ref: ${{ needs.github.outputs.release }} + - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.8 + - name: Install Python dependencies run: | python -m pip install --upgrade pip pip install twine + - name: Set up Node uses: actions/setup-node@v2 with: node-version: 16 + - name: Build Webapp run: | cd ./covalent_ui/webapp yarn install yarn build + - name: Build Stable or Pre-Release Distribution id: pre-or-stable-build run: python setup.py sdist + - name: Transform semver version to pep440 id: version-transform uses: ./.github/actions/version-transform with: version-path: VERSION + - name: Validate Distribution id: validate run: | @@ -211,6 +210,7 @@ jobs: diff -r covalent-${VERSION}/covalent_ui/webapp/build ../covalent_ui/webapp/build rm -rf covalent-${VERSION}/ echo "::set-output name=version::$VERSION" + - name: Upload Distribution if: > steps.pre-or-stable-build.outcome == 'success' @@ -220,43 +220,7 @@ jobs: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: twine upload dist/* - - name: Alert Slack - if: failure() - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_CHANNEL: "covalent-ci" - SLACK_USERNAME: "CovalentOpsBot" - SLACK_MESSAGE: "The release.yml workflow is failing in ${{ github.ref }}!" - SLACK_COLOR: ${{ job.status }} - SLACK_TITLE: ":warning: Attention Required :warning:" - SLACK_WEBHOOK: ${{ secrets.SLACK_ALERT_WEBHOOK }} - conda: - needs: pypi - runs-on: ubuntu-latest - continue-on-error: true - strategy: - fail-fast: false - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - steps: - - name: Check for stable release - if: github.event.inputs.stable_version && !inputs.prerelease - run: echo "STABLE=true" >> $GITHUB_ENV - - name: Conda skeleton publish - uses: AgnostiqHQ/conda-skeleton-publish@main - if: ${{ !github.event.inputs.test_release }} - with: - pypi_package: "covalent" - python_version: ${{ matrix.python-version }} - upload_channel: "agnostiq" - access_token: ${{ secrets.ANACONDA_TOKEN }} - package_version: ${{ needs.pypi.outputs.version }} - stable: ${{ env.STABLE }} - wait: true - name: Alert Slack if: failure() uses: rtCamp/action-slack-notify@v2 @@ -267,128 +231,3 @@ jobs: SLACK_COLOR: ${{ job.status }} SLACK_TITLE: ":warning: Attention Required :warning:" SLACK_WEBHOOK: ${{ secrets.SLACK_ALERT_WEBHOOK }} - - docker: - runs-on: ubuntu-latest - steps: - - name: Check out release tag - uses: actions/checkout@v2 - if: inputs.stable_version - with: - persist-credentials: false - fetch-depth: 0 - ref: "v${{ github.event.inputs.stable_version }}" - - - name: Check out master - uses: actions/checkout@v2 - if: inputs.prerelease - with: - persist-credentials: false - fetch-depth: 0 - - - name: Set up QEMU - uses: docker/setup-qemu-action@master - with: - platforms: "linux/amd64,linux/arm64" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@master - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.ECR_PUBLIC_UPLOAD_ROLE }} - aws-region: us-east-1 - - - name: Generate metadata - run: | - aws --version - docker info - TAG="$(cat ./VERSION)" - echo "TAG: $TAG" - echo "TAG=$TAG" >> $GITHUB_ENV - BUILD_DATE=`date -u +%Y-%m-%d` - echo "BUILD_DATE=$BUILD_DATE" >> $GITHUB_ENV - BUILD_VERSION=${{ github.sha }} - echo "BUILD_VERSION=$BUILD_VERSION" >> $GITHUB_ENV - - - name: Login to Public Registry - run: | - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - - - name: Build and push pre-release - if: > - inputs.prerelease - && !inputs.stable_version - && github.event_name == 'schedule' - uses: docker/build-push-action@v2 - with: - builder: ${{ steps.buildx.outputs.name }} - context: . - file: Dockerfile - platforms: linux/amd64 - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - BUILD_DATE=${{ env.BUILD_DATE }} - BUILD_VERSION=${{ env.BUILD_VERSION }} - push: true - tags: | - public.ecr.aws/covalent/covalent:latest - public.ecr.aws/covalent/covalent:${{ env.TAG }} - - - name: Build and push stable release - if: > - github.event_name == 'workflow_dispatch' - && inputs.stable_version - && !inputs.prerelease - && !github.events.inputs.test_release - && contains(env.PAUL_BLART, github.actor) - uses: docker/build-push-action@v2 - with: - builder: ${{ steps.buildx.outputs.name }} - context: . - file: Dockerfile - platforms: linux/amd64 - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - BUILD_DATE=${{ env.BUILD_DATE }} - BUILD_VERSION=${{ env.BUILD_VERSION }} - push: true - tags: | - public.ecr.aws/covalent/covalent:${{ inputs.stable_version }} - public.ecr.aws/covalent/covalent:stable - - # docker_aws_plugins: - # name: Trigger AWS Plugins Base Executor Image Pre-Release - # needs: pypi - # if: > - # inputs.prerelease - # && !inputs.stable_version - # && github.event_name == 'schedule' - # uses: AgnostiqHQ/covalent-aws-plugins/.github/workflows/docker.yml@develop - # with: - # prerelease: true - - # docker_aws_lambda: - # name: Trigger AWS Lambda Base Executor Image Pre-Release - # needs: pypi - # if: > - # inputs.prerelease - # && !inputs.stable_version - # && github.event_name == 'schedule' - # uses: AgnostiqHQ/covalent-awslambda-plugin/.github/workflows/docker.yml@develop - # with: - # prerelease: true - - # docker_aws_braket: - # name: Trigger AWS Braket Base Executor Image Pre-Release - # needs: pypi - # if: > - # inputs.prerelease - # && !inputs.stable_version - # && github.event_name == 'schedule' - # uses: AgnostiqHQ/covalent-braket-plugin/.github/workflows/docker.yml@develop - # with: - # prerelease: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 710655301..b66ce1ebb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,9 +28,10 @@ on: workflow_dispatch: inputs: commit_sha: - description: "Commit SHA used for testing" + description: "Commit SHA used for testing. If left blank, the default branch will be used." type: string - required: true + default: '' + required: false jobs: build_test_matrix: @@ -66,8 +67,8 @@ jobs: outputs: codecov: ${{ steps.local-codecov.outputs.local_codecov }} steps: - - name: Check out head - if: github.event_name != 'workflow_dispatch' + - name: Check out the default branch + if: github.event_name != 'workflow_dispatch' || github.event.inputs.commit_sha == '' uses: actions/checkout@v4 with: persist-credentials: false @@ -305,86 +306,97 @@ jobs: npm test -- --coverage --watchAll=false --maxWorkers=50% - name: Dump Covalent logs - if: > - steps.covalent_start.outcome == 'success' - && failure() run: covalent logs - - name: Upload SDK report to Codecov + - name: Upload SDK report to Codecov with retry id: upload-sdk-report if: > env.RECOMMENDED_PLATFORM - && github.event_name != 'workflow_dispatch' - && (github.event_name == 'schedule' + && (github.event_name == 'workflow_call' || steps.sdk-coverage.outcome == 'success') - uses: codecov/codecov-action@v3 + uses: Wandalen/wretry.action@master with: - files: ./sdk_coverage.xml - flags: SDK - name: "SDK Unit Tests" - fail_ci_if_error: true - - - name: Upload Dispatcher report to Codecov + action: codecov/codecov-action@v3 + with: | + files: ./sdk_coverage.xml + flags: SDK + name: "SDK Unit Tests" + fail_ci_if_error: true + attempt_limit: 5 + attempt_delay: 5000 + + - name: Upload Dispatcher report to Codecov with retry id: upload-dispatcher-report if: > env.RECOMMENDED_PLATFORM - && github.event_name != 'workflow_dispatch' - && (github.event_name == 'schedule' + && (github.event_name == 'workflow_call' || steps.dispatcher-coverage.outcome == 'success') - uses: codecov/codecov-action@v3 + uses: Wandalen/wretry.action@master with: - files: ./dispatcher_coverage.xml - flags: Dispatcher - name: "Dispatcher Unit Tests" - fail_ci_if_error: true - - - name: Upload Functional report to Codecov + action: codecov/codecov-action@v3 + with: | + files: ./dispatcher_coverage.xml + flags: Dispatcher + name: "Dispatcher Unit Tests" + fail_ci_if_error: true + attempt_limit: 5 + attempt_delay: 5000 + + - name: Upload Functional report to Codecov with retry id: upload-functional-report if: > env.RECOMMENDED_PLATFORM - && github.event_name != 'workflow_dispatch' && steps.functional-coverage.outcome == 'success' - uses: codecov/codecov-action@v3 + uses: Wandalen/wretry.action@master with: - files: ./functional_tests_coverage.xml - flags: Functional_Tests - name: "Functional Tests" - fail_ci_if_error: true - - - name: Upload UI backend report to Codecov + action: codecov/codecov-action@v3 + with: | + files: ./functional_tests_coverage.xml + flags: Functional_Tests + name: "Functional Tests" + fail_ci_if_error: true + attempt_limit: 5 + attempt_delay: 5000 + + - name: Upload UI backend report to Codecov with retry id: upload-ui-backend-report if: > env.RECOMMENDED_PLATFORM - && github.event_name != 'workflow_dispatch' - && (github.event_name == 'schedule' + && (github.event_name == 'workflow_call' || steps.ui-backend-coverage.outcome == 'success') - uses: codecov/codecov-action@v3 + uses: Wandalen/wretry.action@master with: - files: ./ui_backend_coverage.xml - flags: UI_Backend - name: "UI Backend Unit Tests" - fail_ci_if_error: true - - - name: Upload UI frontend report to Codecov + action: codecov/codecov-action@v3 + with: | + files: ./ui_backend_coverage.xml + flags: UI_Backend + name: "UI Backend Unit Tests" + fail_ci_if_error: true + attempt_limit: 5 + attempt_delay: 5000 + + - name: Upload UI frontend report to Codecov with retry id: upload-ui-frontend-report if: > env.RECOMMENDED_PLATFORM - && github.event_name != 'workflow_dispatch' - && (github.event_name == 'schedule' + && (github.event_name == 'workflow_call' || steps.ui-frontend-tests.outcome == 'success') - uses: codecov/codecov-action@v3 + uses: Wandalen/wretry.action@master with: - files: ./covalent_ui/webapp/coverage/clover.xml - flags: UI_Frontend - name: "UI Frontend Unit Tests" - fail_ci_if_error: true + action: codecov/codecov-action@v3 + with: | + files: ./covalent_ui/webapp/coverage/clover.xml + flags: UI_Frontend + name: "UI Frontend Unit Tests" + fail_ci_if_error: true + attempt_limit: 5 + attempt_delay: 5000 - name: Local Codecov id: local-codecov if: > env.RECOMMENDED_PLATFORM - && github.event_name != 'workflow_dispatch' - && github.event_name != 'schedule' + && github.event_name == 'workflow_call' run: | if ${{ steps.upload-sdk-report.outcome == 'skipped' && steps.upload-dispatcher-report.outcome == 'skipped' @@ -397,13 +409,13 @@ jobs: - name: Alert Slack if: > - github.event_name == 'schedule' + github.event_name == 'workflow_call' && failure() uses: rtCamp/action-slack-notify@v2 env: SLACK_CHANNEL: "covalent-ci" SLACK_USERNAME: "CovalentOpsBot" - SLACK_MESSAGE: "The tests.yml workflow is failing in develop!" + SLACK_MESSAGE: "The tests.yml workflow is failing in the last '${{github.event_name}}' event run!" SLACK_COLOR: ${{ job.status }} SLACK_TITLE: ":warning: Attention Required :warning:" SLACK_WEBHOOK: ${{ secrets.SLACK_ALERT_WEBHOOK }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 17cf7a24d..ce526d4da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Operations + +- Added qelectron tests to the `tests` workflow +- Split the `nightly` workflow into 4 manually triggerable workflows, `nightly-tests`, `man_0_assign_version`, `man_1_push_to_master`, and `man_2_create_prerelease` to be run in this order. +- Now only the `nightly-tests` workflow will be run on a daily basis, and the other 3 workflows will be run manually. +- Removed `conda` releases from `release.yml`. +- When pushing to `master`, now the version numbers of `develop` and `master` will be compared in `man_1_push_to_master`. +- Upgraded checkout action to v4 in `release.yml`. + ### Added - Added CRM method to handle Python to TF value conversion (e.g. None->null, True->true, False->false). @@ -36,10 +45,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed the SQLAlchemy warning that used to show up at every fresh server start - Fixed deploy commands' default value of plugins not being propagated to the tfvars file -### Operations - -- Added qelectron tests to the `tests` workflow - ## [0.233.0-rc.0] - 2024-01-07 ### Authors From 88058e57e8e2681cff20d93c9aa8206d2431cb46 Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Wed, 7 Feb 2024 08:31:10 -0500 Subject: [PATCH 04/12] Fixing the github workflows (#1927) * fixing if condition * fixing if condition * now added release as part of nightly tests * updated changelog --- .github/workflows/man_0_assign_version.yml | 9 +- .github/workflows/man_1_push_to_master.yml | 9 +- .github/workflows/man_2_create_prerelease.yml | 11 ++- .github/workflows/nightly-tests.yml | 26 +++++- .github/workflows/tests.yml | 88 +++++++------------ CHANGELOG.md | 2 + 6 files changed, 84 insertions(+), 61 deletions(-) diff --git a/.github/workflows/man_0_assign_version.yml b/.github/workflows/man_0_assign_version.yml index b9ce3d410..8863caf16 100644 --- a/.github/workflows/man_0_assign_version.yml +++ b/.github/workflows/man_0_assign_version.yml @@ -26,6 +26,13 @@ on: description: "WARNING: Make sure the `nightly-tests` or the most recent `tests` workflow has passed successfully in develop before running this workflow. Uncheck this box if it has." + workflow_call: + inputs: + nightly_tests_failed: + type: boolean + required: true + default: false + permissions: id-token: write contents: read @@ -40,6 +47,6 @@ jobs: needs: - license if: > - !github.event.inputs.nightly_tests_failed + !inputs.nightly_tests_failed uses: ./.github/workflows/changelog.yml secrets: inherit # pragma: allowlist secret diff --git a/.github/workflows/man_1_push_to_master.yml b/.github/workflows/man_1_push_to_master.yml index efe6d4db4..80db54dc3 100644 --- a/.github/workflows/man_1_push_to_master.yml +++ b/.github/workflows/man_1_push_to_master.yml @@ -26,6 +26,13 @@ on: description: "WARNING: Make sure the `man_0_assign_version` workflow has passed successfully before running this workflow. Uncheck this box if it has." + workflow_call: + inputs: + assign_version_failed: + type: boolean + required: true + default: false + permissions: id-token: write contents: read @@ -55,7 +62,7 @@ jobs: - name: Perform the push to master if develop is ahead id: push if: > - !github.event.inputs.assign_version_failed + !inputs.assign_version_failed run: | DEVELOP_VERSION="${{ steps.get-versions.outputs.develop_version }}" MASTER_VERSION="${{ steps.get-versions.outputs.master_version }}" diff --git a/.github/workflows/man_2_create_prerelease.yml b/.github/workflows/man_2_create_prerelease.yml index e982b8b37..c6c214870 100644 --- a/.github/workflows/man_2_create_prerelease.yml +++ b/.github/workflows/man_2_create_prerelease.yml @@ -26,6 +26,13 @@ on: description: "WARNING: Make sure the `man_1_push_to_master` workflow has passed successfully before running this workflow. Uncheck this box if it has." + workflow_call: + inputs: + push_to_master_failed: + type: boolean + required: true + default: false + permissions: id-token: write contents: read @@ -35,7 +42,7 @@ jobs: name: Create a Prerelease uses: ./.github/workflows/release.yml if: > - !github.event.inputs.push_to_master_failed + !inputs.push_to_master_failed with: prerelease: true secrets: inherit # pragma: allowlist secret @@ -49,7 +56,7 @@ jobs: - name: Checkout master uses: actions/checkout@v4 with: - ref: ${{ github.event.inputs.branch_name }} + ref: ${{ inputs.branch_name }} - name: Format Slack message run: | diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index da7becdf1..85d507218 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -17,9 +17,6 @@ name: nightly-tests on: - push: - branches: - - develop schedule: - cron: "0 0 * * *" @@ -36,5 +33,28 @@ jobs: tests: name: Unit and Functional Tests + needs: + - license uses: ./.github/workflows/tests.yml secrets: inherit # pragma: allowlist secret + + assign_version: + name: Assign Version + needs: + - tests + uses: ./.github/workflows/assign_version.yml + secrets: inherit # pragma: allowlist secret + + push_to_master: + name: Push to Master + needs: + - assign_version + uses: ./.github/workflows/push_to_master.yml + secrets: inherit # pragma: allowlist secret + + create_prerelease: + name: Create a Prerelease + needs: + - push_to_master + uses: ./.github/workflows/create_prerelease.yml + secrets: inherit # pragma: allowlist secret diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b66ce1ebb..58189ba96 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -308,89 +308,69 @@ jobs: - name: Dump Covalent logs run: covalent logs - - name: Upload SDK report to Codecov with retry + - name: Upload SDK report to Codecov id: upload-sdk-report if: > env.RECOMMENDED_PLATFORM && (github.event_name == 'workflow_call' || steps.sdk-coverage.outcome == 'success') - uses: Wandalen/wretry.action@master + uses: codecov/codecov-action@v3 with: - action: codecov/codecov-action@v3 - with: | - files: ./sdk_coverage.xml - flags: SDK - name: "SDK Unit Tests" - fail_ci_if_error: true - attempt_limit: 5 - attempt_delay: 5000 - - - name: Upload Dispatcher report to Codecov with retry + files: ./sdk_coverage.xml + flags: SDK + name: "SDK Unit Tests" + fail_ci_if_error: true + + - name: Upload Dispatcher report to Codecov id: upload-dispatcher-report if: > env.RECOMMENDED_PLATFORM && (github.event_name == 'workflow_call' || steps.dispatcher-coverage.outcome == 'success') - uses: Wandalen/wretry.action@master + uses: codecov/codecov-action@v3 with: - action: codecov/codecov-action@v3 - with: | - files: ./dispatcher_coverage.xml - flags: Dispatcher - name: "Dispatcher Unit Tests" - fail_ci_if_error: true - attempt_limit: 5 - attempt_delay: 5000 - - - name: Upload Functional report to Codecov with retry + files: ./dispatcher_coverage.xml + flags: Dispatcher + name: "Dispatcher Unit Tests" + fail_ci_if_error: true + + - name: Upload Functional report to Codecov id: upload-functional-report if: > env.RECOMMENDED_PLATFORM && steps.functional-coverage.outcome == 'success' - uses: Wandalen/wretry.action@master + uses: codecov/codecov-action@v3 with: - action: codecov/codecov-action@v3 - with: | - files: ./functional_tests_coverage.xml - flags: Functional_Tests - name: "Functional Tests" - fail_ci_if_error: true - attempt_limit: 5 - attempt_delay: 5000 - - - name: Upload UI backend report to Codecov with retry + files: ./functional_tests_coverage.xml + flags: Functional_Tests + name: "Functional Tests" + fail_ci_if_error: true + + - name: Upload UI backend report to Codecov id: upload-ui-backend-report if: > env.RECOMMENDED_PLATFORM && (github.event_name == 'workflow_call' || steps.ui-backend-coverage.outcome == 'success') - uses: Wandalen/wretry.action@master + uses: codecov/codecov-action@v3 with: - action: codecov/codecov-action@v3 - with: | - files: ./ui_backend_coverage.xml - flags: UI_Backend - name: "UI Backend Unit Tests" - fail_ci_if_error: true - attempt_limit: 5 - attempt_delay: 5000 - - - name: Upload UI frontend report to Codecov with retry + files: ./ui_backend_coverage.xml + flags: UI_Backend + name: "UI Backend Unit Tests" + fail_ci_if_error: true + + - name: Upload UI frontend report to Codecov id: upload-ui-frontend-report if: > env.RECOMMENDED_PLATFORM && (github.event_name == 'workflow_call' || steps.ui-frontend-tests.outcome == 'success') - uses: Wandalen/wretry.action@master + uses: codecov/codecov-action@v3 with: - action: codecov/codecov-action@v3 - with: | - files: ./covalent_ui/webapp/coverage/clover.xml - flags: UI_Frontend - name: "UI Frontend Unit Tests" - fail_ci_if_error: true - attempt_limit: 5 - attempt_delay: 5000 + files: ./covalent_ui/webapp/coverage/clover.xml + flags: UI_Frontend + name: "UI Frontend Unit Tests" + fail_ci_if_error: true - name: Local Codecov id: local-codecov diff --git a/CHANGELOG.md b/CHANGELOG.md index ce526d4da..9f0c12d99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed `conda` releases from `release.yml`. - When pushing to `master`, now the version numbers of `develop` and `master` will be compared in `man_1_push_to_master`. - Upgraded checkout action to v4 in `release.yml`. +- Fixing the if condition for the manual workflows. +- Added pre-release creation as part of `nightly-tests` workflow. ### Added From a306292279c9319721b0fc61a7a7355529a25c62 Mon Sep 17 00:00:00 2001 From: CovalentOpsBot Date: Wed, 7 Feb 2024 13:33:34 +0000 Subject: [PATCH 05/12] The new version will be 0.234.0-rc.0 --- CHANGELOG.md | 20 ++++++++++++++++++++ VERSION | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0c12d99..40f037d97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +## [0.234.0-rc.0] - 2024-02-07 + +### Authors + +- Andrew S. Rosen +- Casey Jao +- Sankalp Sanand +- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> +- ArunPsiog <106462226+ArunPsiog@users.noreply.github.com> +- Co-authored-by: Ara Ghukasyan +- FilipBolt +- sriranjanivenkatesan <116076079+sriranjanivenkatesan@users.noreply.github.com> +- Co-authored-by: batchumanish +- Co-authored-by: Prasy12 +- Co-authored-by: batchumanish <“manish.batchu@psiog.com”> +- Co-authored-by: batchumanish <126003896+batchumanish@users.noreply.github.com> +- Co-authored-by: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> +- Ara Ghukasyan <38226926+araghukas@users.noreply.github.com> + + ### Operations - Added qelectron tests to the `tests` workflow diff --git a/VERSION b/VERSION index 3fea2add4..19b9402f4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.233.0-rc.0 \ No newline at end of file +0.234.0-rc.0 \ No newline at end of file From 43f2bb15f03cadc69677fa9c68069a85567e85fe Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Thu, 8 Feb 2024 09:57:54 -0800 Subject: [PATCH 06/12] Removing my email from the CHANGELOG (#1928) --- CHANGELOG.md | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40f037d97..d1b321496 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Authors -- Andrew S. Rosen +- Andrew S. Rosen (@Andrew_S_Rosen) - Casey Jao - Sankalp Sanand - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> @@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Co-authored-by: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> - Ara Ghukasyan <38226926+araghukas@users.noreply.github.com> - ### Operations - Added qelectron tests to the `tests` workflow @@ -71,14 +70,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Authors -- Andrew S. Rosen +- Andrew S. Rosen (@Andrew_S_Rosen) - Casey Jao - Sankalp Sanand - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> - ArunPsiog <106462226+ArunPsiog@users.noreply.github.com> - Co-authored-by: Ara Ghukasyan - ### Added - Added feature to use custom python files as modules to be used in the electron function @@ -96,7 +94,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pack deps, call_before, and call_after assets into one file. - Changed handling of tuples and sets when building the transport graph - they will be converted to electron lists as well for now - `qelectron_db`, `qelectron_data_exists`, `python_version`, and `covalent_version` - are now optional in the pydantic model definitions. + are now optional in the pydantic model definitions. ### Fixed @@ -116,7 +114,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ara Ghukasyan <38226926+araghukas@users.noreply.github.com> - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> - ### Operations - Ignore custom executor plugin in how-to's when running `test_deploy_status` CLI test. @@ -150,7 +147,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Authors -- Andrew S. Rosen +- Andrew S. Rosen (@Andrew_S_Rosen) - Co-authored-by: Will Cunningham - Co-authored-by: Sankalp Sanand - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> @@ -177,8 +174,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Co-authored-by: Ara Ghukasyan - Co-authored-by: Alejandro Esquivel - - ### Added - Programmatic equivalents of CLI commands `covalent start` and `covalent stop` @@ -196,7 +191,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Contributing guidelines steps for installing for the first time - Updated gitignore to ignore yarn files and folders for latest version of yarn - Fixed the bug that caused ValueError error when using KEYWORD_ONLY parameter in electron func -- Changed code at line 218 in covalent/_shared_files/utils.py +- Changed code at line 218 in covalent/\_shared_files/utils.py - Fixed usage of deprecated pydantic validation methods - Fixed qelectron_db retrieval in result object - Fixed editability of Qelectron on settings page - UI changes @@ -257,7 +252,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Authors -- Andrew S. Rosen +- Andrew S. Rosen (@Andrew_S_Rosen) - Alejandro Esquivel - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> - Co-authored-by: mpvgithub <107603631+mpvgithub@users.noreply.github.com> @@ -316,7 +311,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Authors -- Andrew S. Rosen +- Andrew S. Rosen (@Andrew_S_Rosen) - Co-authored-by: Sankalp Sanand - Will Cunningham - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> @@ -406,7 +401,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Authors -- Andrew S. Rosen +- Andrew S. Rosen (@Andrew_S_Rosen) - Co-authored-by: Sankalp Sanand - Will Cunningham - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> @@ -448,7 +443,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prasanna Venkatesh <54540812+Prasy12@users.noreply.github.com> - Co-authored-by: kamalesh.suresh - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> -- Co-authored-by: Andrew S. Rosen +- Co-authored-by: Andrew S. Rosen (@Andrew_S_Rosen) - Faiyaz Hasan - Co-authored-by: sriranjani venkatesan - Will Cunningham @@ -469,7 +464,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prasanna Venkatesh <54540812+Prasy12@users.noreply.github.com> - Co-authored-by: kamalesh.suresh - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> -- Co-authored-by: Andrew S. Rosen +- Co-authored-by: Andrew S. Rosen (@Andrew_S_Rosen) - Faiyaz Hasan - Co-authored-by: sriranjani venkatesan - Will Cunningham @@ -531,7 +526,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Sankalp Sanand - Co-authored-by: kessler-frost - Faiyaz Hasan -- Andrew S. Rosen +- Andrew S. Rosen (@Andrew_S_Rosen) - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> - Co-authored-by: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> From 0ac14767db7e487ae90d920e9c685dcaf5d56097 Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Mon, 12 Feb 2024 10:53:24 -0500 Subject: [PATCH 07/12] Fixing scheduled nightly (#1930) --- .github/workflows/nightly-tests.yml | 6 +++--- CHANGELOG.md | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index 85d507218..36fad3bf8 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -42,19 +42,19 @@ jobs: name: Assign Version needs: - tests - uses: ./.github/workflows/assign_version.yml + uses: ./.github/workflows/man_0_assign_version.yml secrets: inherit # pragma: allowlist secret push_to_master: name: Push to Master needs: - assign_version - uses: ./.github/workflows/push_to_master.yml + uses: ./.github/workflows/man_1_push_to_master.yml secrets: inherit # pragma: allowlist secret create_prerelease: name: Create a Prerelease needs: - push_to_master - uses: ./.github/workflows/create_prerelease.yml + uses: ./.github/workflows/man_2_create_prerelease.yml secrets: inherit # pragma: allowlist secret diff --git a/CHANGELOG.md b/CHANGELOG.md index d1b321496..03b630fa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Operations + +- Fixed nightly worfkflow's calling of other workflows. + ## [0.234.0-rc.0] - 2024-02-07 ### Authors From 0f212cb896d3cc7e00edb6cb12f314dd4b860f28 Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Tue, 13 Feb 2024 13:27:09 -0500 Subject: [PATCH 08/12] Removing author email from changelog (#1929) * removing author email from changelog action * updated changelog * implemented suggestions --- .github/actions/changelog/index.js | 8 +++----- CHANGELOG.md | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/actions/changelog/index.js b/.github/actions/changelog/index.js index d72726572..a80432356 100644 --- a/.github/actions/changelog/index.js +++ b/.github/actions/changelog/index.js @@ -6,16 +6,14 @@ const readline = require("readline"); const check_author = (author, authors) => { if ( - typeof author === "object" && - "email" in author && - !authors.includes(author.email) + typeof author === "object" ) { - return "- " + author.name + " <" + author.email + ">\n"; + return "- " + author.name + "\n"; } else if ( typeof author === "string" && !authors.includes(author.split(/[<>]/)[1]) ) { - return "- " + author + "\n"; + return "- " + author.split(/[<>]/)[1].trim() + "\n"; } else { return ""; } diff --git a/CHANGELOG.md b/CHANGELOG.md index 03b630fa2..4454374cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Operations +- Removing author email from changelog action - Fixed nightly worfkflow's calling of other workflows. ## [0.234.0-rc.0] - 2024-02-07 From 170f4d696c34f38bbdcd20f6617328720f5799f5 Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Tue, 13 Feb 2024 18:18:19 -0500 Subject: [PATCH 09/12] Further scheduled nightly fixes (#1934) * added input values * updated changelog --------- Co-authored-by: Alejandro Esquivel --- .github/workflows/nightly-tests.yml | 6 ++++++ CHANGELOG.md | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index 36fad3bf8..df31afce7 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -44,6 +44,8 @@ jobs: - tests uses: ./.github/workflows/man_0_assign_version.yml secrets: inherit # pragma: allowlist secret + with: + nightly_tests_failed: false push_to_master: name: Push to Master @@ -51,6 +53,8 @@ jobs: - assign_version uses: ./.github/workflows/man_1_push_to_master.yml secrets: inherit # pragma: allowlist secret + with: + assign_version_failed: false create_prerelease: name: Create a Prerelease @@ -58,3 +62,5 @@ jobs: - push_to_master uses: ./.github/workflows/man_2_create_prerelease.yml secrets: inherit # pragma: allowlist secret + with: + push_to_master_failed: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 4454374cb..5a928eabf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Operations +- Fixed nightly workflow's calling of other workflows. +- Fixed input values for other workflows in `nightly-tests` workflow. + +### Operations + - Removing author email from changelog action - Fixed nightly worfkflow's calling of other workflows. From e6ad647673171bbf387d37af0e822df00fb7e1aa Mon Sep 17 00:00:00 2001 From: Casey Jao Date: Tue, 27 Feb 2024 21:51:45 -0500 Subject: [PATCH 10/12] Parse electron function strings at Electron construction (#1926) * Parse function string in Electron constructor Waiting until `build_graph()` won't work if the graph is being built in a remote executor without the original source code. * Changelog --------- Co-authored-by: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> --- CHANGELOG.md | 4 +++ covalent/_workflow/electron.py | 31 ++----------------- .../covalent_tests/workflow/electron_test.py | 21 +------------ 3 files changed, 8 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a928eabf..17344263c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Fixed + +- Sublattice electron function strings are now parsed correctly + ### Operations - Fixed nightly workflow's calling of other workflows. diff --git a/covalent/_workflow/electron.py b/covalent/_workflow/electron.py index 12f18cbf5..a8a9055be 100644 --- a/covalent/_workflow/electron.py +++ b/covalent/_workflow/electron.py @@ -96,6 +96,7 @@ def __init__( self.metadata = metadata self.task_group_id = task_group_id self._packing_tasks = packing_tasks + self._function_string = get_serialized_function_str(function) @property def packing_tasks(self) -> bool: @@ -442,7 +443,7 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]: ) name = sublattice_prefix + self.function.__name__ - function_string = get_serialized_function_str(self.function) + function_string = self._function_string bound_electron = sub_electron( self.function, json.dumps(parent_metadata), *args, **kwargs ) @@ -463,7 +464,7 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]: name=self.function.__name__, function=self.function, metadata=self.metadata.copy(), - function_string=get_serialized_function_str(self.function), + function_string=self._function_string, task_group_id=self.task_group_id if self.packing_tasks else None, ) self.task_group_id = self.task_group_id if self.packing_tasks else self.node_id @@ -608,32 +609,6 @@ def _auto_dict_node(*args, **kwargs): arg_index=arg_index, ) - def add_collection_node_to_graph(self, graph: "_TransportGraph", prefix: str) -> int: - """ - Adds the node to lattice's transport graph in the case - where a collection of electrons is passed as an argument - to another electron. - - Args: - graph: Transport graph of the lattice - prefix: Prefix of the node - - Returns: - node_id: Node id of the added node - """ - - new_metadata = encode_metadata(DEFAULT_METADATA_VALUES.copy()) - if "executor" in self.metadata: - new_metadata["executor"] = self.metadata["executor"] - new_metadata["executor_data"] = self.metadata["executor_data"] - - node_id = graph.add_node( - name=prefix, - function=to_decoded_electron_collection, - metadata=new_metadata, - function_string=get_serialized_function_str(to_decoded_electron_collection), - ) - return node_id def wait_for(self, electrons: Union["Electron", Iterable["Electron"]]): diff --git a/tests/covalent_tests/workflow/electron_test.py b/tests/covalent_tests/workflow/electron_test.py index a3db76bb7..d7a3e3192 100644 --- a/tests/covalent_tests/workflow/electron_test.py +++ b/tests/covalent_tests/workflow/electron_test.py @@ -36,7 +36,7 @@ to_decoded_electron_collection, ) from covalent._workflow.lattice import Lattice -from covalent._workflow.transport import TransportableObject, _TransportGraph, encode_metadata +from covalent._workflow.transport import TransportableObject, encode_metadata from covalent.executor.executor_plugins.local import LocalExecutor @@ -252,25 +252,6 @@ def test_collection_node_helper_electron(): assert to_decoded_electron_collection(x=dict_collection) == {"a": 1, "b": 2} -def test_electron_add_collection_node(): - """Test `to_decoded_electron_collection` in `Electron.add_collection_node`""" - - def f(x): - return x - - e = Electron(f) - tg = _TransportGraph() - node_id = e.add_collection_node_to_graph(tg, prefix=":") - collection_fn = tg.get_node_value(node_id, "function").get_deserialized() - - collection = [ - TransportableObject.make_transportable(1), - TransportableObject.make_transportable(2), - ] - - assert collection_fn(x=collection) == [1, 2] - - def test_injected_inputs_are_not_in_tg(): """Test that arguments to electrons injected by calldeps aren't added to the transport graph""" From d7841c7d0eb8f113efca1834fdb780b48ef63854 Mon Sep 17 00:00:00 2001 From: Casey Jao Date: Wed, 28 Feb 2024 09:24:36 -0500 Subject: [PATCH 11/12] Do not require dictionary keys to be `str` (#1886) * Unbreak task packing When submitting a task group, only attempt to upload task inputs corresponding to nodes external to the task group since only those will have been resolved. * Lock parent job record when persisting sublattices * Dictionary collector nodes no longer need keys to be `str` The collector electron now assembles the dictionary from two lists -- one list of keys, one list of corresponding values. * Fix tests * Changelog --- .github/workflows/tests.yml | 3 +- CHANGELOG.md | 2 + covalent/_workflow/electron.py | 6 +-- covalent_dispatcher/_core/dispatcher.py | 15 ++++++- covalent_dispatcher/_dal/importers/result.py | 2 +- covalent_dispatcher/_dal/importers/tg.py | 4 +- .../_core/dispatcher_db_integration_test.py | 27 +++++++++-- .../_core/execution_test.py | 28 +++++++++--- .../_dal/importers/result_import_test.py | 45 +++++++++++++++++++ .../covalent_tests/workflow/electron_test.py | 28 ++++++++---- tests/functional_tests/workflow_stack_test.py | 3 +- 11 files changed, 137 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 58189ba96..1349a02ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -217,9 +217,10 @@ jobs: if: env.BUILD_AND_RUN_ALL id: covalent_start run: | + export COVALENT_ENABLE_TASK_PACKING=1 covalent db migrate if [ "${{ matrix.backend }}" = 'dask' ] ; then - COVALENT_ENABLE_TASK_PACKING=1 covalent start -d + covalent start -d elif [ "${{ matrix.backend }}" = 'local' ] ; then covalent start --no-cluster -d else diff --git a/CHANGELOG.md b/CHANGELOG.md index 17344263c..76c2a9720 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Sublattice electron function strings are now parsed correctly +- The keys of dictionary inputs to electrons no longer need be strings. +- Fixed inaccuracies in task packing exposed by no longer uploading null attributes upon dispatch. ### Operations diff --git a/covalent/_workflow/electron.py b/covalent/_workflow/electron.py index a8a9055be..0e80f0a22 100644 --- a/covalent/_workflow/electron.py +++ b/covalent/_workflow/electron.py @@ -572,8 +572,8 @@ def _auto_list_node(*args, **kwargs): elif isinstance(param_value, dict): - def _auto_dict_node(*args, **kwargs): - return dict(kwargs) + def _auto_dict_node(keys, values): + return {keys[i]: values[i] for i in range(len(keys))} dict_electron = Electron( function=_auto_dict_node, @@ -581,7 +581,7 @@ def _auto_dict_node(*args, **kwargs): task_group_id=self.task_group_id, packing_tasks=True and active_lattice.task_packing, ) # Group the auto-generated node with the main node. - bound_electron = dict_electron(**param_value) + bound_electron = dict_electron(list(param_value.keys()), list(param_value.values())) transport_graph.set_node_value(bound_electron.node_id, "name", electron_dict_prefix) transport_graph.add_edge( dict_electron.node_id, diff --git a/covalent_dispatcher/_core/dispatcher.py b/covalent_dispatcher/_core/dispatcher.py index e17547969..b0ecd27b6 100644 --- a/covalent_dispatcher/_core/dispatcher.py +++ b/covalent_dispatcher/_core/dispatcher.py @@ -183,6 +183,7 @@ async def _submit_task_group(dispatch_id: str, sorted_nodes: List[int], task_gro app_log.debug("8A: Update node success (run_planned_workflow).") else: + # Nodes whose values have already been resolved known_nodes = [] # Skip the group if all task outputs can be reused from a @@ -196,6 +197,8 @@ async def _submit_task_group(dispatch_id: str, sorted_nodes: List[int], task_gro # Gather inputs for each task and send the task spec sequence to the runner task_specs = [] + sorted_nodes_set = set(sorted_nodes) + for node_id in sorted_nodes: app_log.debug(f"Gathering inputs for task {node_id} (run_planned_workflow).") @@ -214,8 +217,16 @@ async def _submit_task_group(dispatch_id: str, sorted_nodes: List[int], task_gro "args_ids": abs_task_input["args"], "kwargs_ids": abs_task_input["kwargs"], } - known_nodes += abs_task_input["args"] - known_nodes += list(abs_task_input["kwargs"].values()) + # Task inputs that don't belong to the task group have already beeen resolved + external_task_args = filter( + lambda x: x not in sorted_nodes_set, abs_task_input["args"] + ) + known_nodes.extend(external_task_args) + external_task_kwargs = filter( + lambda x: x not in sorted_nodes_set, abs_task_input["kwargs"].values() + ) + known_nodes.extend(external_task_kwargs) + task_specs.append(task_spec) app_log.debug( diff --git a/covalent_dispatcher/_dal/importers/result.py b/covalent_dispatcher/_dal/importers/result.py index 395516b86..7e4bd36f9 100644 --- a/covalent_dispatcher/_dal/importers/result.py +++ b/covalent_dispatcher/_dal/importers/result.py @@ -72,7 +72,6 @@ def import_result( # Main case: insert new lattice, electron, edge, and job records storage_path = os.path.join(base_path, dispatch_id) - os.makedirs(storage_path) lattice_record_kwargs = _get_result_meta(res, storage_path, electron_id) lattice_record_kwargs.update(_get_lattice_meta(res.lattice, storage_path)) @@ -143,6 +142,7 @@ def _connect_result_to_electron( fields={"id", "cancel_requested"}, equality_filters={"id": parent_electron_record.job_id}, membership_filters={}, + for_update=True, )[0] cancel_requested = parent_job_record.cancel_requested diff --git a/covalent_dispatcher/_dal/importers/tg.py b/covalent_dispatcher/_dal/importers/tg.py index 468abdadf..c67cc34b9 100644 --- a/covalent_dispatcher/_dal/importers/tg.py +++ b/covalent_dispatcher/_dal/importers/tg.py @@ -51,7 +51,9 @@ def import_transport_graph( # Propagate parent electron id's `cancel_requested` property to the sublattice electrons if electron_id is not None: parent_e_record = Electron.meta_type.get_by_primary_key(session, electron_id) - job_record = Job.get_by_primary_key(session=session, primary_key=parent_e_record.job_id) + job_record = Job.get_by_primary_key( + session=session, primary_key=parent_e_record.job_id, for_update=True + ) cancel_requested = job_record.cancel_requested else: cancel_requested = False diff --git a/tests/covalent_dispatcher_tests/_core/dispatcher_db_integration_test.py b/tests/covalent_dispatcher_tests/_core/dispatcher_db_integration_test.py index 53444a7b6..1544b753c 100644 --- a/tests/covalent_dispatcher_tests/_core/dispatcher_db_integration_test.py +++ b/tests/covalent_dispatcher_tests/_core/dispatcher_db_integration_test.py @@ -104,7 +104,7 @@ def list_workflow(arg): @ct.lattice def dict_workflow(arg): - return dict_task(arg) + return dict_task(arg=arg) # 1 2 # \ \ @@ -159,7 +159,7 @@ async def mock_get_incoming_edges(dispatch_id, node_id): # dict-type inputs - # Nodes 0=task, 1=:electron_dict:, 2=1, 3=2 + # Nodes 0=task, 1=:electron_dict:, 2=["a" (3), "b" (4)], 5=[1 (6), 2 (7)] dict_workflow.build_graph({"a": 1, "b": 2}) abstract_args = {"a": 2, "b": 3} tg = dict_workflow.transport_graph @@ -172,10 +172,31 @@ async def mock_get_incoming_edges(dispatch_id, node_id): mock_get_incoming_edges, ) + task_inputs = await _get_abstract_task_inputs( + result_object.dispatch_id, 0, tg.get_node_value(0, "name") + ) + expected_inputs = {"args": [], "kwargs": {"arg": 1}} + + assert task_inputs == expected_inputs + task_inputs = await _get_abstract_task_inputs( result_object.dispatch_id, 1, tg.get_node_value(1, "name") ) - expected_inputs = {"args": [], "kwargs": abstract_args} + expected_inputs = {"args": [2, 5], "kwargs": {}} + + assert task_inputs == expected_inputs + + task_inputs = await _get_abstract_task_inputs( + result_object.dispatch_id, 2, tg.get_node_value(2, "name") + ) + expected_inputs = {"args": [3, 4], "kwargs": {}} + + assert task_inputs == expected_inputs + + task_inputs = await _get_abstract_task_inputs( + result_object.dispatch_id, 5, tg.get_node_value(5, "name") + ) + expected_inputs = {"args": [6, 7], "kwargs": {}} assert task_inputs == expected_inputs diff --git a/tests/covalent_dispatcher_tests/_core/execution_test.py b/tests/covalent_dispatcher_tests/_core/execution_test.py index 6d521691f..4e2c20ac9 100644 --- a/tests/covalent_dispatcher_tests/_core/execution_test.py +++ b/tests/covalent_dispatcher_tests/_core/execution_test.py @@ -116,7 +116,7 @@ def list_workflow(arg): @ct.lattice def dict_workflow(arg): - return dict_task(arg) + return dict_task(arg=arg) # 1 2 # \ \ @@ -167,20 +167,36 @@ def multivar_workflow(x, y): # dict-type inputs dict_workflow.build_graph({"a": 1, "b": 2}) - serialized_args = {"a": ct.TransportableObject(1), "b": ct.TransportableObject(2)} # Nodes 0=task, 1=:electron_dict:, 2=1, 3=2 + # Nodes 0=task, 1=:electron_dict:, 2=["a" (3), "b" (4)], 5=[1 (6), 2 (7)] + sdkres = Result(lattice=dict_workflow, dispatch_id="asdf_dict_workflow") result_object = get_mock_srvresult(sdkres, test_db) tg = result_object.lattice.transport_graph - tg.set_node_value(2, "output", ct.TransportableObject(1)) - tg.set_node_value(3, "output", ct.TransportableObject(2)) + + tg.set_node_value(1, "output", ct.TransportableObject("node_1_output")) + tg.set_node_value(3, "output", ct.TransportableObject("a")) + tg.set_node_value(4, "output", ct.TransportableObject("b")) + tg.set_node_value(6, "output", ct.TransportableObject(1)) + tg.set_node_value(7, "output", ct.TransportableObject(2)) mock_get_result = mocker.patch( "covalent_dispatcher._core.runner.datasvc.get_result_object", return_value=result_object ) - task_inputs = await _get_task_inputs(1, tg.get_node_value(1, "name"), result_object) - expected_inputs = {"args": [], "kwargs": serialized_args} + serialized_kwargs = {"arg": ct.TransportableObject("node_1_output")} + task_inputs = await _get_task_inputs(0, tg.get_node_value(0, "name"), result_object) + expected_inputs = {"args": [], "kwargs": serialized_kwargs} + + serialized_args = [ct.TransportableObject("a"), ct.TransportableObject("b")] + task_inputs = await _get_task_inputs(2, tg.get_node_value(2, "name"), result_object) + expected_inputs = {"args": serialized_args, "kwargs": {}} + + assert task_inputs == expected_inputs + + serialized_args = [ct.TransportableObject(1), ct.TransportableObject(2)] + task_inputs = await _get_task_inputs(5, tg.get_node_value(5, "name"), result_object) + expected_inputs = {"args": serialized_args, "kwargs": {}} assert task_inputs == expected_inputs diff --git a/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py b/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py index 440742cba..819f88bc6 100644 --- a/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py +++ b/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py @@ -27,6 +27,7 @@ from covalent._shared_files.schemas.result import AssetSchema, ResultSchema from covalent._shared_files.util_classes import RESULT_STATUS from covalent_dispatcher._dal.importers.result import SERVER_URL, handle_redispatch, import_result +from covalent_dispatcher._dal.job import Job from covalent_dispatcher._dal.result import get_result_object from covalent_dispatcher._db.datastore import DataStore @@ -140,6 +141,7 @@ def test_import_previously_imported_result(mocker, test_db): prefix="covalent-" ) as srv_dir: sub_res = get_mock_result(sub_dispatch_id, sdk_dir) + sub_res.metadata.root_dispatch_id = dispatch_id import_result(sub_res, srv_dir, None) srv_res = get_result_object(dispatch_id, bare=True) parent_node = srv_res.lattice.transport_graph.get_node(0) @@ -152,6 +154,49 @@ def test_import_previously_imported_result(mocker, test_db): assert sub_srv_res._electron_id == parent_node._electron_id +def test_import_subdispatch_cancel_req(mocker, test_db): + """Test that Job.cancel_requested is propagated to sublattices""" + + dispatch_id = "test_propagate_cancel_requested" + sub_dispatch_id = "test_propagate_cancel_requested_sub" + + mocker.patch("covalent_dispatcher._dal.base.workflow_db", test_db) + + mock_filter_uris = mocker.patch( + "covalent_dispatcher._dal.importers.result._filter_remote_uris" + ) + + with tempfile.TemporaryDirectory(prefix="covalent-") as sdk_dir, tempfile.TemporaryDirectory( + prefix="covalent-" + ) as srv_dir: + res = get_mock_result(dispatch_id, sdk_dir) + import_result(res, srv_dir, None) + + with test_db.Session() as session: + Job.update_bulk( + session, values={"cancel_requested": True}, equality_filters={}, membership_filters={} + ) + session.commit() + + with tempfile.TemporaryDirectory(prefix="covalent-") as sdk_dir, tempfile.TemporaryDirectory( + prefix="covalent-" + ) as srv_dir: + sub_res = get_mock_result(sub_dispatch_id, sdk_dir) + sub_res.metadata.root_dispatch_id = dispatch_id + srv_res = get_result_object(dispatch_id, bare=True) + parent_node = srv_res.lattice.transport_graph.get_node(0) + import_result(sub_res, srv_dir, parent_node._electron_id) + + with tempfile.TemporaryDirectory(prefix="covalent-") as srv_dir: + import_result(sub_res, srv_dir, parent_node._electron_id) + + with test_db.Session() as session: + uncancelled = Job.get( + session, fields=[], equality_filters={"cancel_requested": False}, membership_filters={} + ) + assert len(uncancelled) == 0 + + @pytest.mark.parametrize( "parent_status,new_status", [ diff --git a/tests/covalent_tests/workflow/electron_test.py b/tests/covalent_tests/workflow/electron_test.py index d7a3e3192..2d5936ed3 100644 --- a/tests/covalent_tests/workflow/electron_test.py +++ b/tests/covalent_tests/workflow/electron_test.py @@ -377,18 +377,30 @@ def workflow(x): g = workflow.transport_graph._graph # Account for postprocessing node - assert list(g.nodes) == [0, 1, 2, 3, 4] + assert list(g.nodes) == [0, 1, 2, 3, 4, 5, 6, 7, 8] fn = g.nodes[1]["function"].get_deserialized() - assert fn(x=2, y=5, z=7) == {"x": 2, "y": 5, "z": 7} - assert g.nodes[2]["value"].get_deserialized() == 5 - assert g.nodes[3]["value"].get_deserialized() == 7 + assert fn(["x", "y", "z"], [2, 5, 7]) == {"x": 2, "y": 5, "z": 7} + fn = g.nodes[2]["function"].get_deserialized() + assert fn("x", "y") == ["x", "y"] + keys = [g.nodes[3]["value"].get_deserialized(), g.nodes[4]["value"].get_deserialized()] + fn = g.nodes[5]["function"].get_deserialized() + assert fn(2, 3) == [2, 3] + vals = [g.nodes[6]["value"].get_deserialized(), g.nodes[7]["value"].get_deserialized()] + assert keys == ["x", "y"] + assert vals == [5, 7] assert set(g.edges) == { (1, 0, 0), (2, 1, 0), - (3, 1, 0), - (0, 4, 0), - (0, 4, 1), - (1, 4, 0), + (3, 2, 0), + (4, 2, 0), + (5, 1, 0), + (6, 5, 0), + (7, 5, 0), + (0, 8, 0), + (0, 8, 1), + (1, 8, 0), + (2, 8, 0), + (5, 8, 0), } diff --git a/tests/functional_tests/workflow_stack_test.py b/tests/functional_tests/workflow_stack_test.py index 47e1578a7..f20b3a0f9 100644 --- a/tests/functional_tests/workflow_stack_test.py +++ b/tests/functional_tests/workflow_stack_test.py @@ -800,7 +800,8 @@ def workflow(x): res_1 = sum_values(x) return square(res_1) - dispatch_id = ct.dispatch(workflow)({"x": 1, "y": 2, "z": 3}) + # Check that non-string keys are allowed + dispatch_id = ct.dispatch(workflow)({"x": 1, "y": 2, 3: 3}) res_obj = rm.get_result(dispatch_id, wait=True) From 5486a1493cc91646e2e243acce5f0e1510b64420 Mon Sep 17 00:00:00 2001 From: CovalentOpsBot Date: Fri, 10 May 2024 12:13:31 +0000 Subject: [PATCH 12/12] The new version will be 0.234.1-rc.0 --- CHANGELOG.md | 11 +++++++++++ VERSION | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c2a9720..e3e6a1376 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +## [0.234.1-rc.0] - 2024-05-10 + +### Authors + +- Andrew S. Rosen +- Sankalp Sanand +- Co-authored-by: Alejandro Esquivel +- Casey Jao +- Co-authored-by: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> + + ### Fixed - Sublattice electron function strings are now parsed correctly diff --git a/VERSION b/VERSION index 19b9402f4..9d818f2d5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.234.0-rc.0 \ No newline at end of file +0.234.1-rc.0 \ No newline at end of file