diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8497d4f60..7637837ff 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -22,7 +22,7 @@ name: nightly on: schedule: - - cron: "0 4 * * *" + - cron: "0 */4 * * *" jobs: license: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d18a6c170..71a47751b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -215,9 +215,9 @@ jobs: run: | covalent db migrate if [ "${{ matrix.backend }}" = 'dask' ] ; then - covalent start + covalent start -d elif [ "${{ matrix.backend }}" = 'local' ] ; then - covalent start --no-cluster + covalent start --no-cluster -d else echo "Invalid backend specified in test matrix." exit 1 @@ -225,12 +225,22 @@ jobs: env: COVALENT_EXECUTOR_DIR: doc/source/how_to/execution/custom_executors + - name: Run functional tests and measure coverage + id: functional-tests + if: env.BUILD_AND_RUN_ALL + run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/functional_tests --cov=covalent --cov=covalent_dispatcher --cov-config=.coveragerc + + - name: Generate functional test coverage report + id: functional-coverage + if: steps.functional-tests.outcome == 'success' + run: coverage xml -o functional_tests_coverage.xml + - name: Run SDK tests and measure coverage id: sdk-tests if: > steps.modified-files.outputs.sdk == 'true' || env.BUILD_AND_RUN_ALL - run: PYTHONPATH=$PWD/ pytest -vv --reruns=5 tests/covalent_tests --cov=covalent --cov-config=.coveragerc + run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/covalent_tests --cov=covalent --cov-config=.coveragerc - name: Generate SDK coverage report id: sdk-coverage @@ -242,29 +252,19 @@ jobs: if: > steps.modified-files.outputs.dispatcher == 'true' || env.BUILD_AND_RUN_ALL - run: PYTHONPATH=$PWD/ pytest -vv --reruns=5 tests/covalent_dispatcher_tests --cov=covalent_dispatcher --cov-config=.coveragerc + run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/covalent_dispatcher_tests --cov=covalent_dispatcher --cov-config=.coveragerc - name: Generate dispatcher coverage report id: dispatcher-coverage if: steps.dispatcher-tests.outcome == 'success' run: coverage xml -o dispatcher_coverage.xml - - name: Run functional tests and measure coverage - id: functional-tests - if: env.BUILD_AND_RUN_ALL - run: PYTHONPATH=$PWD/ pytest -vv --reruns=5 tests/functional_tests --cov=covalent --cov=covalent_dispatcher --cov-config=.coveragerc - - - name: Generate functional test coverage report - id: functional-coverage - if: steps.functional-tests.outcome == 'success' - run: coverage xml -o functional_tests_coverage.xml - - name: Run UI backend tests and measure coverage id: ui-backend-tests if: > steps.modified-files.outputs.ui_backend == 'true' || env.BUILD_AND_RUN_ALL - run: PYTHONPATH=$PWD/ pytest -vv --reruns=5 tests/covalent_ui_backend_tests --cov=covalent_ui --cov-config=.coveragerc + run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/covalent_ui_backend_tests --cov=covalent_ui --cov-config=.coveragerc - name: Generate UI backend coverage report id: ui-backend-coverage @@ -283,11 +283,7 @@ jobs: - name: Dump Covalent logs if: > steps.covalent_start.outcome == 'success' - && (steps.sdk-tests.outcome == 'failed' - || steps.dispatcher-tests.outcome == 'failed' - || steps.functional-tests.outcome == 'failed' - || steps.ui-backend-tests.outcome == 'failed' - || steps.ui-frontend-tests.outcome == 'failed') + && failure() run: covalent logs - name: Upload SDK report to Codecov diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ed37d34..a0716afc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Added + +- Added a `py.typed` file to support type-checking + +### Changed + +- Raised the minimum version of Pydantic from 1.10.1 to 2.1.1 in `requirements.txt` + +## [0.228.0-rc.0] - 2023-08-31 + +### Authors + +- Andrew S. Rosen +- Co-authored-by: Sankalp Sanand +- Will Cunningham +- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> +- Co-authored-by: Casey Jao +- WingCode +- Nick Tyler +- Co-authored-by: Alejandro Esquivel +- Aravind <100823292+Aravind-psiog@users.noreply.github.com> +- Co-authored-by: Manjunath PV +- Co-authored-by: ArunPsiog +- Co-authored-by: RaviPsiog +- Co-authored-by: Prasy12 +- Co-authored-by: mpvgithub <107603631+mpvgithub@users.noreply.github.com> +- Co-authored-by: Will Cunningham +- dwelsch-esi <116022979+dwelsch-esi@users.noreply.github.com> +- Co-authored-by: dwelsch-memverge +- Co-authored-by: kessler-frost +- Faiyaz Hasan +- Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> + + +### Fixed + +- Fixed dispatcher address not showing when covalent server starts. +- Fixed the failing tests in the `nightly` workflow. + ### Operations @@ -17,6 +56,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Reduce the number of pinned version numbers in the `setup.py`, `requirements.txt`, and `requirements-client.txt` - Updated the `wci.yml` file with new features - Bumped pre-commit versions +- Temporarily running nightly hourly to test whether the fix worked +- Reverted to daily frequency for nightly ### Added @@ -42,7 +83,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed the upper limit from `dask` and `distributed` packages' versions until we find a version which is incompatible with Covalent. - When the server is stopped, any workflows in a non-terminal state are first cancelled - Pinned sqlalchemy version with upper limit <2.0.0. -- Added rich support to cli for better printing statements. +- Added rich support to cli for better printing statements. +- Performed minor modifications and rearrangementsto fix the broken tests in the `nightly` workflow. ### Tests diff --git a/MANIFEST.in b/MANIFEST.in index dcc69da71..742cda7d1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include VERSION include requirements.txt include requirements-client.txt +include covalent/py.typed recursive-include covalent/executor/executor_plugins/ * recursive-include covalent_dispatcher/_service/ * recursive-include covalent_migrations/ * diff --git a/VERSION b/VERSION index 2434e02d8..196b52d75 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.227.0-rc.0 \ No newline at end of file +0.228.0-rc.0 \ No newline at end of file diff --git a/covalent/_results_manager/wait.py b/covalent/_results_manager/wait.py index cbbb83022..43fac8167 100644 --- a/covalent/_results_manager/wait.py +++ b/covalent/_results_manager/wait.py @@ -1,3 +1,5 @@ +# Copyright 2021 Agnostiq Inc. +# # This file is part of Covalent. # # Licensed under the GNU Affero General Public License 3.0 (the "License"). @@ -16,6 +18,7 @@ # # Relief from the License may be granted by purchasing a commercial license. + import sys from dataclasses import dataclass @@ -28,6 +31,7 @@ def __int__(self) -> int: return self.RETRIES +SHORT = Wait(30) LONG = Wait(3000) VERY_LONG = Wait(60000) EXTREME = Wait(sys.maxsize) diff --git a/covalent/executor/executor_plugins/local.py b/covalent/executor/executor_plugins/local.py index 0a9ff3897..548c82901 100644 --- a/covalent/executor/executor_plugins/local.py +++ b/covalent/executor/executor_plugins/local.py @@ -58,6 +58,7 @@ "create_unique_workdir": False, } + proc_pool = ProcessPoolExecutor() diff --git a/covalent/py.typed b/covalent/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/covalent_dispatcher/_cli/service.py b/covalent_dispatcher/_cli/service.py index 24fd6415a..8645d88cb 100644 --- a/covalent_dispatcher/_cli/service.py +++ b/covalent_dispatcher/_cli/service.py @@ -468,7 +468,6 @@ def start( "Covalent was unable to start due to the following error: ", fg="red", bold=True ) click.secho(traceback.format_exc(), fg="lightgrey") - return ctx.exit(1) set_config("user_interface.port", port) set_config("dispatcher.port", port) @@ -503,6 +502,9 @@ def start( console.print(config_table) console.print("\nServer Status: [green]:heavy_check_mark:[/green] Running", style="bold") + dispatcher_address = f"http://{str(get_config('dispatcher.address'))}:{str(port)}" + console.print(f"\nCovalent UI can be accessed at {dispatcher_address}") + if not no_footer: console.print("\nFor a summary of the system status, use 'covalent status'") print_footer(console) diff --git a/covalent_dispatcher/_core/data_manager.py b/covalent_dispatcher/_core/data_manager.py index 0f015de00..d5e5a1910 100644 --- a/covalent_dispatcher/_core/data_manager.py +++ b/covalent_dispatcher/_core/data_manager.py @@ -339,7 +339,7 @@ def make_derived_dispatch( def get_result_object(dispatch_id: str) -> Result: - return _registered_dispatches[dispatch_id] + return _registered_dispatches.get(dispatch_id) def _register_result_object(result_object: Result): diff --git a/covalent_dispatcher/_core/dispatcher.py b/covalent_dispatcher/_core/dispatcher.py index 0d853c458..fdf7902ab 100644 --- a/covalent_dispatcher/_core/dispatcher.py +++ b/covalent_dispatcher/_core/dispatcher.py @@ -382,7 +382,11 @@ async def cancel_dispatch(dispatch_id: str, task_ids: List[int] = None) -> None: if not dispatch_id: return - tg = datasvc.get_result_object(dispatch_id=dispatch_id).lattice.transport_graph + res_object = datasvc.get_result_object(dispatch_id) + if res_object is None: + return + + tg = res_object.lattice.transport_graph if task_ids: app_log.debug(f"Cancelling tasks {task_ids} in dispatch {dispatch_id}") else: diff --git a/covalent_ui/heartbeat.py b/covalent_ui/heartbeat.py index 22d058e3d..f726a22c6 100644 --- a/covalent_ui/heartbeat.py +++ b/covalent_ui/heartbeat.py @@ -82,8 +82,8 @@ async def cancel_all_with_status(status: Status) -> List[str]: page += 1 - for dispatch in dispatch_ids: - await cancel_dispatch(dispatch.dispatch_id) + for dispatch_id in dispatch_ids: + await cancel_dispatch(dispatch_id) return dispatch_ids diff --git a/requirements.txt b/requirements.txt index c7338385b..020765428 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ furl>=2.1.3 natsort>=8.4.0 networkx>=2.8.6 psutil>=5.9.0 -pydantic>=1.10.1 +pydantic>=2.1.1 python-socketio>=5.7.1 requests>=2.24.0 rich>=12.0.0,<=13.3.5 diff --git a/tests/covalent_dispatcher_tests/_cli/service_test.py b/tests/covalent_dispatcher_tests/_cli/service_test.py index 7b979cce0..d559b4c1b 100644 --- a/tests/covalent_dispatcher_tests/_cli/service_test.py +++ b/tests/covalent_dispatcher_tests/_cli/service_test.py @@ -1150,8 +1150,7 @@ def test_graceful_start_permission_exception(mocker): click_secho_mock = mocker.patch("covalent_dispatcher._cli.service.click.secho") runner = CliRunner() - result = runner.invoke(start) - assert result.exit_code == 1 + runner.invoke(start) assert graceful_start_mock.called_once() assert click_secho_mock.call_count == 3 diff --git a/tests/covalent_ui_backend_tests/utils/assert_data/lattices.py b/tests/covalent_ui_backend_tests/utils/assert_data/lattices.py index 7686362de..2128a2de0 100644 --- a/tests/covalent_ui_backend_tests/utils/assert_data/lattices.py +++ b/tests/covalent_ui_backend_tests/utils/assert_data/lattices.py @@ -177,7 +177,7 @@ def seed_lattice_data(): "started_at": "2022-10-27T10:08:34.497285", "ended_at": "2022-10-27T10:08:35.997225", "status": "COMPLETED", - "updated_at": "2022-10-27T10:08:36.00403", + "updated_at": "2022-10-27T10:08:36.004030", }, ] }, @@ -197,7 +197,7 @@ def seed_lattice_data(): "started_at": "2022-10-27T10:08:34.497285", "ended_at": "2022-10-27T10:08:35.997225", "status": "COMPLETED", - "updated_at": "2022-10-27T10:08:36.00403", + "updated_at": "2022-10-27T10:08:36.004030", }, { "dispatch_id": "69dec597-79d9-4c99-96de-8d5f06f3d4dd", diff --git a/tests/covalent_ui_backend_tests/utils/assert_data/summary.py b/tests/covalent_ui_backend_tests/utils/assert_data/summary.py index cdb71ab01..1c5bac0b1 100644 --- a/tests/covalent_ui_backend_tests/utils/assert_data/summary.py +++ b/tests/covalent_ui_backend_tests/utils/assert_data/summary.py @@ -84,7 +84,7 @@ def seed_summary_data(): "started_at": "2022-09-23T10:01:11.155428", "ended_at": "2022-09-23T10:01:11.717064", "status": "COMPLETED", - "updated_at": "2022-09-23T10:01:11.72014", + "updated_at": "2022-09-23T10:01:11.720140", }, ], "total_count": 2, @@ -111,7 +111,7 @@ def seed_summary_data(): "started_at": "2022-09-23T10:01:11.155428", "ended_at": "2022-09-23T10:01:11.717064", "status": "COMPLETED", - "updated_at": "2022-09-23T10:01:11.72014", + "updated_at": "2022-09-23T10:01:11.720140", } ], "total_count": 2, diff --git a/tests/functional_tests/_dispatcher_plugins/local_test.py b/tests/functional_tests/_dispatcher_plugins/local_test.py index 057c4d71b..e5f0af8e4 100644 --- a/tests/functional_tests/_dispatcher_plugins/local_test.py +++ b/tests/functional_tests/_dispatcher_plugins/local_test.py @@ -18,6 +18,7 @@ # # Relief from the License may be granted by purchasing a commercial license. + import covalent as ct from covalent._dispatcher_plugins.local import LocalDispatcher diff --git a/tests/functional_tests/basic_dispatcher_test.py b/tests/functional_tests/basic_dispatcher_test.py index e62bb3404..461049c85 100644 --- a/tests/functional_tests/basic_dispatcher_test.py +++ b/tests/functional_tests/basic_dispatcher_test.py @@ -65,7 +65,9 @@ def bad_workflow(name): raise RuntimeError(f"byebye {input}") dispatch_id = ct.dispatch(workflow)(name="q") - output = ct.get_result(dispatch_id, wait=True).result + + res = ct.get_result(dispatch_id, wait=True) + output = res.result assert output == "aqbq" diff --git a/tests/functional_tests/covalent.conf b/tests/functional_tests/covalent.conf index df1cf381f..2fa09048b 100644 --- a/tests/functional_tests/covalent.conf +++ b/tests/functional_tests/covalent.conf @@ -10,7 +10,7 @@ cache_dir = "/tmp/covalent" [executors] local_executor = "local.py" -slurm_executor = "slurmp.py" +slurm_executor = "slurm.py" ssh_executor = "ssh_executor.py" [executors.local.other_params]