Merge branch 'develop' into arosen93-patch-1

.github/workflows/nightly.yml

@@ -22,7 +22,7 @@ name: nightly
 
 on:
   schedule:
-    - cron: "0 4 * * *"
+    - cron: "0 */4 * * *"
 
 jobs:
   license:

.github/workflows/tests.yml

@@ -215,22 +215,32 @@ jobs:
         run: |
           covalent db migrate
           if [ "${{ matrix.backend }}" = 'dask' ] ; then
-            covalent start
+            covalent start -d
           elif [ "${{ matrix.backend }}" = 'local' ] ; then
-            covalent start --no-cluster
+            covalent start --no-cluster -d
           else
             echo "Invalid backend specified in test matrix."
             exit 1
           fi
         env:
           COVALENT_EXECUTOR_DIR: doc/source/how_to/execution/custom_executors
 
+      - name: Run functional tests and measure coverage
+        id: functional-tests
+        if: env.BUILD_AND_RUN_ALL
+        run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/functional_tests  --cov=covalent --cov=covalent_dispatcher --cov-config=.coveragerc
+
+      - name: Generate functional test coverage report
+        id: functional-coverage
+        if: steps.functional-tests.outcome == 'success'
+        run: coverage xml -o functional_tests_coverage.xml
+
       - name: Run SDK tests and measure coverage
         id: sdk-tests
         if: >
           steps.modified-files.outputs.sdk == 'true'
           || env.BUILD_AND_RUN_ALL
-        run: PYTHONPATH=$PWD/ pytest -vv --reruns=5 tests/covalent_tests --cov=covalent --cov-config=.coveragerc
+        run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/covalent_tests --cov=covalent --cov-config=.coveragerc
 
       - name: Generate SDK coverage report
         id: sdk-coverage
@@ -242,29 +252,19 @@ jobs:
         if: >
           steps.modified-files.outputs.dispatcher == 'true'
           || env.BUILD_AND_RUN_ALL
-        run: PYTHONPATH=$PWD/ pytest -vv --reruns=5 tests/covalent_dispatcher_tests --cov=covalent_dispatcher --cov-config=.coveragerc
+        run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/covalent_dispatcher_tests --cov=covalent_dispatcher --cov-config=.coveragerc
 
       - name: Generate dispatcher coverage report
         id: dispatcher-coverage
         if: steps.dispatcher-tests.outcome == 'success'
         run: coverage xml -o dispatcher_coverage.xml
 
-      - name: Run functional tests and measure coverage
-        id: functional-tests
-        if: env.BUILD_AND_RUN_ALL
-        run: PYTHONPATH=$PWD/ pytest -vv --reruns=5 tests/functional_tests  --cov=covalent --cov=covalent_dispatcher --cov-config=.coveragerc
-
-      - name: Generate functional test coverage report
-        id: functional-coverage
-        if: steps.functional-tests.outcome == 'success'
-        run: coverage xml -o functional_tests_coverage.xml
-
       - name: Run UI backend tests and measure coverage
         id: ui-backend-tests
         if: >
           steps.modified-files.outputs.ui_backend == 'true'
           || env.BUILD_AND_RUN_ALL
-        run: PYTHONPATH=$PWD/ pytest -vv --reruns=5 tests/covalent_ui_backend_tests --cov=covalent_ui --cov-config=.coveragerc
+        run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/covalent_ui_backend_tests --cov=covalent_ui --cov-config=.coveragerc
 
       - name: Generate UI backend coverage report
         id: ui-backend-coverage
@@ -283,11 +283,7 @@ jobs:
       - name: Dump Covalent logs
         if: >
           steps.covalent_start.outcome == 'success'
-          && (steps.sdk-tests.outcome == 'failed'
-          || steps.dispatcher-tests.outcome == 'failed'
-          || steps.functional-tests.outcome == 'failed'
-          || steps.ui-backend-tests.outcome == 'failed'
-          || steps.ui-frontend-tests.outcome == 'failed')
+          && failure()
         run: covalent logs
 
       - name: Upload SDK report to Codecov

CHANGELOG.md

@@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Added
+
+- Added a `py.typed` file to support type-checking
+
+### Changed
+
+- Raised the minimum version of Pydantic from 1.10.1 to 2.1.1 in `requirements.txt`
+
+## [0.228.0-rc.0] - 2023-08-31
+
+### Authors
+
+- Andrew S. Rosen <[email protected]>
+- Co-authored-by: Sankalp Sanand <[email protected]>
+- Will Cunningham <[email protected]>
+- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
+- Co-authored-by: Casey Jao <[email protected]>
+- WingCode <[email protected]>
+- Nick Tyler <[email protected]>
+- Co-authored-by: Alejandro Esquivel <[email protected]>
+- Aravind <[email protected]>
+- Co-authored-by: Manjunath PV <[email protected]>
+- Co-authored-by: ArunPsiog <[email protected]>
+- Co-authored-by: RaviPsiog <[email protected]>
+- Co-authored-by: Prasy12 <[email protected]>
+- Co-authored-by: mpvgithub <[email protected]>
+- Co-authored-by: Will Cunningham <[email protected]>
+- dwelsch-esi <[email protected]>
+- Co-authored-by: dwelsch-memverge <[email protected]>
+- Co-authored-by: kessler-frost <[email protected]>
+- Faiyaz Hasan <[email protected]>
+- Santosh kumar <[email protected]>
+
+
+### Fixed
+
+- Fixed dispatcher address not showing when covalent server starts.
+- Fixed the failing tests in the `nightly` workflow.
+
 ### Operations
 
 
@@ -17,6 +56,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Reduce the number of pinned version numbers in the `setup.py`, `requirements.txt`, and `requirements-client.txt`
 - Updated the `wci.yml` file with new features
 - Bumped pre-commit versions
+- Temporarily running nightly hourly to test whether the fix worked
+- Reverted to daily frequency for nightly
 
 ### Added
 
@@ -42,7 +83,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Removed the upper limit from `dask` and `distributed` packages' versions until we find a version which is incompatible with Covalent.
 - When the server is stopped, any workflows in a non-terminal state are first cancelled
 - Pinned sqlalchemy version with upper limit <2.0.0.
-- Added rich support to cli for better printing statements. 
+- Added rich support to cli for better printing statements.
+- Performed minor modifications and rearrangementsto fix the broken tests in the `nightly` workflow.
 
 ### Tests
 

MANIFEST.in

@@ -1,6 +1,7 @@
 include VERSION
 include requirements.txt
 include requirements-client.txt
+include covalent/py.typed
 recursive-include covalent/executor/executor_plugins/ *
 recursive-include covalent_dispatcher/_service/ *
 recursive-include covalent_migrations/ *

VERSION

@@ -1 +1 @@
-0.227.0-rc.0
+0.228.0-rc.0

covalent/_results_manager/wait.py

@@ -1,3 +1,5 @@
+# Copyright 2021 Agnostiq Inc.
+#
 # This file is part of Covalent.
 #
 # Licensed under the GNU Affero General Public License 3.0 (the "License").
@@ -16,6 +18,7 @@
 #
 # Relief from the License may be granted by purchasing a commercial license.
 
+
 import sys
 from dataclasses import dataclass
 
@@ -28,6 +31,7 @@ def __int__(self) -> int:
         return self.RETRIES
 
 
+SHORT = Wait(30)
 LONG = Wait(3000)
 VERY_LONG = Wait(60000)
 EXTREME = Wait(sys.maxsize)

covalent/executor/executor_plugins/local.py

@@ -58,6 +58,7 @@
     "create_unique_workdir": False,
 }
 
+
 proc_pool = ProcessPoolExecutor()
 
 

covalent_dispatcher/_cli/service.py

@@ -468,7 +468,6 @@ def start(
             "Covalent was unable to start due to the following error: ", fg="red", bold=True
         )
         click.secho(traceback.format_exc(), fg="lightgrey")
-        return ctx.exit(1)
 
     set_config("user_interface.port", port)
     set_config("dispatcher.port", port)
@@ -503,6 +502,9 @@ def start(
     console.print(config_table)
     console.print("\nServer Status: [green]:heavy_check_mark:[/green] Running", style="bold")
 
+    dispatcher_address = f"http://{str(get_config('dispatcher.address'))}:{str(port)}"
+    console.print(f"\nCovalent UI can be accessed at {dispatcher_address}")
+
     if not no_footer:
         console.print("\nFor a summary of the system status, use 'covalent status'")
         print_footer(console)

covalent_dispatcher/_core/data_manager.py

@@ -339,7 +339,7 @@ def make_derived_dispatch(
 
 
 def get_result_object(dispatch_id: str) -> Result:
-    return _registered_dispatches[dispatch_id]
+    return _registered_dispatches.get(dispatch_id)
 
 
 def _register_result_object(result_object: Result):

covalent_dispatcher/_core/dispatcher.py

@@ -382,7 +382,11 @@ async def cancel_dispatch(dispatch_id: str, task_ids: List[int] = None) -> None:
     if not dispatch_id:
         return
 
-    tg = datasvc.get_result_object(dispatch_id=dispatch_id).lattice.transport_graph
+    res_object = datasvc.get_result_object(dispatch_id)
+    if res_object is None:
+        return
+
+    tg = res_object.lattice.transport_graph
     if task_ids:
         app_log.debug(f"Cancelling tasks {task_ids} in dispatch {dispatch_id}")
     else:

covalent_ui/heartbeat.py

@@ -82,8 +82,8 @@ async def cancel_all_with_status(status: Status) -> List[str]:
 
         page += 1
 
-    for dispatch in dispatch_ids:
-        await cancel_dispatch(dispatch.dispatch_id)
+    for dispatch_id in dispatch_ids:
+        await cancel_dispatch(dispatch_id)
 
     return dispatch_ids
 

requirements.txt

@@ -10,7 +10,7 @@ furl>=2.1.3
 natsort>=8.4.0
 networkx>=2.8.6
 psutil>=5.9.0
-pydantic>=1.10.1
+pydantic>=2.1.1
 python-socketio>=5.7.1
 requests>=2.24.0
 rich>=12.0.0,<=13.3.5

tests/covalent_dispatcher_tests/_cli/service_test.py

@@ -1150,8 +1150,7 @@ def test_graceful_start_permission_exception(mocker):
     click_secho_mock = mocker.patch("covalent_dispatcher._cli.service.click.secho")
 
     runner = CliRunner()
-    result = runner.invoke(start)
-    assert result.exit_code == 1
+    runner.invoke(start)
 
     assert graceful_start_mock.called_once()
     assert click_secho_mock.call_count == 3

tests/covalent_ui_backend_tests/utils/assert_data/lattices.py

@@ -177,7 +177,7 @@ def seed_lattice_data():
                             "started_at": "2022-10-27T10:08:34.497285",
                             "ended_at": "2022-10-27T10:08:35.997225",
                             "status": "COMPLETED",
-                            "updated_at": "2022-10-27T10:08:36.00403",
+                            "updated_at": "2022-10-27T10:08:36.004030",
                         },
                     ]
                 },
@@ -197,7 +197,7 @@ def seed_lattice_data():
                             "started_at": "2022-10-27T10:08:34.497285",
                             "ended_at": "2022-10-27T10:08:35.997225",
                             "status": "COMPLETED",
-                            "updated_at": "2022-10-27T10:08:36.00403",
+                            "updated_at": "2022-10-27T10:08:36.004030",
                         },
                         {
                             "dispatch_id": "69dec597-79d9-4c99-96de-8d5f06f3d4dd",

tests/covalent_ui_backend_tests/utils/assert_data/summary.py

@@ -84,7 +84,7 @@ def seed_summary_data():
                             "started_at": "2022-09-23T10:01:11.155428",
                             "ended_at": "2022-09-23T10:01:11.717064",
                             "status": "COMPLETED",
-                            "updated_at": "2022-09-23T10:01:11.72014",
+                            "updated_at": "2022-09-23T10:01:11.720140",
                         },
                     ],
                     "total_count": 2,
@@ -111,7 +111,7 @@ def seed_summary_data():
                             "started_at": "2022-09-23T10:01:11.155428",
                             "ended_at": "2022-09-23T10:01:11.717064",
                             "status": "COMPLETED",
-                            "updated_at": "2022-09-23T10:01:11.72014",
+                            "updated_at": "2022-09-23T10:01:11.720140",
                         }
                     ],
                     "total_count": 2,

tests/functional_tests/_dispatcher_plugins/local_test.py

@@ -18,6 +18,7 @@
 #
 # Relief from the License may be granted by purchasing a commercial license.
 
+
 import covalent as ct
 from covalent._dispatcher_plugins.local import LocalDispatcher
 

tests/functional_tests/basic_dispatcher_test.py

@@ -65,7 +65,9 @@ def bad_workflow(name):
         raise RuntimeError(f"byebye {input}")
 
     dispatch_id = ct.dispatch(workflow)(name="q")
-    output = ct.get_result(dispatch_id, wait=True).result
+
+    res = ct.get_result(dispatch_id, wait=True)
+    output = res.result
 
     assert output == "aqbq"
 

tests/functional_tests/covalent.conf

@@ -10,7 +10,7 @@ cache_dir = "/tmp/covalent"
 
 [executors]
 local_executor = "local.py"
-slurm_executor = "slurmp.py"
+slurm_executor = "slurm.py"
 ssh_executor = "ssh_executor.py"
 
 [executors.local.other_params]