From e6ad647673171bbf387d37af0e822df00fb7e1aa Mon Sep 17 00:00:00 2001 From: Casey Jao Date: Tue, 27 Feb 2024 21:51:45 -0500 Subject: [PATCH 01/18] Parse electron function strings at Electron construction (#1926) * Parse function string in Electron constructor Waiting until `build_graph()` won't work if the graph is being built in a remote executor without the original source code. * Changelog --------- Co-authored-by: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> --- CHANGELOG.md | 4 +++ covalent/_workflow/electron.py | 31 ++----------------- .../covalent_tests/workflow/electron_test.py | 21 +------------ 3 files changed, 8 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a928eabf..17344263c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Fixed + +- Sublattice electron function strings are now parsed correctly + ### Operations - Fixed nightly workflow's calling of other workflows. diff --git a/covalent/_workflow/electron.py b/covalent/_workflow/electron.py index 12f18cbf5..a8a9055be 100644 --- a/covalent/_workflow/electron.py +++ b/covalent/_workflow/electron.py @@ -96,6 +96,7 @@ def __init__( self.metadata = metadata self.task_group_id = task_group_id self._packing_tasks = packing_tasks + self._function_string = get_serialized_function_str(function) @property def packing_tasks(self) -> bool: @@ -442,7 +443,7 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]: ) name = sublattice_prefix + self.function.__name__ - function_string = get_serialized_function_str(self.function) + function_string = self._function_string bound_electron = sub_electron( self.function, json.dumps(parent_metadata), *args, **kwargs ) @@ -463,7 +464,7 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]: name=self.function.__name__, function=self.function, metadata=self.metadata.copy(), - function_string=get_serialized_function_str(self.function), + function_string=self._function_string, task_group_id=self.task_group_id if self.packing_tasks else None, ) self.task_group_id = self.task_group_id if self.packing_tasks else self.node_id @@ -608,32 +609,6 @@ def _auto_dict_node(*args, **kwargs): arg_index=arg_index, ) - def add_collection_node_to_graph(self, graph: "_TransportGraph", prefix: str) -> int: - """ - Adds the node to lattice's transport graph in the case - where a collection of electrons is passed as an argument - to another electron. - - Args: - graph: Transport graph of the lattice - prefix: Prefix of the node - - Returns: - node_id: Node id of the added node - """ - - new_metadata = encode_metadata(DEFAULT_METADATA_VALUES.copy()) - if "executor" in self.metadata: - new_metadata["executor"] = self.metadata["executor"] - new_metadata["executor_data"] = self.metadata["executor_data"] - - node_id = graph.add_node( - name=prefix, - function=to_decoded_electron_collection, - metadata=new_metadata, - function_string=get_serialized_function_str(to_decoded_electron_collection), - ) - return node_id def wait_for(self, electrons: Union["Electron", Iterable["Electron"]]): diff --git a/tests/covalent_tests/workflow/electron_test.py b/tests/covalent_tests/workflow/electron_test.py index a3db76bb7..d7a3e3192 100644 --- a/tests/covalent_tests/workflow/electron_test.py +++ b/tests/covalent_tests/workflow/electron_test.py @@ -36,7 +36,7 @@ to_decoded_electron_collection, ) from covalent._workflow.lattice import Lattice -from covalent._workflow.transport import TransportableObject, _TransportGraph, encode_metadata +from covalent._workflow.transport import TransportableObject, encode_metadata from covalent.executor.executor_plugins.local import LocalExecutor @@ -252,25 +252,6 @@ def test_collection_node_helper_electron(): assert to_decoded_electron_collection(x=dict_collection) == {"a": 1, "b": 2} -def test_electron_add_collection_node(): - """Test `to_decoded_electron_collection` in `Electron.add_collection_node`""" - - def f(x): - return x - - e = Electron(f) - tg = _TransportGraph() - node_id = e.add_collection_node_to_graph(tg, prefix=":") - collection_fn = tg.get_node_value(node_id, "function").get_deserialized() - - collection = [ - TransportableObject.make_transportable(1), - TransportableObject.make_transportable(2), - ] - - assert collection_fn(x=collection) == [1, 2] - - def test_injected_inputs_are_not_in_tg(): """Test that arguments to electrons injected by calldeps aren't added to the transport graph""" From d7841c7d0eb8f113efca1834fdb780b48ef63854 Mon Sep 17 00:00:00 2001 From: Casey Jao Date: Wed, 28 Feb 2024 09:24:36 -0500 Subject: [PATCH 02/18] Do not require dictionary keys to be `str` (#1886) * Unbreak task packing When submitting a task group, only attempt to upload task inputs corresponding to nodes external to the task group since only those will have been resolved. * Lock parent job record when persisting sublattices * Dictionary collector nodes no longer need keys to be `str` The collector electron now assembles the dictionary from two lists -- one list of keys, one list of corresponding values. * Fix tests * Changelog --- .github/workflows/tests.yml | 3 +- CHANGELOG.md | 2 + covalent/_workflow/electron.py | 6 +-- covalent_dispatcher/_core/dispatcher.py | 15 ++++++- covalent_dispatcher/_dal/importers/result.py | 2 +- covalent_dispatcher/_dal/importers/tg.py | 4 +- .../_core/dispatcher_db_integration_test.py | 27 +++++++++-- .../_core/execution_test.py | 28 +++++++++--- .../_dal/importers/result_import_test.py | 45 +++++++++++++++++++ .../covalent_tests/workflow/electron_test.py | 28 ++++++++---- tests/functional_tests/workflow_stack_test.py | 3 +- 11 files changed, 137 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 58189ba96..1349a02ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -217,9 +217,10 @@ jobs: if: env.BUILD_AND_RUN_ALL id: covalent_start run: | + export COVALENT_ENABLE_TASK_PACKING=1 covalent db migrate if [ "${{ matrix.backend }}" = 'dask' ] ; then - COVALENT_ENABLE_TASK_PACKING=1 covalent start -d + covalent start -d elif [ "${{ matrix.backend }}" = 'local' ] ; then covalent start --no-cluster -d else diff --git a/CHANGELOG.md b/CHANGELOG.md index 17344263c..76c2a9720 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Sublattice electron function strings are now parsed correctly +- The keys of dictionary inputs to electrons no longer need be strings. +- Fixed inaccuracies in task packing exposed by no longer uploading null attributes upon dispatch. ### Operations diff --git a/covalent/_workflow/electron.py b/covalent/_workflow/electron.py index a8a9055be..0e80f0a22 100644 --- a/covalent/_workflow/electron.py +++ b/covalent/_workflow/electron.py @@ -572,8 +572,8 @@ def _auto_list_node(*args, **kwargs): elif isinstance(param_value, dict): - def _auto_dict_node(*args, **kwargs): - return dict(kwargs) + def _auto_dict_node(keys, values): + return {keys[i]: values[i] for i in range(len(keys))} dict_electron = Electron( function=_auto_dict_node, @@ -581,7 +581,7 @@ def _auto_dict_node(*args, **kwargs): task_group_id=self.task_group_id, packing_tasks=True and active_lattice.task_packing, ) # Group the auto-generated node with the main node. - bound_electron = dict_electron(**param_value) + bound_electron = dict_electron(list(param_value.keys()), list(param_value.values())) transport_graph.set_node_value(bound_electron.node_id, "name", electron_dict_prefix) transport_graph.add_edge( dict_electron.node_id, diff --git a/covalent_dispatcher/_core/dispatcher.py b/covalent_dispatcher/_core/dispatcher.py index e17547969..b0ecd27b6 100644 --- a/covalent_dispatcher/_core/dispatcher.py +++ b/covalent_dispatcher/_core/dispatcher.py @@ -183,6 +183,7 @@ async def _submit_task_group(dispatch_id: str, sorted_nodes: List[int], task_gro app_log.debug("8A: Update node success (run_planned_workflow).") else: + # Nodes whose values have already been resolved known_nodes = [] # Skip the group if all task outputs can be reused from a @@ -196,6 +197,8 @@ async def _submit_task_group(dispatch_id: str, sorted_nodes: List[int], task_gro # Gather inputs for each task and send the task spec sequence to the runner task_specs = [] + sorted_nodes_set = set(sorted_nodes) + for node_id in sorted_nodes: app_log.debug(f"Gathering inputs for task {node_id} (run_planned_workflow).") @@ -214,8 +217,16 @@ async def _submit_task_group(dispatch_id: str, sorted_nodes: List[int], task_gro "args_ids": abs_task_input["args"], "kwargs_ids": abs_task_input["kwargs"], } - known_nodes += abs_task_input["args"] - known_nodes += list(abs_task_input["kwargs"].values()) + # Task inputs that don't belong to the task group have already beeen resolved + external_task_args = filter( + lambda x: x not in sorted_nodes_set, abs_task_input["args"] + ) + known_nodes.extend(external_task_args) + external_task_kwargs = filter( + lambda x: x not in sorted_nodes_set, abs_task_input["kwargs"].values() + ) + known_nodes.extend(external_task_kwargs) + task_specs.append(task_spec) app_log.debug( diff --git a/covalent_dispatcher/_dal/importers/result.py b/covalent_dispatcher/_dal/importers/result.py index 395516b86..7e4bd36f9 100644 --- a/covalent_dispatcher/_dal/importers/result.py +++ b/covalent_dispatcher/_dal/importers/result.py @@ -72,7 +72,6 @@ def import_result( # Main case: insert new lattice, electron, edge, and job records storage_path = os.path.join(base_path, dispatch_id) - os.makedirs(storage_path) lattice_record_kwargs = _get_result_meta(res, storage_path, electron_id) lattice_record_kwargs.update(_get_lattice_meta(res.lattice, storage_path)) @@ -143,6 +142,7 @@ def _connect_result_to_electron( fields={"id", "cancel_requested"}, equality_filters={"id": parent_electron_record.job_id}, membership_filters={}, + for_update=True, )[0] cancel_requested = parent_job_record.cancel_requested diff --git a/covalent_dispatcher/_dal/importers/tg.py b/covalent_dispatcher/_dal/importers/tg.py index 468abdadf..c67cc34b9 100644 --- a/covalent_dispatcher/_dal/importers/tg.py +++ b/covalent_dispatcher/_dal/importers/tg.py @@ -51,7 +51,9 @@ def import_transport_graph( # Propagate parent electron id's `cancel_requested` property to the sublattice electrons if electron_id is not None: parent_e_record = Electron.meta_type.get_by_primary_key(session, electron_id) - job_record = Job.get_by_primary_key(session=session, primary_key=parent_e_record.job_id) + job_record = Job.get_by_primary_key( + session=session, primary_key=parent_e_record.job_id, for_update=True + ) cancel_requested = job_record.cancel_requested else: cancel_requested = False diff --git a/tests/covalent_dispatcher_tests/_core/dispatcher_db_integration_test.py b/tests/covalent_dispatcher_tests/_core/dispatcher_db_integration_test.py index 53444a7b6..1544b753c 100644 --- a/tests/covalent_dispatcher_tests/_core/dispatcher_db_integration_test.py +++ b/tests/covalent_dispatcher_tests/_core/dispatcher_db_integration_test.py @@ -104,7 +104,7 @@ def list_workflow(arg): @ct.lattice def dict_workflow(arg): - return dict_task(arg) + return dict_task(arg=arg) # 1 2 # \ \ @@ -159,7 +159,7 @@ async def mock_get_incoming_edges(dispatch_id, node_id): # dict-type inputs - # Nodes 0=task, 1=:electron_dict:, 2=1, 3=2 + # Nodes 0=task, 1=:electron_dict:, 2=["a" (3), "b" (4)], 5=[1 (6), 2 (7)] dict_workflow.build_graph({"a": 1, "b": 2}) abstract_args = {"a": 2, "b": 3} tg = dict_workflow.transport_graph @@ -172,10 +172,31 @@ async def mock_get_incoming_edges(dispatch_id, node_id): mock_get_incoming_edges, ) + task_inputs = await _get_abstract_task_inputs( + result_object.dispatch_id, 0, tg.get_node_value(0, "name") + ) + expected_inputs = {"args": [], "kwargs": {"arg": 1}} + + assert task_inputs == expected_inputs + task_inputs = await _get_abstract_task_inputs( result_object.dispatch_id, 1, tg.get_node_value(1, "name") ) - expected_inputs = {"args": [], "kwargs": abstract_args} + expected_inputs = {"args": [2, 5], "kwargs": {}} + + assert task_inputs == expected_inputs + + task_inputs = await _get_abstract_task_inputs( + result_object.dispatch_id, 2, tg.get_node_value(2, "name") + ) + expected_inputs = {"args": [3, 4], "kwargs": {}} + + assert task_inputs == expected_inputs + + task_inputs = await _get_abstract_task_inputs( + result_object.dispatch_id, 5, tg.get_node_value(5, "name") + ) + expected_inputs = {"args": [6, 7], "kwargs": {}} assert task_inputs == expected_inputs diff --git a/tests/covalent_dispatcher_tests/_core/execution_test.py b/tests/covalent_dispatcher_tests/_core/execution_test.py index 6d521691f..4e2c20ac9 100644 --- a/tests/covalent_dispatcher_tests/_core/execution_test.py +++ b/tests/covalent_dispatcher_tests/_core/execution_test.py @@ -116,7 +116,7 @@ def list_workflow(arg): @ct.lattice def dict_workflow(arg): - return dict_task(arg) + return dict_task(arg=arg) # 1 2 # \ \ @@ -167,20 +167,36 @@ def multivar_workflow(x, y): # dict-type inputs dict_workflow.build_graph({"a": 1, "b": 2}) - serialized_args = {"a": ct.TransportableObject(1), "b": ct.TransportableObject(2)} # Nodes 0=task, 1=:electron_dict:, 2=1, 3=2 + # Nodes 0=task, 1=:electron_dict:, 2=["a" (3), "b" (4)], 5=[1 (6), 2 (7)] + sdkres = Result(lattice=dict_workflow, dispatch_id="asdf_dict_workflow") result_object = get_mock_srvresult(sdkres, test_db) tg = result_object.lattice.transport_graph - tg.set_node_value(2, "output", ct.TransportableObject(1)) - tg.set_node_value(3, "output", ct.TransportableObject(2)) + + tg.set_node_value(1, "output", ct.TransportableObject("node_1_output")) + tg.set_node_value(3, "output", ct.TransportableObject("a")) + tg.set_node_value(4, "output", ct.TransportableObject("b")) + tg.set_node_value(6, "output", ct.TransportableObject(1)) + tg.set_node_value(7, "output", ct.TransportableObject(2)) mock_get_result = mocker.patch( "covalent_dispatcher._core.runner.datasvc.get_result_object", return_value=result_object ) - task_inputs = await _get_task_inputs(1, tg.get_node_value(1, "name"), result_object) - expected_inputs = {"args": [], "kwargs": serialized_args} + serialized_kwargs = {"arg": ct.TransportableObject("node_1_output")} + task_inputs = await _get_task_inputs(0, tg.get_node_value(0, "name"), result_object) + expected_inputs = {"args": [], "kwargs": serialized_kwargs} + + serialized_args = [ct.TransportableObject("a"), ct.TransportableObject("b")] + task_inputs = await _get_task_inputs(2, tg.get_node_value(2, "name"), result_object) + expected_inputs = {"args": serialized_args, "kwargs": {}} + + assert task_inputs == expected_inputs + + serialized_args = [ct.TransportableObject(1), ct.TransportableObject(2)] + task_inputs = await _get_task_inputs(5, tg.get_node_value(5, "name"), result_object) + expected_inputs = {"args": serialized_args, "kwargs": {}} assert task_inputs == expected_inputs diff --git a/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py b/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py index 440742cba..819f88bc6 100644 --- a/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py +++ b/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py @@ -27,6 +27,7 @@ from covalent._shared_files.schemas.result import AssetSchema, ResultSchema from covalent._shared_files.util_classes import RESULT_STATUS from covalent_dispatcher._dal.importers.result import SERVER_URL, handle_redispatch, import_result +from covalent_dispatcher._dal.job import Job from covalent_dispatcher._dal.result import get_result_object from covalent_dispatcher._db.datastore import DataStore @@ -140,6 +141,7 @@ def test_import_previously_imported_result(mocker, test_db): prefix="covalent-" ) as srv_dir: sub_res = get_mock_result(sub_dispatch_id, sdk_dir) + sub_res.metadata.root_dispatch_id = dispatch_id import_result(sub_res, srv_dir, None) srv_res = get_result_object(dispatch_id, bare=True) parent_node = srv_res.lattice.transport_graph.get_node(0) @@ -152,6 +154,49 @@ def test_import_previously_imported_result(mocker, test_db): assert sub_srv_res._electron_id == parent_node._electron_id +def test_import_subdispatch_cancel_req(mocker, test_db): + """Test that Job.cancel_requested is propagated to sublattices""" + + dispatch_id = "test_propagate_cancel_requested" + sub_dispatch_id = "test_propagate_cancel_requested_sub" + + mocker.patch("covalent_dispatcher._dal.base.workflow_db", test_db) + + mock_filter_uris = mocker.patch( + "covalent_dispatcher._dal.importers.result._filter_remote_uris" + ) + + with tempfile.TemporaryDirectory(prefix="covalent-") as sdk_dir, tempfile.TemporaryDirectory( + prefix="covalent-" + ) as srv_dir: + res = get_mock_result(dispatch_id, sdk_dir) + import_result(res, srv_dir, None) + + with test_db.Session() as session: + Job.update_bulk( + session, values={"cancel_requested": True}, equality_filters={}, membership_filters={} + ) + session.commit() + + with tempfile.TemporaryDirectory(prefix="covalent-") as sdk_dir, tempfile.TemporaryDirectory( + prefix="covalent-" + ) as srv_dir: + sub_res = get_mock_result(sub_dispatch_id, sdk_dir) + sub_res.metadata.root_dispatch_id = dispatch_id + srv_res = get_result_object(dispatch_id, bare=True) + parent_node = srv_res.lattice.transport_graph.get_node(0) + import_result(sub_res, srv_dir, parent_node._electron_id) + + with tempfile.TemporaryDirectory(prefix="covalent-") as srv_dir: + import_result(sub_res, srv_dir, parent_node._electron_id) + + with test_db.Session() as session: + uncancelled = Job.get( + session, fields=[], equality_filters={"cancel_requested": False}, membership_filters={} + ) + assert len(uncancelled) == 0 + + @pytest.mark.parametrize( "parent_status,new_status", [ diff --git a/tests/covalent_tests/workflow/electron_test.py b/tests/covalent_tests/workflow/electron_test.py index d7a3e3192..2d5936ed3 100644 --- a/tests/covalent_tests/workflow/electron_test.py +++ b/tests/covalent_tests/workflow/electron_test.py @@ -377,18 +377,30 @@ def workflow(x): g = workflow.transport_graph._graph # Account for postprocessing node - assert list(g.nodes) == [0, 1, 2, 3, 4] + assert list(g.nodes) == [0, 1, 2, 3, 4, 5, 6, 7, 8] fn = g.nodes[1]["function"].get_deserialized() - assert fn(x=2, y=5, z=7) == {"x": 2, "y": 5, "z": 7} - assert g.nodes[2]["value"].get_deserialized() == 5 - assert g.nodes[3]["value"].get_deserialized() == 7 + assert fn(["x", "y", "z"], [2, 5, 7]) == {"x": 2, "y": 5, "z": 7} + fn = g.nodes[2]["function"].get_deserialized() + assert fn("x", "y") == ["x", "y"] + keys = [g.nodes[3]["value"].get_deserialized(), g.nodes[4]["value"].get_deserialized()] + fn = g.nodes[5]["function"].get_deserialized() + assert fn(2, 3) == [2, 3] + vals = [g.nodes[6]["value"].get_deserialized(), g.nodes[7]["value"].get_deserialized()] + assert keys == ["x", "y"] + assert vals == [5, 7] assert set(g.edges) == { (1, 0, 0), (2, 1, 0), - (3, 1, 0), - (0, 4, 0), - (0, 4, 1), - (1, 4, 0), + (3, 2, 0), + (4, 2, 0), + (5, 1, 0), + (6, 5, 0), + (7, 5, 0), + (0, 8, 0), + (0, 8, 1), + (1, 8, 0), + (2, 8, 0), + (5, 8, 0), } diff --git a/tests/functional_tests/workflow_stack_test.py b/tests/functional_tests/workflow_stack_test.py index 47e1578a7..f20b3a0f9 100644 --- a/tests/functional_tests/workflow_stack_test.py +++ b/tests/functional_tests/workflow_stack_test.py @@ -800,7 +800,8 @@ def workflow(x): res_1 = sum_values(x) return square(res_1) - dispatch_id = ct.dispatch(workflow)({"x": 1, "y": 2, "z": 3}) + # Check that non-string keys are allowed + dispatch_id = ct.dispatch(workflow)({"x": 1, "y": 2, 3: 3}) res_obj = rm.get_result(dispatch_id, wait=True) From 5486a1493cc91646e2e243acce5f0e1510b64420 Mon Sep 17 00:00:00 2001 From: CovalentOpsBot Date: Fri, 10 May 2024 12:13:31 +0000 Subject: [PATCH 03/18] The new version will be 0.234.1-rc.0 --- CHANGELOG.md | 11 +++++++++++ VERSION | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c2a9720..e3e6a1376 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +## [0.234.1-rc.0] - 2024-05-10 + +### Authors + +- Andrew S. Rosen +- Sankalp Sanand +- Co-authored-by: Alejandro Esquivel +- Casey Jao +- Co-authored-by: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> + + ### Fixed - Sublattice electron function strings are now parsed correctly diff --git a/VERSION b/VERSION index 19b9402f4..9d818f2d5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.234.0-rc.0 \ No newline at end of file +0.234.1-rc.0 \ No newline at end of file From d855231adaeb4571912cbc1829cd82919021a5a1 Mon Sep 17 00:00:00 2001 From: Ara Ghukasyan <38226926+araghukas@users.noreply.github.com> Date: Tue, 28 May 2024 14:07:31 -0400 Subject: [PATCH 04/18] Note possible limit on concurrent electrons that use SSH-based executors (#1919) * add note for slurm ssh limit * update changelog * update changelog * update note * fix changelog * add whitespace around operators for pre-commit check * more add whitespace --- CHANGELOG.md | 5 ++++ covalent/_dispatcher_plugins/local.py | 4 +-- doc/source/api/executors/slurm.rst | 25 ++----------------- .../_db/write_result_to_db_test.py | 8 +++--- .../stress_tests/scripts/mnist_sublattices.py | 2 +- .../stress_tests/scripts/sublattices_mixed.py | 2 +- tests/stress_tests/scripts/tasks.py | 2 +- 7 files changed, 16 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3e6a1376..34d43fd92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Changed + +- Updated Slurm plugin docs to note possible SSH limitation +- Updated Slurm plugin docs to remove `sshproxy` section + ## [0.234.1-rc.0] - 2024-05-10 ### Authors diff --git a/covalent/_dispatcher_plugins/local.py b/covalent/_dispatcher_plugins/local.py index 8760cec96..117637add 100644 --- a/covalent/_dispatcher_plugins/local.py +++ b/covalent/_dispatcher_plugins/local.py @@ -596,7 +596,7 @@ def _upload(assets: List[AssetSchema]): number_uploaded = 0 for i, asset in enumerate(assets): if not asset.remote_uri or not asset.uri: - app_log.debug(f"Skipping asset {i+1} out of {total}") + app_log.debug(f"Skipping asset {i + 1} out of {total}") continue if asset.remote_uri.startswith(local_scheme_prefix): copy_file_locally(asset.uri, asset.remote_uri) @@ -604,7 +604,7 @@ def _upload(assets: List[AssetSchema]): else: _upload_asset(asset.uri, asset.remote_uri) number_uploaded += 1 - app_log.debug(f"Uploaded asset {i+1} out of {total}.") + app_log.debug(f"Uploaded asset {i + 1} out of {total}.") app_log.debug(f"uploaded {number_uploaded} assets.") diff --git a/doc/source/api/executors/slurm.rst b/doc/source/api/executors/slurm.rst index e9df75496..1412cd7d8 100644 --- a/doc/source/api/executors/slurm.rst +++ b/doc/source/api/executors/slurm.rst @@ -133,27 +133,6 @@ Here the corresponding submit script contains the following commands: srun --ntasks-per-node 1 dcgmi profile --resume +.. note:: -sshproxy --------- - -Some users may need two-factor authentication (2FA) to connect to a cluster. This plugin supports one form of 2FA using the `sshproxy `_ service developed by NERSC. When this plugin is configured to support ``sshproxy``, the user's SSH key and certificate will be refreshed automatically by Covalent if either it does not exist or it is expired. We assume that the user has already `configured 2FA `_, used the ``sshproxy`` service on the command line without issue, and added the executable to their ``PATH``. Note that this plugin assumes the script is called ``sshproxy``, not ``sshproxy.sh``. Further note that using ``sshproxy`` within Covalent is not required; a user can still run it manually and provide ``ssh_key_file`` and ``cert_file`` in the plugin constructor. - -In order to enable ``sshproxy`` in this plugin, add the following block to your Covalent configuration while the server is stopped: - -.. code:: bash - - [executors.slurm.sshproxy] - hosts = [ "perlmutter-p1.nersc.gov" ] - password = "" - secret = "" - -For details on how to modify your Covalent configuration, refer to the documentation `here `_. - -Then, reinstall this plugin using ``pip install covalent-slurm-plugin[sshproxy]`` in order to pull in the ``oathtool`` package which will generate one-time passwords. - -The ``hosts`` parameter is a list of hostnames for which the ``sshproxy`` service will be used. If the address provided in the plugin constructor is not present in this list, ``sshproxy`` will not be used. The ``password`` is the user's password, not including the 6-digit OTP. The ``secret`` is the 2FA secret provided when a user registers a new device on `Iris `_. Rather than scan the QR code into an authenticator app, inspect the Oath Seed URL for a string labeled ``secret=...``, typically consisting of numbers and capital letters. Users can validate that correct OTP codes are being generated by using the command ``oathtool `` and using the 6-digit number returned in the "Test" option on the Iris 2FA page. Note that these values are stored in plaintext in the Covalent configuration file. If a user suspects credentials have been stolen or compromised, contact your systems administrator immediately to report the incident and request deactivation. - -.. autoclass:: covalent_slurm_plugin.SlurmExecutor - :members: - :inherited-members: +Each electron that uses the Slurm executor opens a separate SSH connection to the remote system. When executing 10 or more concurrent electrons, be mindful of client and/or server-side limitations on the total number of SSH connections. diff --git a/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py b/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py index 26b114913..759dbbe1b 100644 --- a/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py +++ b/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py @@ -257,8 +257,8 @@ def test_insert_lattices_data(test_db, mocker): lattice_args = get_lattice_kwargs( dispatch_id=f"dispatch_{i + 1}", name=f"workflow_{i + 1}", - docstring_filename=f"docstring_{i+1}.txt", - storage_path=f"results/dispatch_{i+1}/", + docstring_filename=f"docstring_{i + 1}.txt", + storage_path=f"results/dispatch_{i + 1}/", executor="dask", workflow_executor="dask", created_at=cur_time, @@ -276,10 +276,10 @@ def test_insert_lattices_data(test_db, mocker): assert lattice.dispatch_id == f"dispatch_{i + 1}" assert lattice.electron_id is None assert lattice.name == f"workflow_{i + 1}" - assert lattice.docstring_filename == f"docstring_{i+1}.txt" + assert lattice.docstring_filename == f"docstring_{i + 1}.txt" assert lattice.status == "RUNNING" assert lattice.storage_type == STORAGE_TYPE - assert lattice.storage_path == f"results/dispatch_{i+1}/" + assert lattice.storage_path == f"results/dispatch_{i + 1}/" assert lattice.function_filename == FUNCTION_FILENAME assert lattice.function_string_filename == FUNCTION_STRING_FILENAME assert lattice.executor == "dask" diff --git a/tests/stress_tests/scripts/mnist_sublattices.py b/tests/stress_tests/scripts/mnist_sublattices.py index 31ad46fc8..2b0e0aab0 100644 --- a/tests/stress_tests/scripts/mnist_sublattices.py +++ b/tests/stress_tests/scripts/mnist_sublattices.py @@ -146,7 +146,7 @@ def test( correct += (pred.argmax(1) == y).type(torch.float).sum().item() test_loss /= num_batches correct /= size - print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") + print(f"Test Error: \n Accuracy: {(100 * correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") def train_model( diff --git a/tests/stress_tests/scripts/sublattices_mixed.py b/tests/stress_tests/scripts/sublattices_mixed.py index 4dc085f0f..d5984b55c 100644 --- a/tests/stress_tests/scripts/sublattices_mixed.py +++ b/tests/stress_tests/scripts/sublattices_mixed.py @@ -147,7 +147,7 @@ def test( correct += (pred.argmax(1) == y).type(torch.float).sum().item() test_loss /= num_batches correct /= size - print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") + print(f"Test Error: \n Accuracy: {(100 * correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") def train_model( diff --git a/tests/stress_tests/scripts/tasks.py b/tests/stress_tests/scripts/tasks.py index 55d9ab8c9..181c4c33a 100644 --- a/tests/stress_tests/scripts/tasks.py +++ b/tests/stress_tests/scripts/tasks.py @@ -175,7 +175,7 @@ def test( correct += (pred.argmax(1) == y).type(torch.float).sum().item() test_loss /= num_batches correct /= size - print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") + print(f"Test Error: \n Accuracy: {(100 * correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") def train_model( From f939961ea698c5b374e8c08ad036b7fc2242f50f Mon Sep 17 00:00:00 2001 From: Casey Jao Date: Wed, 29 May 2024 13:58:00 -0400 Subject: [PATCH 05/18] Abstract API base endpoint (#1950) --- .github/workflows/requirements.yml | 2 +- CHANGELOG.md | 15 ++ covalent/_api/apiclient.py | 10 +- covalent/_dispatcher_plugins/local.py | 16 +- covalent/_results_manager/result.py | 33 +-- covalent/_serialize/electron.py | 4 +- covalent/_serialize/lattice.py | 49 +--- covalent/_shared_files/schemas/electron.py | 15 +- covalent/_shared_files/schemas/lattice.py | 29 +- covalent/_shared_files/schemas/result.py | 6 +- covalent/_workflow/electron.py | 22 +- covalent/_workflow/lattice.py | 12 +- covalent/triggers/database_trigger.py | 9 +- covalent_dispatcher/_cli/cli.py | 14 +- covalent_dispatcher/_cli/migrate.py | 208 --------------- covalent_dispatcher/_cli/service.py | 12 - .../_dal/db_interfaces/lattice_utils.py | 8 - .../_dal/importers/electron.py | 8 +- covalent_dispatcher/_dal/importers/lattice.py | 16 +- covalent_dispatcher/_db/dispatchdb.py | 33 --- covalent_dispatcher/_db/models.py | 8 +- covalent_dispatcher/_db/upsert.py | 12 - covalent_dispatcher/_db/write_result_to_db.py | 8 - covalent_dispatcher/_service/app.py | 2 +- covalent_dispatcher/_service/models.py | 4 - covalent_ui/app.py | 1 - covalent_ui/result_webhook.py | 9 +- doc/source/api/cli.rst | 2 +- doc/source/version_migrations/index.rst | 18 +- requirements-client.txt | 1 - requirements.txt | 4 +- setup.py | 5 +- .../_cli/cli_test.py | 1 - .../_cli/migrate_test.py | 249 ------------------ .../result.pkl | Bin 22463 -> 0 bytes .../_dal/importers/result_import_test.py | 4 +- .../_dal/lattice_test.py | 3 - .../_db/update_test.py | 13 - .../_db/write_result_to_db_test.py | 16 -- .../_service/app_test.py | 2 +- .../serialize/lattice_serialization_test.py | 6 +- .../triggers/database_trigger_test.py | 4 +- .../workflow/dispatch_source_test.py | 94 ------- .../covalent_tests/workflow/electron_test.py | 15 -- .../workflow/lattice_serialization_test.py | 1 - .../functional_tests/webhook_test.py | 5 +- 46 files changed, 118 insertions(+), 890 deletions(-) delete mode 100644 covalent_dispatcher/_cli/migrate.py delete mode 100644 tests/covalent_dispatcher_tests/_cli/migrate_test.py delete mode 100644 tests/covalent_dispatcher_tests/_cli/sample_results_dir/652dc473-fa37-4846-85f3-b314204fd432/result.pkl delete mode 100644 tests/covalent_tests/workflow/dispatch_source_test.py diff --git a/.github/workflows/requirements.yml b/.github/workflows/requirements.yml index 4502b183f..2eb1b7932 100644 --- a/.github/workflows/requirements.yml +++ b/.github/workflows/requirements.yml @@ -39,7 +39,7 @@ jobs: run: python -m pip install pip-check-reqs - name: Check extra core requirements - run: pip-extra-reqs -r werkzeug -r python-multipart covalent covalent_dispatcher covalent_ui --ignore-requirement=qiskit --ignore-requirement=qiskit-ibm-provider --ignore-requirement=amazon-braket-pennylane-plugin + run: pip-extra-reqs -r python-multipart covalent covalent_dispatcher covalent_ui --ignore-requirement=qiskit --ignore-requirement=qiskit-ibm-provider --ignore-requirement=amazon-braket-pennylane-plugin - name: Check missing SDK requirements run: > diff --git a/CHANGELOG.md b/CHANGELOG.md index 34d43fd92..9411e401d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated Slurm plugin docs to note possible SSH limitation - Updated Slurm plugin docs to remove `sshproxy` section +- API base endpoint is now configurable from an environment variable +- Removed unused lattice attributes to reduce asset uploads + +### Fixed + +- Improved handling of Covalent version mismatches between client and + executor environments + +### Removed + +- Removed obsolete `migrate-pickled-result-object` command + +### Operations + +- Allow installing a specific commit sha to ease testing ## [0.234.1-rc.0] - 2024-05-10 diff --git a/covalent/_api/apiclient.py b/covalent/_api/apiclient.py index c4c2a5492..d3be6bd4a 100644 --- a/covalent/_api/apiclient.py +++ b/covalent/_api/apiclient.py @@ -33,7 +33,7 @@ def __init__(self, dispatcher_addr: str, adapter: HTTPAdapter = None, auto_raise self.adapter = adapter self.auto_raise = auto_raise - def prepare_headers(self, **kwargs): + def prepare_headers(self, kwargs): extra_headers = CovalentAPIClient.get_extra_headers() headers = kwargs.get("headers", {}) if headers: @@ -42,7 +42,7 @@ def prepare_headers(self, **kwargs): return headers def get(self, endpoint: str, **kwargs): - headers = self.prepare_headers(**kwargs) + headers = self.prepare_headers(kwargs) url = self.dispatcher_addr + endpoint try: with requests.Session() as session: @@ -62,7 +62,7 @@ def get(self, endpoint: str, **kwargs): return r def put(self, endpoint: str, **kwargs): - headers = self.prepare_headers() + headers = self.prepare_headers(kwargs) url = self.dispatcher_addr + endpoint try: with requests.Session() as session: @@ -81,7 +81,7 @@ def put(self, endpoint: str, **kwargs): return r def post(self, endpoint: str, **kwargs): - headers = self.prepare_headers() + headers = self.prepare_headers(kwargs) url = self.dispatcher_addr + endpoint try: with requests.Session() as session: @@ -100,7 +100,7 @@ def post(self, endpoint: str, **kwargs): return r def delete(self, endpoint: str, **kwargs): - headers = self.prepare_headers() + headers = self.prepare_headers(kwargs) url = self.dispatcher_addr + endpoint try: with requests.Session() as session: diff --git a/covalent/_dispatcher_plugins/local.py b/covalent/_dispatcher_plugins/local.py index 117637add..9857342cf 100644 --- a/covalent/_dispatcher_plugins/local.py +++ b/covalent/_dispatcher_plugins/local.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import tempfile from copy import deepcopy from functools import wraps @@ -47,6 +48,9 @@ dispatch_cache_dir.mkdir(parents=True, exist_ok=True) +BASE_ENDPOINT = os.getenv("COVALENT_DISPATCH_BASE_ENDPOINT", "/api/v2/dispatches") + + def get_redispatch_request_body_v2( dispatch_id: str, staging_dir: str, @@ -540,10 +544,10 @@ def register_manifest( dispatcher_addr = format_server_url() stripped = strip_local_uris(manifest) if push_assets else manifest - endpoint = "/api/v2/dispatches" + endpoint = BASE_ENDPOINT if parent_dispatch_id: - endpoint = f"{endpoint}/{parent_dispatch_id}/subdispatches" + endpoint = f"{BASE_ENDPOINT}/{parent_dispatch_id}/sublattices" r = APIClient(dispatcher_addr).post(endpoint, data=stripped.model_dump_json()) r.raise_for_status() @@ -615,6 +619,7 @@ def _upload_asset(local_uri, remote_uri): else: local_path = local_uri + filesize = os.path.getsize(local_path) with open(local_path, "rb") as reader: app_log.debug(f"uploading to {remote_uri}") f = furl(remote_uri) @@ -624,6 +629,11 @@ def _upload_asset(local_uri, remote_uri): dispatcher_addr = f"{scheme}://{host}:{port}" endpoint = str(f.path) api_client = APIClient(dispatcher_addr) + if f.query: + endpoint = f"{endpoint}?{f.query}" + + # Workaround for Requests bug when streaming from empty files + data = reader.read() if filesize < 50 else reader - r = api_client.put(endpoint, data=reader) + r = api_client.put(endpoint, headers={"Content-Length": str(filesize)}, data=data) r.raise_for_status() diff --git a/covalent/_results_manager/result.py b/covalent/_results_manager/result.py index a42f514a6..8a6e3520b 100644 --- a/covalent/_results_manager/result.py +++ b/covalent/_results_manager/result.py @@ -18,7 +18,7 @@ import os import re from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Set, Union +from typing import TYPE_CHECKING, Any, Dict, List, Union from .._shared_files import logger from .._shared_files.config import get_config @@ -516,34 +516,3 @@ def _convert_to_electron_result(self) -> Any: """ return self._result - - -def _filter_cova_decorators(function_string: str, cova_imports: Set[str]) -> str: - """ - Given a string representing a function, comment out any Covalent-related decorators. - - Args - function_string: A string representation of a workflow function. - - Returns: - The function string with Covalent-related decorators commented out. - """ - - has_cova_decorator = False - in_decorator = 0 - function_lines = function_string.split("\n") - for i in range(len(function_lines)): - line = function_lines[i].strip() - if in_decorator > 0: - function_lines[i] = f"# {function_lines[i]}" - in_decorator += line.count("(") - in_decorator -= line.count(")") - elif line.startswith("@"): - decorator_name = line.split("@")[1].split(".")[0].split("(")[0] - if decorator_name in cova_imports: - function_lines[i] = f"# {function_lines[i]}" - has_cova_decorator = True - in_decorator += line.count("(") - in_decorator -= line.count(")") - - return "\n".join(function_lines) if has_cova_decorator else function_string diff --git a/covalent/_serialize/electron.py b/covalent/_serialize/electron.py index fe5763675..b90879fbf 100644 --- a/covalent/_serialize/electron.py +++ b/covalent/_serialize/electron.py @@ -210,8 +210,8 @@ def _get_node_custom_assets(node_attrs: dict) -> Dict[str, AssetSchema]: def serialize_node(node_id: int, node_attrs: dict, node_storage_path) -> ElectronSchema: meta = _serialize_node_metadata(node_attrs, node_storage_path) assets = _serialize_node_assets(node_attrs, node_storage_path) - custom_assets = _get_node_custom_assets(node_attrs) - return ElectronSchema(id=node_id, metadata=meta, assets=assets, custom_assets=custom_assets) + assets._custom = _get_node_custom_assets(node_attrs) + return ElectronSchema(id=node_id, metadata=meta, assets=assets) def deserialize_node(e: ElectronSchema, metadata_only: bool = False) -> dict: diff --git a/covalent/_serialize/lattice.py b/covalent/_serialize/lattice.py index 3ab39f2bc..3d61fcfc1 100644 --- a/covalent/_serialize/lattice.py +++ b/covalent/_serialize/lattice.py @@ -40,10 +40,6 @@ "workflow_function_string": AssetType.TEXT, "doc": AssetType.TEXT, "inputs": AssetType.TRANSPORTABLE, - "named_args": AssetType.TRANSPORTABLE, - "named_kwargs": AssetType.TRANSPORTABLE, - "cova_imports": AssetType.JSONABLE, - "lattice_imports": AssetType.TEXT, "hooks": AssetType.JSONABLE, } @@ -112,33 +108,6 @@ def _serialize_lattice_assets(lat, storage_path: str) -> LatticeAssets: lat.inputs, ASSET_TYPES["inputs"], storage_path, ASSET_FILENAME_MAP["inputs"] ) - # Deprecate - named_args_asset = save_asset( - lat.named_args, - ASSET_TYPES["named_args"], - storage_path, - ASSET_FILENAME_MAP["named_args"], - ) - named_kwargs_asset = save_asset( - lat.named_kwargs, - ASSET_TYPES["named_kwargs"], - storage_path, - ASSET_FILENAME_MAP["named_kwargs"], - ) - cova_imports_asset = save_asset( - lat.cova_imports, - ASSET_TYPES["cova_imports"], - storage_path, - ASSET_FILENAME_MAP["cova_imports"], - ) - lattice_imports_asset = save_asset( - lat.lattice_imports, - ASSET_TYPES["lattice_imports"], - storage_path, - ASSET_FILENAME_MAP["lattice_imports"], - ) - - # NOTE: these are actually JSONable hooks_asset = save_asset( lat.metadata["hooks"], ASSET_TYPES["hooks"], @@ -151,10 +120,6 @@ def _serialize_lattice_assets(lat, storage_path: str) -> LatticeAssets: workflow_function_string=workflow_func_str_asset, doc=docstring_asset, inputs=inputs_asset, - named_args=named_args_asset, - named_kwargs=named_kwargs_asset, - cova_imports=cova_imports_asset, - lattice_imports=lattice_imports_asset, hooks=hooks_asset, ) @@ -166,20 +131,12 @@ def _deserialize_lattice_assets(assets: LatticeAssets) -> dict: ) doc = load_asset(assets.doc, ASSET_TYPES["doc"]) inputs = load_asset(assets.inputs, ASSET_TYPES["inputs"]) - named_args = load_asset(assets.named_args, ASSET_TYPES["named_args"]) - named_kwargs = load_asset(assets.named_kwargs, ASSET_TYPES["named_kwargs"]) - cova_imports = load_asset(assets.cova_imports, ASSET_TYPES["cova_imports"]) - lattice_imports = load_asset(assets.lattice_imports, ASSET_TYPES["lattice_imports"]) hooks = load_asset(assets.hooks, ASSET_TYPES["hooks"]) return { "workflow_function": workflow_function, "workflow_function_string": workflow_function_string, "__doc__": doc, "inputs": inputs, - "named_args": named_args, - "named_kwargs": named_kwargs, - "cova_imports": cova_imports, - "lattice_imports": lattice_imports, "metadata": { "hooks": hooks, }, @@ -194,12 +151,10 @@ def _get_lattice_custom_assets(lat: Lattice) -> Dict[str, AssetSchema]: def serialize_lattice(lat, storage_path: str) -> LatticeSchema: meta = _serialize_lattice_metadata(lat) assets = _serialize_lattice_assets(lat, storage_path) - custom_assets = _get_lattice_custom_assets(lat) + assets._custom = _get_lattice_custom_assets(lat) tg = serialize_transport_graph(lat.transport_graph, storage_path) - return LatticeSchema( - metadata=meta, assets=assets, custom_assets=custom_assets, transport_graph=tg - ) + return LatticeSchema(metadata=meta, assets=assets, transport_graph=tg) def deserialize_lattice(model: LatticeSchema) -> Lattice: diff --git a/covalent/_shared_files/schemas/electron.py b/covalent/_shared_files/schemas/electron.py index b245cc93d..c5da65e1d 100644 --- a/covalent/_shared_files/schemas/electron.py +++ b/covalent/_shared_files/schemas/electron.py @@ -19,7 +19,7 @@ from datetime import datetime from typing import Dict, Optional -from pydantic import BaseModel, field_validator +from pydantic import BaseModel from .asset import AssetSchema from .common import StatusEnum @@ -91,6 +91,8 @@ class ElectronAssets(BaseModel): # user dependent assets hooks: AssetSchema + _custom: Optional[Dict[str, AssetSchema]] = None + class ElectronMetadata(BaseModel): task_group_id: int @@ -103,6 +105,8 @@ class ElectronMetadata(BaseModel): start_time: Optional[datetime] = None end_time: Optional[datetime] = None + _custom: Optional[Dict] = None + # For use by redispatch def reset(self): self.status = StatusEnum.NEW_OBJECT @@ -114,12 +118,3 @@ class ElectronSchema(BaseModel): id: int metadata: ElectronMetadata assets: ElectronAssets - custom_assets: Optional[Dict[str, AssetSchema]] = None - - @field_validator("custom_assets") - def check_custom_asset_keys(cls, v): - if v is not None: - for key in v: - if key in ASSET_FILENAME_MAP: - raise ValueError(f"Asset {key} conflicts with built-in key") - return v diff --git a/covalent/_shared_files/schemas/lattice.py b/covalent/_shared_files/schemas/lattice.py index 6a3e2bbf9..783b966ee 100644 --- a/covalent/_shared_files/schemas/lattice.py +++ b/covalent/_shared_files/schemas/lattice.py @@ -18,7 +18,7 @@ from typing import Dict, Optional -from pydantic import BaseModel, field_validator +from pydantic import BaseModel from .asset import AssetSchema from .transport_graph import TransportGraphSchema @@ -39,10 +39,6 @@ "workflow_function_string", "__doc__", "inputs", - "named_args", - "named_kwargs", - "cova_imports", - "lattice_imports", # user dependent assets "hooks", } @@ -83,14 +79,18 @@ class LatticeAssets(BaseModel): workflow_function_string: AssetSchema doc: AssetSchema # __doc__ inputs: AssetSchema - named_args: AssetSchema - named_kwargs: AssetSchema - cova_imports: AssetSchema - lattice_imports: AssetSchema + + # Deprecated + named_args: AssetSchema = AssetSchema(size=0) + named_kwargs: AssetSchema = AssetSchema(size=0) + cova_imports: AssetSchema = AssetSchema(size=0) + lattice_imports: AssetSchema = AssetSchema(size=0) # lattice.metadata hooks: AssetSchema + _custom: Optional[Dict[str, AssetSchema]] = None + class LatticeMetadata(BaseModel): name: str # __name__ @@ -101,18 +101,11 @@ class LatticeMetadata(BaseModel): python_version: Optional[str] = None covalent_version: Optional[str] = None + _custom: Optional[Dict] = None + class LatticeSchema(BaseModel): metadata: LatticeMetadata assets: LatticeAssets - custom_assets: Optional[Dict[str, AssetSchema]] = None transport_graph: TransportGraphSchema - - @field_validator("custom_assets") - def check_custom_asset_keys(cls, v): - if v is not None: - for key in v: - if key in ASSET_FILENAME_MAP: - raise ValueError(f"Asset {key} conflicts with built-in key") - return v diff --git a/covalent/_shared_files/schemas/result.py b/covalent/_shared_files/schemas/result.py index fa771bf9b..3160c3708 100644 --- a/covalent/_shared_files/schemas/result.py +++ b/covalent/_shared_files/schemas/result.py @@ -17,7 +17,7 @@ """FastAPI models for /api/v1/resultv2 endpoints""" from datetime import datetime -from typing import Optional +from typing import Dict, Optional from pydantic import BaseModel @@ -54,6 +54,8 @@ class ResultMetadata(BaseModel): start_time: Optional[datetime] = None end_time: Optional[datetime] = None + _custom: Optional[Dict] = None + # For use by redispatch def reset(self): self.dispatch_id = "" @@ -67,6 +69,8 @@ class ResultAssets(BaseModel): result: AssetSchema error: AssetSchema + _custom: Optional[Dict[str, AssetSchema]] = None + class ResultSchema(BaseModel): metadata: ResultMetadata diff --git a/covalent/_workflow/electron.py b/covalent/_workflow/electron.py index 0e80f0a22..e6a6e4648 100644 --- a/covalent/_workflow/electron.py +++ b/covalent/_workflow/electron.py @@ -429,6 +429,11 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]: active_lattice.replace_electrons[name] = replacement_electron return bound_electron + # Avoid direct attribute access since that might trigger + # Electron.__getattr__ when executors build sublattices + # constructed with older versions of Covalent + function_string = self.__dict__.get("_function_string") + # Handle sublattices by injecting _build_sublattice_graph node if isinstance(self.function, Lattice): parent_metadata = active_lattice.metadata.copy() @@ -443,7 +448,6 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]: ) name = sublattice_prefix + self.function.__name__ - function_string = self._function_string bound_electron = sub_electron( self.function, json.dumps(parent_metadata), *args, **kwargs ) @@ -464,7 +468,7 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]: name=self.function.__name__, function=self.function, metadata=self.metadata.copy(), - function_string=self._function_string, + function_string=function_string, task_group_id=self.task_group_id if self.packing_tasks else None, ) self.task_group_id = self.task_group_id if self.packing_tasks else self.node_id @@ -847,16 +851,6 @@ def wait(child, parents): return child -@electron -def to_decoded_electron_collection(**x): - """Interchanges order of serialize -> collection""" - collection = list(x.values())[0] - if isinstance(collection, list): - return TransportableObject.deserialize_list(collection) - elif isinstance(collection, dict): - return TransportableObject.deserialize_dict(collection) - - # Copied from runner.py def _build_sublattice_graph(sub: Lattice, json_parent_metadata: str, *args, **kwargs): import os @@ -868,6 +862,8 @@ def _build_sublattice_graph(sub: Lattice, json_parent_metadata: str, *args, **kw sub.build_graph(*args, **kwargs) + DISABLE_LEGACY_SUBLATTICES = os.environ.get("COVALENT_DISABLE_LEGACY_SUBLATTICES") == "1" + try: # Attempt multistage sublattice dispatch. For now we require # the executor to reach the Covalent server @@ -891,5 +887,7 @@ def _build_sublattice_graph(sub: Lattice, json_parent_metadata: str, *args, **kw except Exception as ex: # Fall back to legacy sublattice handling + if DISABLE_LEGACY_SUBLATTICES: + raise print("Falling back to legacy sublattice handling") return sub.serialize_to_json() diff --git a/covalent/_workflow/lattice.py b/covalent/_workflow/lattice.py index 84f74f6b1..146b837d9 100644 --- a/covalent/_workflow/lattice.py +++ b/covalent/_workflow/lattice.py @@ -47,7 +47,7 @@ from ..executor import BaseExecutor from ..triggers import BaseTrigger -from .._shared_files.utils import get_imports, get_serialized_function_str +from .._shared_files.utils import get_serialized_function_str consumable_constraints = [] @@ -81,10 +81,7 @@ def __init__( self.__doc__ = self.workflow_function.__doc__ self.post_processing = False self.inputs = None - self.named_args = None - self.named_kwargs = None self.electron_outputs = {} - self.lattice_imports, self.cova_imports = get_imports(self.workflow_function) self.workflow_function = TransportableObject.make_transportable(self.workflow_function) @@ -105,8 +102,6 @@ def serialize_to_json(self) -> str: attributes["transport_graph"] = self.transport_graph.serialize_to_json() attributes["inputs"] = self.inputs.to_dict() - attributes["named_args"] = self.named_args.to_dict() - attributes["named_kwargs"] = self.named_kwargs.to_dict() attributes["electron_outputs"] = {} for node_name, output in self.electron_outputs.items(): @@ -121,8 +116,6 @@ def deserialize_from_json(json_data: str) -> None: for node_name, object_dict in attributes["electron_outputs"].items(): attributes["electron_outputs"][node_name] = TransportableObject.from_dict(object_dict) - attributes["named_kwargs"] = TransportableObject.from_dict(attributes["named_kwargs"]) - attributes["named_args"] = TransportableObject.from_dict(attributes["named_args"]) attributes["inputs"] = TransportableObject.from_dict(attributes["inputs"]) if attributes["transport_graph"]: @@ -209,9 +202,6 @@ def build_graph(self, *args, **kwargs) -> None: new_kwargs = dict(named_kwargs.items()) self.inputs = TransportableObject({"args": args, "kwargs": kwargs}) - self.named_args = TransportableObject(named_args) - self.named_kwargs = TransportableObject(named_kwargs) - self.lattice_imports, self.cova_imports = get_imports(workflow_function) # Set any lattice metadata not explicitly set by the user constraint_names = {"executor", "workflow_executor", "hooks"} diff --git a/covalent/triggers/database_trigger.py b/covalent/triggers/database_trigger.py index d2a53402e..6baa0141c 100644 --- a/covalent/triggers/database_trigger.py +++ b/covalent/triggers/database_trigger.py @@ -19,9 +19,6 @@ from threading import Event from typing import List -from sqlalchemy import create_engine -from sqlalchemy.orm import Session - from covalent._shared_files import logger from .base import BaseTrigger @@ -88,6 +85,12 @@ def observe(self) -> None: where conditions are met or until stop has being called """ + # Since these modules are only used server-side, delay their + # imports to avoid introducing a sqlalchemy requirement to + # SDK-only installs + from sqlalchemy import create_engine + from sqlalchemy.orm import Session + app_log.debug("Inside DatabaseTrigger's observe") event_count = 0 diff --git a/covalent_dispatcher/_cli/cli.py b/covalent_dispatcher/_cli/cli.py index f24f24aaf..f352305e6 100644 --- a/covalent_dispatcher/_cli/cli.py +++ b/covalent_dispatcher/_cli/cli.py @@ -25,18 +25,7 @@ from rich.console import Console from .groups import db, deploy -from .service import ( - cluster, - config, - logs, - migrate_legacy_result_object, - print_header, - purge, - restart, - start, - status, - stop, -) +from .service import cluster, config, logs, print_header, purge, restart, start, status, stop # Main entrypoint @@ -73,7 +62,6 @@ def cli(ctx: click.Context, version: bool) -> None: cli.add_command(cluster) cli.add_command(db) cli.add_command(config) -cli.add_command(migrate_legacy_result_object) cli.add_command(deploy) if __name__ == "__main__": diff --git a/covalent_dispatcher/_cli/migrate.py b/covalent_dispatcher/_cli/migrate.py deleted file mode 100644 index 032aafbf0..000000000 --- a/covalent_dispatcher/_cli/migrate.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright 2021 Agnostiq Inc. -# -# This file is part of Covalent. -# -# Licensed under the Apache License 2.0 (the "License"). A copy of the -# License may be obtained with this software package or at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Use of this file is prohibited except in compliance with the License. -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utils for migrating legacy (0.110-era) result object to a modern result object.""" - -import pickle - -from covalent._results_manager import Result -from covalent._shared_files import logger -from covalent._shared_files.defaults import ( - attr_prefix, - electron_dict_prefix, - electron_list_prefix, - generator_prefix, - parameter_prefix, - subscript_prefix, -) -from covalent._shared_files.utils import get_named_params -from covalent._workflow.electron import to_decoded_electron_collection -from covalent._workflow.lattice import Lattice -from covalent._workflow.transport import TransportableObject, _TransportGraph, encode_metadata - -from .._db import update - -app_log = logger.app_log -log_stack_info = logger.log_stack_info - - -def process_node(node: dict) -> dict: - """Convert a node from a 0.110.2-vintage transport graph - - Args: - node: dictionary of node attributes - - Returns: - the converted node attributes - """ - - if "metadata" in node: - node["metadata"] = encode_metadata(node["metadata"]) - if "deps" not in node["metadata"]: - node["metadata"]["deps"] = {} - if "call_before" not in node["metadata"]: - node["metadata"]["call_before"] = [] - if "call_after" not in node["metadata"]: - node["metadata"]["call_after"] = [] - - node_name = node["name"] - - # encode output, remove "attribute_name", strip "attr_prefix" from name - if node_name.startswith(attr_prefix): - node["output"] = TransportableObject.make_transportable(node["output"]) - if "attribute_name" in node: - del node["attribute_name"] - new_node_name = node_name.replace(attr_prefix, "") - node["name"] = new_node_name - - # encode output, remove "key", strip "generator_prefix" from name - elif node_name.startswith(generator_prefix): - node["output"] = TransportableObject.make_transportable(node["output"]) - if "key" in node: - del node["key"] - new_node_name = node_name.replace(generator_prefix, "") - node["name"] = new_node_name - - # encode output, remove "key", strip "subscript_prefix" from name - elif node_name.startswith(subscript_prefix): - node["output"] = TransportableObject.make_transportable(node["output"]) - if "key" in node: - del node["key"] - new_node_name = node_name.replace(subscript_prefix, "") - node["name"] = new_node_name - - # Replace function for collection nodes - elif node_name.startswith(electron_list_prefix) or node_name.startswith(electron_dict_prefix): - node["function"] = TransportableObject(to_decoded_electron_collection) - - # Encode "value" and "output" for parameter nodes - elif node_name.startswith(parameter_prefix): - node["value"] = TransportableObject.make_transportable(node["value"]) - node["output"] = TransportableObject.make_transportable(node["output"]) - - # Function nodes: encode output and sublattice_result - else: - node["output"] = TransportableObject.make_transportable(node["output"]) - if "sublattice_result" in node: - if node["sublattice_result"] is not None: - node["sublattice_result"] = process_result_object(node["sublattice_result"]) - - return node - - -def process_transport_graph(tg: _TransportGraph) -> _TransportGraph: - """Convert a 0.110.2-vintage transport graph to a modern transport graph - - Args: - tg: old Transport Graph - - Returns: - the modernized Transport Graph - """ - tg_new = _TransportGraph() - g = tg.get_internal_graph_copy() - for node_id in g.nodes: - app_log.debug(f"Processing node {node_id}") - process_node(g.nodes[node_id]) - - if tg.lattice_metadata: - tg.lattice_metadata = encode_metadata(tg.lattice_metadata) - - tg_new._graph = g - return tg_new - - -def process_lattice(lattice: Lattice) -> Lattice: - """Convert a "legacy" (0.110.2) Lattice to a modern Lattice - - Args: - lattice: old lattice - - Returns: - the modernized lattice - """ - - workflow_function = lattice.workflow_function - lattice.workflow_function = TransportableObject.make_transportable(workflow_function) - inputs = {"args": lattice.args, "kwargs": lattice.kwargs} - lattice.inputs = TransportableObject(inputs) - - workflow_function = lattice.workflow_function.get_deserialized() - - named_args, named_kwargs = get_named_params(workflow_function, lattice.args, lattice.kwargs) - lattice.named_args = TransportableObject(named_args) - lattice.named_kwargs = TransportableObject(named_kwargs) - - metadata = lattice.metadata - - if "workflow_executor" not in metadata: - metadata["workflow_executor"] = "local" - - metadata = encode_metadata(metadata) - lattice.metadata = metadata - lattice.metadata["deps"] = {} - lattice.metadata["call_before"] = [] - lattice.metadata["call_after"] = [] - - lattice.transport_graph = process_transport_graph(lattice.transport_graph) - lattice.transport_graph.lattice_metadata = lattice.metadata - app_log.debug("Processed transport graph") - - # Delete raw inputs - del lattice.__dict__["args"] - del lattice.__dict__["kwargs"] - - return lattice - - -def process_result_object(result_object: Result) -> Result: - """Convert a "legacy" (0.110.2) Result object to a modern Result object - - Args: - result_object: the old Result object - - Returns: - the modernized result object - """ - - app_log.debug(f"Processing result object for dispatch {result_object.dispatch_id}") - process_lattice(result_object._lattice) - app_log.debug("Processed lattice") - - result_object._result = TransportableObject.make_transportable(result_object._result) - tg = result_object.lattice.transport_graph - for n in tg._graph.nodes: - tg.dirty_nodes.append(n) - - del result_object.__dict__["_inputs"] - return result_object - - -def migrate_pickled_result_object(path: str) -> None: - """Save legacy (0.110.2) result pickle file to a DataStore. - - This first transforms certain legacy properties of the result - object and then persists the result object to the datastore. - - Args: - path: path of the `result.pkl` file - """ - - with open(path, "rb") as f: - result_object = pickle.load(f) - - process_result_object(result_object) - update.persist(result_object) diff --git a/covalent_dispatcher/_cli/service.py b/covalent_dispatcher/_cli/service.py index df2299a8f..73d63da0b 100644 --- a/covalent_dispatcher/_cli/service.py +++ b/covalent_dispatcher/_cli/service.py @@ -56,7 +56,6 @@ from covalent._shared_files.config import ConfigManager, get_config, reload_config, set_config from .._db.datastore import DataStore -from .migrate import migrate_pickled_result_object UI_PIDFILE = get_config("dispatcher.cache_dir") + "/ui.pid" UI_LOGFILE = get_config("user_interface.log_dir") + "/covalent_ui.log" @@ -787,17 +786,6 @@ def logs() -> None: ) -@click.command() -@click.argument("result_pickle_path") -def migrate_legacy_result_object(result_pickle_path) -> None: - """Migrate a legacy result object - - Example: `covalent migrate-legacy-result-object result.pkl` - """ - - migrate_pickled_result_object(result_pickle_path) - - # Cluster CLI handlers (client side wrappers for the async handlers exposed # in the dask cluster process) async def _get_cluster_status(uri: str): diff --git a/covalent_dispatcher/_dal/db_interfaces/lattice_utils.py b/covalent_dispatcher/_dal/db_interfaces/lattice_utils.py index 676d0b68c..9871caefb 100644 --- a/covalent_dispatcher/_dal/db_interfaces/lattice_utils.py +++ b/covalent_dispatcher/_dal/db_interfaces/lattice_utils.py @@ -28,10 +28,6 @@ "name", "doc", "inputs", - "named_args", - "named_kwargs", - "cova_imports", - "lattice_imports", } METADATA_KEYS = lattice.LATTICE_METADATA_KEYS.copy() @@ -68,10 +64,6 @@ "workflow_function_string": "function_string_filename", "doc": "docstring_filename", "inputs": "inputs_filename", - "named_args": "named_args_filename", - "named_kwargs": "named_kwargs_filename", - "cova_imports": "cova_imports_filename", - "lattice_imports": "lattice_imports_filename", "executor_data": "executor_data_filename", "workflow_executor_data": "workflow_executor_data_filename", "hooks": "hooks_filename", diff --git a/covalent_dispatcher/_dal/importers/electron.py b/covalent_dispatcher/_dal/importers/electron.py index d4b5047c5..1f3ca51fc 100644 --- a/covalent_dispatcher/_dal/importers/electron.py +++ b/covalent_dispatcher/_dal/importers/electron.py @@ -133,6 +133,10 @@ def import_electron_assets( asset_recs = {} for asset_key, asset in e.assets: + # Register these later + if asset_key == "_custom": + continue + node_storage_path, object_key = object_store.get_uri_components( dispatch_id, e.id, @@ -157,8 +161,8 @@ def import_electron_assets( asset.remote_uri = f"file://{local_uri}" # Register custom assets - if e.custom_assets: - for asset_key, asset in e.custom_assets.items(): + if e.assets._custom: + for asset_key, asset in e.assets._custom.items(): object_key = f"{asset_key}.data" local_uri = os.path.join(node_storage_path, object_key) diff --git a/covalent_dispatcher/_dal/importers/lattice.py b/covalent_dispatcher/_dal/importers/lattice.py index a14938f98..55fa50925 100644 --- a/covalent_dispatcher/_dal/importers/lattice.py +++ b/covalent_dispatcher/_dal/importers/lattice.py @@ -24,16 +24,12 @@ from covalent._shared_files.config import get_config from covalent._shared_files.schemas.lattice import ( - LATTICE_COVA_IMPORTS_FILENAME, LATTICE_DOCSTRING_FILENAME, LATTICE_ERROR_FILENAME, LATTICE_FUNCTION_FILENAME, LATTICE_FUNCTION_STRING_FILENAME, LATTICE_HOOKS_FILENAME, LATTICE_INPUTS_FILENAME, - LATTICE_LATTICE_IMPORTS_FILENAME, - LATTICE_NAMED_ARGS_FILENAME, - LATTICE_NAMED_KWARGS_FILENAME, LATTICE_RESULTS_FILENAME, LATTICE_STORAGE_TYPE, LatticeAssets, @@ -71,12 +67,8 @@ def _get_lattice_meta(lat: LatticeSchema, storage_path) -> dict: "function_string_filename": LATTICE_FUNCTION_STRING_FILENAME, "error_filename": LATTICE_ERROR_FILENAME, "inputs_filename": LATTICE_INPUTS_FILENAME, - "named_args_filename": LATTICE_NAMED_ARGS_FILENAME, - "named_kwargs_filename": LATTICE_NAMED_KWARGS_FILENAME, "results_filename": LATTICE_RESULTS_FILENAME, "hooks_filename": LATTICE_HOOKS_FILENAME, - "cova_imports_filename": LATTICE_COVA_IMPORTS_FILENAME, - "lattice_imports_filename": LATTICE_LATTICE_IMPORTS_FILENAME, } kwargs.update(legacy_kwargs) return kwargs @@ -94,6 +86,10 @@ def import_lattice_assets( # Register built-in assets for asset_key, asset in lat.assets: + # Deal with these later + if asset_key == "_custom": + continue + storage_path, object_key = object_store.get_uri_components( dispatch_id=dispatch_id, node_id=None, @@ -118,8 +114,8 @@ def import_lattice_assets( asset.remote_uri = f"file://{local_uri}" # Register custom assets - if lat.custom_assets: - for asset_key, asset in lat.custom_assets.items(): + if lat.assets._custom: + for asset_key, asset in lat.assets._custom.items(): object_key = f"{asset_key}.data" local_uri = os.path.join(storage_path, object_key) diff --git a/covalent_dispatcher/_db/dispatchdb.py b/covalent_dispatcher/_db/dispatchdb.py index 621022777..e78a02d6d 100644 --- a/covalent_dispatcher/_db/dispatchdb.py +++ b/covalent_dispatcher/_db/dispatchdb.py @@ -20,7 +20,6 @@ from datetime import datetime import networkx as nx -import simplejson import covalent.executor as covalent_executor from covalent._shared_files import logger @@ -125,38 +124,6 @@ def result_encoder(obj): return str(obj) -def encode_result(result_obj): - lattice = result_obj.lattice - - result_string = result_obj.encoded_result.json - if not result_string: - result_string = result_obj.encoded_result.object_string - - named_args = {k: v.object_string for k, v in lattice.named_args.items()} - named_kwargs = {k: v.object_string for k, v in lattice.named_kwargs.items()} - result_dict = { - "dispatch_id": result_obj.dispatch_id, - "status": result_obj.status, - "result": result_string, - "start_time": result_obj.start_time, - "end_time": result_obj.end_time, - "results_dir": result_obj.results_dir, - "error": result_obj.error, - "lattice": { - "function_string": lattice.workflow_function_string, - "doc": lattice.__doc__, - "name": lattice.__name__, - "inputs": encode_dict({**named_args, **named_kwargs}), - "metadata": extract_metadata(lattice.metadata), - }, - "graph": extract_graph(result_obj.lattice.transport_graph._graph), - } - - jsonified_result = simplejson.dumps(result_dict, default=result_encoder, ignore_nan=True) - - return jsonified_result - - class DispatchDB: """ Wrapper for the database of workflows. diff --git a/covalent_dispatcher/_db/models.py b/covalent_dispatcher/_db/models.py index 7e0521c35..e61f725ef 100644 --- a/covalent_dispatcher/_db/models.py +++ b/covalent_dispatcher/_db/models.py @@ -92,10 +92,10 @@ class Lattice(Base): # Name of the file containing the serialized input data inputs_filename = Column(Text) - # Name of the file containing the serialized named args + # DEPRECATED: Name of the file containing the serialized named args named_args_filename = Column(Text) - # Name of the file containing the serialized named kwargs + # DEPRECATED: Name of the file containing the serialized named kwargs named_kwargs_filename = Column(Text) # name of the file containing the serialized output @@ -104,10 +104,10 @@ class Lattice(Base): # Name of the file containing the default electron hooks hooks_filename = Column(Text) - # Name of the file containing the set of cova imports + # DEPRECATED: Name of the file containing the set of cova imports cova_imports_filename = Column(Text) - # Name of the file containing the set of lattice imports + # DEPRECATED: Name of the file containing the set of lattice imports lattice_imports_filename = Column(Text) # Results directory (will be deprecated soon) diff --git a/covalent_dispatcher/_db/upsert.py b/covalent_dispatcher/_db/upsert.py index 3bd7f0ca7..70ef99a45 100644 --- a/covalent_dispatcher/_db/upsert.py +++ b/covalent_dispatcher/_db/upsert.py @@ -57,12 +57,8 @@ LATTICE_DOCSTRING_FILENAME = LATTICE_FILENAMES["doc"] LATTICE_ERROR_FILENAME = LATTICE_FILENAMES["error"] LATTICE_INPUTS_FILENAME = LATTICE_FILENAMES["inputs"] -LATTICE_NAMED_ARGS_FILENAME = LATTICE_FILENAMES["named_args"] -LATTICE_NAMED_KWARGS_FILENAME = LATTICE_FILENAMES["named_kwargs"] LATTICE_RESULTS_FILENAME = LATTICE_FILENAMES["result"] LATTICE_HOOKS_FILENAME = LATTICE_FILENAMES["hooks"] -LATTICE_COVA_IMPORTS_FILENAME = LATTICE_FILENAMES["cova_imports"] -LATTICE_LATTICE_IMPORTS_FILENAME = LATTICE_FILENAMES["lattice_imports"] LATTICE_STORAGE_TYPE = "file" CUSTOM_ASSETS_FIELD = "custom_asset_keys" @@ -108,12 +104,8 @@ def _lattice_data(session: Session, result: Result, electron_id: int = None) -> ("doc", LATTICE_DOCSTRING_FILENAME, result.lattice.__doc__), ("error", LATTICE_ERROR_FILENAME, result.error), ("inputs", LATTICE_INPUTS_FILENAME, result.lattice.inputs), - ("named_args", LATTICE_NAMED_ARGS_FILENAME, result.lattice.named_args), - ("named_kwargs", LATTICE_NAMED_KWARGS_FILENAME, result.lattice.named_kwargs), ("result", LATTICE_RESULTS_FILENAME, result._result), ("hooks", LATTICE_HOOKS_FILENAME, result.lattice.metadata["hooks"]), - ("cova_imports", LATTICE_COVA_IMPORTS_FILENAME, result.lattice.cova_imports), - ("lattice_imports", LATTICE_LATTICE_IMPORTS_FILENAME, result.lattice.lattice_imports), ]: digest, size = local_store.store_file(data_storage_path, filename, data) asset_record_kwargs = { @@ -161,12 +153,8 @@ def _lattice_data(session: Session, result: Result, electron_id: int = None) -> "workflow_executor_data": json.dumps(result.lattice.metadata["workflow_executor_data"]), "error_filename": LATTICE_ERROR_FILENAME, "inputs_filename": LATTICE_INPUTS_FILENAME, - "named_args_filename": LATTICE_NAMED_ARGS_FILENAME, - "named_kwargs_filename": LATTICE_NAMED_KWARGS_FILENAME, "results_filename": LATTICE_RESULTS_FILENAME, "hooks_filename": LATTICE_HOOKS_FILENAME, - "cova_imports_filename": LATTICE_COVA_IMPORTS_FILENAME, - "lattice_imports_filename": LATTICE_LATTICE_IMPORTS_FILENAME, "results_dir": results_dir, "root_dispatch_id": result.root_dispatch_id, "python_version": result.lattice.python_version, diff --git a/covalent_dispatcher/_db/write_result_to_db.py b/covalent_dispatcher/_db/write_result_to_db.py index 9d928c1ec..08da952ca 100644 --- a/covalent_dispatcher/_db/write_result_to_db.py +++ b/covalent_dispatcher/_db/write_result_to_db.py @@ -95,12 +95,8 @@ def transaction_insert_lattices_data( workflow_executor_data: str, error_filename: str, inputs_filename: str, - named_args_filename: str, - named_kwargs_filename: str, results_filename: str, hooks_filename: str, - cova_imports_filename: str, - lattice_imports_filename: str, results_dir: str, root_dispatch_id: str, created_at: dt, @@ -133,12 +129,8 @@ def transaction_insert_lattices_data( workflow_executor_data=workflow_executor_data, error_filename=error_filename, inputs_filename=inputs_filename, - named_args_filename=named_args_filename, - named_kwargs_filename=named_kwargs_filename, results_filename=results_filename, hooks_filename=hooks_filename, - cova_imports_filename=cova_imports_filename, - lattice_imports_filename=lattice_imports_filename, results_dir=results_dir, root_dispatch_id=root_dispatch_id, is_active=True, diff --git a/covalent_dispatcher/_service/app.py b/covalent_dispatcher/_service/app.py index 9a9c7d460..03e71186d 100644 --- a/covalent_dispatcher/_service/app.py +++ b/covalent_dispatcher/_service/app.py @@ -191,7 +191,7 @@ async def register(manifest: ResultSchema) -> ResultSchema: ) from e -@router.post("/dispatches/{dispatch_id}/subdispatches", status_code=201) +@router.post("/dispatches/{dispatch_id}/sublattices", status_code=201) async def register_subdispatch( manifest: ResultSchema, dispatch_id: str, diff --git a/covalent_dispatcher/_service/models.py b/covalent_dispatcher/_service/models.py index 2d2f7db10..18a33a071 100644 --- a/covalent_dispatcher/_service/models.py +++ b/covalent_dispatcher/_service/models.py @@ -41,11 +41,7 @@ class LatticeAssetKey(str, Enum): workflow_function_string = "workflow_function_string" doc = "doc" inputs = "inputs" - named_args = "named_args" - named_kwargs = "named_kwargs" hooks = "hooks" - cova_imports = "cova_imports" - lattice_imports = "lattice_imports" class ElectronAssetKey(str, Enum): diff --git a/covalent_ui/app.py b/covalent_ui/app.py index bf1d473eb..aa09d8854 100644 --- a/covalent_ui/app.py +++ b/covalent_ui/app.py @@ -133,7 +133,6 @@ def get_home(request: Request, rest_of_path: str): app_name, host=host, port=port, - debug=DEBUG, reload=RELOAD, log_config=log_config(), ) diff --git a/covalent_ui/result_webhook.py b/covalent_ui/result_webhook.py index 3caf03c10..f5d311421 100644 --- a/covalent_ui/result_webhook.py +++ b/covalent_ui/result_webhook.py @@ -22,7 +22,7 @@ import covalent_ui.app as ui_server from covalent._results_manager import Result from covalent._shared_files import logger -from covalent._shared_files.utils import get_ui_url +from covalent._shared_files.utils import get_named_params, get_ui_url from covalent_dispatcher._db.dispatchdb import encode_dict, extract_graph, extract_metadata app_log = logger.app_log @@ -78,8 +78,11 @@ def send_draw_request(lattice) -> None: graph = lattice.transport_graph.get_internal_graph_copy() - named_args = lattice.named_args.get_deserialized() - named_kwargs = lattice.named_kwargs.get_deserialized() + inputs = lattice.inputs.get_deserialized() + fn = lattice.workflow_function.get_deserialized() + args = inputs["args"] + kwargs = inputs["kwargs"] + named_args, named_kwargs = get_named_params(fn, args, kwargs) draw_request = json.dumps( { diff --git a/doc/source/api/cli.rst b/doc/source/api/cli.rst index 1dd83f117..22b5c1fea 100644 --- a/doc/source/api/cli.rst +++ b/doc/source/api/cli.rst @@ -5,5 +5,5 @@ The command line interface (CLI) tool is used to manage the Covalent server. .. click:: covalent_dispatcher._cli.cli:cli :prog: covalent - :commands: start,stop,restart,status,purge,logs,db,migrate-legacy-result-object,cluster + :commands: start,stop,restart,status,purge,logs,db,cluster :nested: full diff --git a/doc/source/version_migrations/index.rst b/doc/source/version_migrations/index.rst index 48995ae73..a93a38901 100644 --- a/doc/source/version_migrations/index.rst +++ b/doc/source/version_migrations/index.rst @@ -52,20 +52,4 @@ If you are using Covalent v0.110.2 or later you can upgrade to Covalent v0.177.0 $ covalent start Covalent server has started at http://localhost:48008 - -6. Use the data migration tool to migrate any workflows you want to port to the new version of Covalent. - - For example, for a workflow with dispatch ID :code:`e0ba03a2-fdc0-474e-9997-7fa8e82932c5`: - - .. code:: bash - - $ covalent migrate-legacy-result-object ./results/e0ba03a2-fdc0-474e-9997-7fa8e82932c5/result.pkl - Processing result object for dispatch e0ba03a2-fdc0-474e-9997-7fa8e82932c5 - Processing node 0 - Processing node 1 - Processing node 2 - Processing node 3 - Processed transport graph - Processed lattice - -7. Navigate to the UI (http://localhost:48008) to view your workflows. +6. Navigate to the UI (http://localhost:48008) to view your workflows. diff --git a/requirements-client.txt b/requirements-client.txt index ede6a20e3..1d74ffb04 100644 --- a/requirements-client.txt +++ b/requirements-client.txt @@ -7,6 +7,5 @@ furl>=2.1.3 networkx>=2.8.6 pydantic>=2.1.1 requests>=2.24.0 -simplejson>=3.17.6 toml>=0.10.2 watchdog>=2.0.3 diff --git a/requirements.txt b/requirements.txt index f9d0fa34c..15ecc9091 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,11 +16,9 @@ python-multipart>=0.0.6 python-socketio>=5.7.1 requests>=2.24.0 rich>=12.0.0,<=13.3.5 -simplejson>=3.17.6 sqlalchemy>=1.4.37,<2.0.0 sqlalchemy_utils>=0.38.3 toml>=0.10.2 typing-extensions>=4.8.0 -uvicorn[standard]==0.18.3 +uvicorn[standard] watchdog>=2.2.1 -werkzeug>=2.0.3 diff --git a/setup.py b/setup.py index 317a469c7..6762fdf6d 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,9 @@ with open("VERSION") as f: version = f.read().strip() +# Allow installing a particular commit for testing +commit_sha = os.getenv("COVALENT_COMMIT_SHA") +artifact_id = commit_sha if commit_sha else f"v{version}" requirements_file = "requirements.txt" exclude_modules = [ @@ -202,7 +205,7 @@ def find_sources(self): "version": version, "maintainer": "Agnostiq", "url": "https://github.com/AgnostiqHQ/covalent", - "download_url": f"https://github.com/AgnostiqHQ/covalent/archive/v{version}.tar.gz", + "download_url": f"https://github.com/AgnostiqHQ/covalent/archive/{artifact_id}.tar.gz", "license": "Apache License 2.0", "author": "Agnostiq", "author_email": "support@agnostiq.ai", diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py index a50b083da..aac119712 100644 --- a/tests/covalent_dispatcher_tests/_cli/cli_test.py +++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py @@ -61,7 +61,6 @@ def test_cli_commands(): "db", "deploy", "logs", - "migrate-legacy-result-object", "purge", "restart", "start", diff --git a/tests/covalent_dispatcher_tests/_cli/migrate_test.py b/tests/covalent_dispatcher_tests/_cli/migrate_test.py deleted file mode 100644 index 18289bcca..000000000 --- a/tests/covalent_dispatcher_tests/_cli/migrate_test.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright 2021 Agnostiq Inc. -# -# This file is part of Covalent. -# -# Licensed under the Apache License 2.0 (the "License"). A copy of the -# License may be obtained with this software package or at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Use of this file is prohibited except in compliance with the License. -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Testing results_dir migration script""" - -import pickle -from pathlib import Path - -from covalent._results_manager import Result -from covalent._shared_files.defaults import attr_prefix, generator_prefix, subscript_prefix -from covalent._workflow.transport import TransportableObject, _TransportGraph -from covalent_dispatcher._cli.migrate import ( - migrate_pickled_result_object, - process_lattice, - process_node, - process_result_object, - process_transport_graph, - to_decoded_electron_collection, -) - -dispatch_id = "652dc473-fa37-4846-85f3-b314204fd432" -sub_dispatch_id = "c333d0b3-8711-4595-9374-421f5482a592" - -basedir = Path(__file__).parent -sample_results_dir = basedir / Path("sample_results_dir") -result_pkl = sample_results_dir / dispatch_id / "result.pkl" - -# task node 0, parameter node 1 -# attribute node 2 -# sublattice node 3 -# task node 4, generator nodes 5, 6 -# subscript node 7 - - -def get_sample_result_object(): - with open(result_pkl, "rb") as f: - result_object = pickle.load(f) - return result_object - - -def compare_nodes_and_edges(tg_orig: _TransportGraph, tg_new: _TransportGraph): - """Convenience function for comparing a legacy transport graph with a processed one.""" - - # Check metadata - for n in tg_new._graph.nodes: - metadata = tg_new._graph.nodes[n]["metadata"] - assert "deps" in metadata - assert "call_before" in metadata - assert "call_after" in metadata - - # Check other node attributes - task_node = tg_new._graph.nodes[0] - orig_output = tg_orig._graph.nodes[0]["output"] - - assert isinstance(task_node["output"], TransportableObject) - assert task_node["output"].get_deserialized().__dict__ == orig_output.__dict__ - - collection_node = tg_new._graph.nodes[1] - assert ( - collection_node["function"].get_serialized() - == TransportableObject(to_decoded_electron_collection).get_serialized() - ) - - param_node = tg_new._graph.nodes[2] - orig_output = tg_orig._graph.nodes[2]["output"] - orig_value = tg_orig._graph.nodes[2]["value"] - - assert isinstance(param_node["output"], TransportableObject) - assert isinstance(param_node["value"], TransportableObject) - assert param_node["output"].get_deserialized() == orig_output - - param_node = tg_new._graph.nodes[3] - orig_output = tg_orig._graph.nodes[3]["output"] - orig_value = tg_orig._graph.nodes[3]["value"] - - assert isinstance(param_node["output"], TransportableObject) - assert isinstance(param_node["value"], TransportableObject) - assert param_node["output"].get_deserialized() == orig_output - - attr_node = tg_new._graph.nodes[4] - orig_output = tg_orig._graph.nodes[4]["output"] - - assert isinstance(attr_node["output"], TransportableObject) - assert attr_node["output"].get_deserialized() == orig_output - assert "attribute_name" not in attr_node - assert attr_prefix not in attr_node["name"] - - subl_node = tg_new._graph.nodes[5] - orig_output = tg_orig._graph.nodes[5]["output"] - - assert isinstance(subl_node["output"], TransportableObject) - assert isinstance(subl_node["sublattice_result"], Result) - assert subl_node["output"].get_deserialized() == orig_output - - task_node = tg_new._graph.nodes[6] - orig_output = tg_orig._graph.nodes[6]["output"] - - assert isinstance(task_node["output"], TransportableObject) - assert task_node["output"].get_deserialized() == orig_output - - gen_node = tg_new._graph.nodes[7] - orig_output = tg_orig._graph.nodes[7]["output"] - - assert isinstance(gen_node["output"], TransportableObject) - assert gen_node["output"].get_deserialized() == orig_output - assert "key" not in gen_node - assert generator_prefix not in gen_node["name"] - - gen_node = tg_new._graph.nodes[8] - orig_output = tg_orig._graph.nodes[8]["output"] - - assert isinstance(gen_node["output"], TransportableObject) - assert gen_node["output"].get_deserialized() == orig_output - assert "key" not in gen_node - assert generator_prefix not in gen_node["name"] - - subscript_node = tg_new._graph.nodes[9] - orig_output = tg_orig._graph.nodes[9]["output"] - - assert isinstance(subscript_node["output"], TransportableObject) - assert subscript_node["output"].get_deserialized() == orig_output - assert "key" not in subscript_node - assert subscript_prefix not in subscript_node["name"] - - assert tg_orig._graph.edges == tg_new._graph.edges - - -def test_process_legacy_node(): - """Test process_node""" - - ro = get_sample_result_object() - ro_orig = get_sample_result_object() - tg = ro.lattice.transport_graph - tg_orig = ro_orig.lattice.transport_graph - - task_node = tg._graph.nodes[0] - orig_output = tg_orig._graph.nodes[0]["output"] - process_node(task_node) - - param_node = tg._graph.nodes[2] - orig_output = tg_orig._graph.nodes[2]["output"] - orig_value = tg_orig._graph.nodes[2]["value"] - process_node(param_node) - - param_node = tg._graph.nodes[3] - orig_output = tg_orig._graph.nodes[3]["output"] - orig_value = tg_orig._graph.nodes[3]["value"] - process_node(param_node) - - attr_node = tg._graph.nodes[4] - orig_output = tg_orig._graph.nodes[4]["output"] - assert "attribute_name" in attr_node - assert attr_prefix in attr_node["name"] - process_node(attr_node) - - subl_node = tg._graph.nodes[5] - orig_output = tg_orig._graph.nodes[5]["output"] - assert "sublattice_result" in subl_node - process_node(subl_node) - - task_node = tg._graph.nodes[6] - orig_output = tg_orig._graph.nodes[6]["output"] - process_node(task_node) - - gen_node = tg._graph.nodes[7] - orig_output = tg_orig._graph.nodes[7]["output"] - assert "key" in gen_node - assert generator_prefix in gen_node["name"] - process_node(gen_node) - - gen_node = tg._graph.nodes[8] - orig_output = tg_orig._graph.nodes[8]["output"] - assert "key" in gen_node - assert generator_prefix in gen_node["name"] - process_node(gen_node) - - subscript_node = tg._graph.nodes[9] - orig_output = tg_orig._graph.nodes[9]["output"] - assert "key" in subscript_node - assert subscript_prefix in subscript_node["name"] - process_node(subscript_node) - - -def test_process_transport_graph(): - """Test process_transport_graph""" - - ro = get_sample_result_object() - - tg = ro.lattice.transport_graph - tg_new = process_transport_graph(tg) - compare_nodes_and_edges(tg, tg_new) - assert "dirty_nodes" in tg_new.__dict__ - - -def test_process_lattice(): - """Test process_lattice""" - - ro = get_sample_result_object() - ro_orig = get_sample_result_object() - lattice = process_lattice(ro._lattice) - lattice.named_args = lattice.named_args.get_deserialized() - lattice.named_kwargs = lattice.named_kwargs.get_deserialized() - - assert isinstance(lattice.workflow_function, TransportableObject) - assert list(lattice.named_args.keys()) == ["z"] - assert list(lattice.named_kwargs.keys()) == ["zz"] - assert lattice.metadata["executor_data"]["short_name"] == "local" - assert lattice.metadata["workflow_executor"] == "local" - assert lattice.metadata["workflow_executor_data"] == {} - assert lattice.metadata["deps"] == {} - assert lattice.metadata["call_before"] == [] - assert lattice.metadata["call_after"] == [] - - -def test_process_result_object(): - """Test process_result_object""" - - ro = get_sample_result_object() - old_inputs = ro._inputs - ro_new = process_result_object(ro) - inputs = ro_new.inputs.get_deserialized() - assert old_inputs["args"] == inputs["args"] - assert old_inputs["kwargs"] == inputs["kwargs"] - assert isinstance(ro_new._result, TransportableObject) - assert "dirty_nodes" in ro_new.lattice.transport_graph.__dict__ - - -def test_migrate_pickled_result_object(mocker): - """Test migrate_pickled_result_object""" - - mock_process_ro = mocker.patch("covalent_dispatcher._cli.migrate.process_result_object") - mock_persist = mocker.patch("covalent_dispatcher._db.update.persist") - - migrate_pickled_result_object(result_pkl) - mock_process_ro.assert_called_once() - mock_persist.assert_called_once() diff --git a/tests/covalent_dispatcher_tests/_cli/sample_results_dir/652dc473-fa37-4846-85f3-b314204fd432/result.pkl b/tests/covalent_dispatcher_tests/_cli/sample_results_dir/652dc473-fa37-4846-85f3-b314204fd432/result.pkl deleted file mode 100644 index 1fe2d20c734d50ccb615822c332f174301d5cfcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22463 zcmdUXd5k0JbsyQ?)6>&4JG)w~R)=kbnH2<%HN)o7oEeV2(iYhyyPIrsntkxlPUvD) zkspgi7Kc@AHrw5fHoUA(~!7hbv98;@RIIS_j*Om8bIl4vNy6`|454YT=VMbc%I*{fVJ6~i># z+p2ZjH=CNVVuOk+c>Y6?XxRqD4b~6`Q?{XSd z>-~;oHucVU^j=BR`*OD_wKOF#bv+(ERju`#n%V4B&0&{NHCoP!?V2p^(k)@f&t4cU zC9tMRNoI~Ca0$PO#2wRf?D-|{$=var>OGk^=Z@!bw_wiylIP{QlPf2S$5&+U`#mRD z<+&ZtIES9zPhw9OP0?sogX59+a6GzU^lR?oKmfchG&a%Bdo&)+9pP&pxEZs%qpQ@G zu15>_4p~28j59?0(X*P;5T&6zSfD!uG98qRGe48K;hA5WU&Lp5eraxe`?iTST0~au zic$-dSgnq8*nnzBY|}Mewnbpbx~w;`XVo!_^>eu#bsqiX6h^19Tdj)*f#hyP)lf`O zvdKwu?S!UfFR_QC8`bKQzNp#rq2TFiRqS+hG{yEAnC)adx>>EZ2C`BY`I#XQ(QW2_ z8F1=y-vUnqxgs0tX3e$QIld+E-hhbr)Cbu-3~$JDf;=`~TaruDNs>_dKb z+ixA2%oY6mD~SiD=TlzV^G|u+@V@C;^E`eIdp&=$a6I=0_WyX{%{i~<$upnws`JNl zReX*;$KFqSY5zZgJzYv_ie&bPGOsjErH#G30n}96iYel7i(~ePwthkEHBhzK8dw)k zD|WZ4UQ;mu_PmA90quM3^l=PGdPf#ll+J;5`tDkUv7$9=E8U^V^iCx3V8y`nckOMT zoYEDSd{&$(IUQXc^u%sg>5aGk*qd1P%q@AJ@!s?Cl74Jp)K~& z(X(d{@tD0d`iZl}0cXlFi~mTL1v+Mbb@U8qyroo?cGnzMB}LPS{T9Yd$JeXCFfP-~ z1Ra7vEHub;3M|_2a_aDMez0Wm18c8G*LK1G5}IfjU?%J?J~8*`XFime^EgM4eZ)Ra znC2DdXdJUmcFZy-W44uJ05i`7_9(};bFA=q%yv;e1nAFMpc^e<&eS^~Zkq0OA~c)1 z(VjT`Z3-gx+Gx=xrPl!-7Uv7yt3{Q1^g&FFMpo|yYY&ug{eCE1Usc4AEXm@SA98TK?DpKx1F=O z5jhySfA4;SDcG#SL-*L z9b-kKRe|Y^)|Y+wQ&9AO{eLySl7 zh6t$kPXZNYh4aoA}#o1i|U%z_nCBA;LbnI2p6g^STwx{4JdCt4S7}t-N-?;In=lD8(*C9#Nr|`Ln ztA*n$Z!UN}$2VqL%;Ol9{_C8_!x2IFOmccT7h68DBl~)a+3$5MWLgSXVA<^~Kn6W@ z?`Sv&(+_>SF7_>n;#TOg9wFNxJXMxHPu)*ED>nY__2( zV6QFk(7QIoexuF-z}jl36UoSjzAVnJPp|vBIz$_k2T}8(qThzXjRooiP2X_FQZ5+E z61We2YfY{{ioGEX*;0nkqp*nYdpx;mbVXBQ)uwDAt>0-r>4S1*fYq!wl^%^xw5%MO zN{@iH6XgbJR{>19}6SBC3K#akF%#_e~sR3or{12Q2%< z%I9p?cm$n_^ywRGHj+0wb$vW~rU89QRt%-r6t(7&B13_+h_rg3^bD+dJbI3|arOop zj0yCQM^E<@i_gI7k^vO-?WSQ}i3z3No)HA$41 zLbxA~-ZSVy@xu5$MQj^YQ?HU%jX3}jtiwJomo>fN0^r=xT+j^JuZ@|K~!@Xp!)7QDFrv&HLvl-=|$Q|Y%Cu1@O)qsR}i z3aF~=ccG5{$T2XbzF`Rj0f{Oi%OeY9cv;1)&y2D;ZZn0?JQM+4SCC!CCZ_6_w}0Ls1rC@d|G{UaEK z*Pu&CRW#S8)I7^M!~TGMjr}1${|=vj&%QQ#ztr#bz}JZxR7o2rI}E09QyaeqJ84om zW?$>q3_k0Z*(;VK#?|Xfe~D>Sf_`cP7$760$yC?}WUS0sENU35A(}YHD{`sr!VY0E zfTK^WmRTP~CJLo{-ac4Uwu|}^J|S0q)fl02)B3r@h*9rRyGs_a&c|d2ry%x1V_P^T zvw?@-MnQ&)sE}!-E z{;GG-*|8L?IY#eu&ghiKgbSVVDDjeHx|Z=W-Pjt!GF%|OY4?%;@U|sat#ifYGRl^h zmm#>^vtQ*RdCdL{hMda;(j=yF0ByJ9qTOipQys-5&<^3(UM(^UHYo@nV@jS1% zsM`lbStTQ`3hAw!JFwfL+3B3$;&~4IMKfHnRt}TxIo=LdjM5`bstOGe4wA=XV|1lT zK$2CxPz6AAai!V?tFo?)z9fy+SFp@}ZS;)2n-dohouIF%6~e--IDdN`uYCZs)6POI-fN6U53DPn z)Wnr`}zu>c5I4qx9NG8bn29*$7hA=b6pCQrb%n6nb3j6Go6AJMx@Md#rwj`klpx`YU{z(MS_$v)qa{J`#`>N$-g`YSpuKjstJnr*x@NM& zMNPTQS-B$u{6}|uM@Kd*f5Eb@h@bd^cYIy32d5KQn~#@-xKRjg%l9I0R}K1)!0*X0 ze!)i^tXTsewAzMw@gZ8Gec;e(1p#7Hq17zHn~_+thCL-HLeA63_8zp@SJDDRWa(6qwjal*CW+i80VWI{baaA=wZQ(|7W`M82MnbTE_gf*p}3zEwgl@hv6Twvj9O&BC8af03?FLQ z_Uyg3G02s|n<24y7!-@qy1cFpDwUmGlxKF+DYM)zuzjm6rKK`^nRFwe3T&+wE@(65 zNh7{9h-U@Eo5Bud}$eI_v;SaXLAIKL$ARfUze7 z&zLV%TQ3|Dth@SP3+1g!vBG5RnS}mY^4?Yg>#7xFEtJ_~D_e2YRnjrl?w3VBwU&+@ z$)!!+24j^;$%V@8&_QK=->8)GdnMpMLE``?<#x*8^R0)0GgB8f279G&h>5^C z_D8Fw_k*=?v<`gn{cS~QZ&*_sg_V^33VY} z7t-4Tr{k6(ux>$L(+5e+VN~e4ED{ar`JmC<9&89gWxJax?d3rBg@JFbbO0`eLa-G@x&k2+tusi@9oPjc9IHThHkm!Kl8O#HZLO ziHS;HOvv~!8Ebih@k6Y;lipL~`erK{s#kW>+6FdpJ!h8-{W1H!(IOrw211cBG{|ct z6ZrFE*v>W|3f_jVX)2Ck5@q0j>P?ufR+PcgIZ+7yo7JCl>fx6X&Au^Kp8;bWE$&!% zw#kC~U14``2a_i9TaU8I-Q?QXsNo?OS3t+?b27tA5eTUR{wx%Z39sKk8~rGD5k%coGx6- zQ3+XZaQ`{w@D^zFkQbJO|EGW2vE-;B{n=8Grt=>LxTDbU-@eVIB;?{BVJ?SG zOhf1l*6P9z6B}W}V7mpdr*fn)tcCAE9PD@H^u9@K zVQoF9;W(OLp{xQHS|+wX#lGZCVkHm*ZTSeSF)c=9=lW5_9Wz?5g*Ts6in$=##6^rN zl*Bz^)#YX^TrO=Kh-u8R6I08joTltN0Beu+E9vAxr5FuML$Jq4g=H}w7ezZULo99_ zNb6b<{rX$O)zxey2blLvDV>5iO?4`ztpkh^23yx^o$Z5CL_Uz(W*clak|p*4SfqBW zpY6n>fN5QhY%*y*u1VcO?%sB4EzuAX2_d;HY_G->0#2=9s$ii<^7V9l!s$OF z)*Z}hGa{GRxQh7?YF$fgR@$0T*(b4i*vtYyQZuI4!XYgi&Ot2iN3+8y!Q3kE#DIg! z5y~-k(!@9&z!%Mi54*tsKq(c_Z}ljvHV&k)7Od=qT=*T_Td>EbV3)8r4$Z9grSw&B*oP1!+g)P4n@tVkoc8}M73)2)R4x*oy{ycN5$7C1 z9m-{k!L4ns2J!QgSQCod@$`0lSJ*7%cLoWtW|y@?&8e{3`o4xer}DV632~Ti6xDbo zSPKES&c0}%)9@juEwLEuYsv7cus5p~iE)UXR+OtqVj`+Pf+!(XNlc{l{03Jc4&`FX zsI^n3y|<~!*<6K+QfL962emZEeV4>~tqrwfKSr>0q;_f`CRQJS&Q;L<(W7Scau_!T zk3g@i){Y*KibQZbs)Q<#*fS%Q^nSZs+Bf8Nt^z@YgUWPP{mEb)6Y6IO_^`xeyOD<~ z^OmYNGDJGVCPL#YOm6DiVV2x&vlKpQF|nzywW4|<+z?cuODbj$I8s5g zhPIgp&Os~b{Z;URkerSUX}`hWj>Kr;h*dzBY4%853v<<))QX)HM1Hax!@g%}>_jxb zCq;^u2v4lOI4io<|3(7q*;BI3AG$qO|Cq#!8ye`paqsK4P*(rvY|T?&_?%Pozfhz8 zZ~FSb_5m5vHtSJY z#(P%s;YpgXpU*a4Ja>6dPmk;mpIzJaFa33A1*g__{YxKrYG&64C{C{`#}}3BXXejc zQ}~=UsqeM z+ssl6+1>AVjnAJmlYjN>Qg4cXFoERbb*OO>59u_1BNQ=@htBf-+*tZENcH3bR4m0IhVb8FH;T~+OU~Vn3z71OnLU4aefX$jBW(Z*bW(Yy& z2)vEKj-xotC{vaKwIxONo73f5(g<;z3qs$vxnOfQxqx<+G*Vv3MkXJCEAY4v01AZG zC;Qk^kSbkAsH95!t^ifzhqE|3jkEn21Q%C2cS>PW)^c3H!Ist_%!1f2jf8=ih{yQ; zq#KeL7q`{`k5t-cwY1}>fW4@{eS&hmJXnYDY}LY!fQ8?L+yA+c)VBR5ahYuBDgOs- zd*IyhElAOVjJ+aZ+X3A)N%#}~V0Y@tVj~M%IuBdgQG$Ld9)ebw{^Da^xcm>txLdrcK!AAyV zzb|)eV_+u)0wX34y;-H6lPkWmAl!jNcNUe99<1@5OJWq5K%n4Vr8 z>`WOi{h4zG8dc*kMxgNHOq?<(r2a%HzqcpG@BMffSnD{m?M7OS#~npXtCiwG_!M1* zEVG}1f@Q!!fRc@UB|~8&%(gF3>K%HX@@=Nbw+Z8yEAe_Vq3Nk~V_%G?vRld1N_f}h zAz~t@7r~pk5*|OKv(iQ%eG>|%n9$(A>EdJR>&1qFHJJGHOE^sMfiZwKQljM9k_)94E2$aWuoFH3%Hcj414H%{~S%zC4nZ|IGUgwOzw z3c`2U-&iMqu%Ro7RwmJS0{V_&o)l<4ejs%U^$Ho;=_X_y`1S?#O9pnWlN-R-OwIxr zCv;w-M-r5DImyqMPCPqPPV+s3pK@y4P#=y8?x$7~BtPN13-2Fg+adDPb}MPkGLXPS z){<6P0o2BgCi3A;K z!38;P&5?>RdA%@%q11(pC*L&u(XpWd+G#)!u#Gm}4|@ZiQ~-|Jd*TM34Zs+K(N!mX zLegUAwn?{BEwO#r1wX0dIYO{f+B^$qx+$pfE*v*qLxdb8A7-t+e-TVYd(D@@opSb& z&SfgS-y*+093tSko0e>aa|5ssH5g3N>L3MQKh{#@23H(7sidD+*ZZ*n=%&T}$^w^V zY8k#&wX@#@pKx_4_^~zUA}!#8Su&cbVj4P6WE1{b%P7VEz<@iH&UJzWQyt(M{(tR| z93Yi6$%u(#2#zi|b*7y~g}PHFm$1L6lfo%Dmf*iXHD}hn&+o}4z%8&TM_EB{g>8L; z=pXwe!Ff{#J;KRCXKxe6EBS__gv(@l;*^4{3(4zod@MtC4Zd^G67)jCb%>u0l9lyGk?@wH$CD&mSB(9h(z@X#$QvE}^KvrD4J zU|8`~jp=-mxcV{-;7Qb@D&y7;d{iIf8V~gO&(s5bwMOMLX)$g@*;|^o#Y(KqDm*uf zf^OMgV^#KZh@yOFbFY}f>i<8bTAhtiJm!0RX=2qR)K4v=HIMe=q+&-Y&IJ6aBAU3_ zRO+jIc%rtK)K_o~65>CXlh91q-%L04H5$YY3z-lf1<6c3*LR)(QDg}FG6&+aAQ3wf zkOzspHNk#A9$^w0uXui3gcYcHQB;J3y|YmdrH?FoJTLeXdUx1sHkj!P2g4zslYH#k z=tzO(7Z5YH%QHm~RqsP?0vw4vDs;2lQS8shb=GV?}IqwYvZ$ zpmYMvY-_yR9$+v$iSxBs-y%68XZtRu;8^%d(?BZbY;@s5k?EV1qSKKB_t>3WfQHrf{YqGdidNWl4hMY7FIXX`7wPZ@z%kEZp6OXFd zd^)|6OIIJ|3%gb?9*I1OMRhtovZ7P1Hb}l=2>y5LaO~73 zTh_J_b|%;TE)k?%b=k~_leOL@;EY&x2rZG%?d%qWjofZ^U&!tyZFF7$)^x^+Gn5sU zGCMnV&dLJBn?9;?E#TbBE?utK4c6j!d`gF2qhW`mTrv>C&JVy|1IWvFCa-mu!EzFu zr18cBACV@dK7tX0b>i(AC{fx-`Xf$`bGAHn1Oh;%|L1Q_5puw46|nl;dgx1m&nhQ~ z-z+BM>-l`9`bgMaN4(~(6|vh~87MWT>#bWa6jk(5@y?IwpKT$M{@YKscuZIs{WKtEe)WA{Z1Q?qpx6H2(6n#%?o zNTDeR>|VdaA)bOzvN#C3^29G^$x~nsAak|ch2|Lefy4aHu z!qjO^b(yVj+fNRf-z_7kK{40}5XaBoIx`tc&y?NqQHQCi(P*~d%3T7>NUCV_vOzsS-5%2Etv< zLDGF}_RfU6&bWm04B>xxY_O_rAmlZ^X+H|krej)o91Gm?cr=vj;Stt=Kf*$M&%{0d z!#~1WdpjOs#hLR63!#bk&V7W1(8#wf)+6=yKEnE5LL{%!lXL&Y0g|^psb8?YKd>;* zdri~I?>Y5ufaJFjP{I96>Yw21f{4hB_lF3B{5v=;qf=2De-CA|cJLLRmqPhaZn(xj zM5CEN#S6nVev_t{4cDMhlrPZ$ZYT=S6SbGG{?_T5`%cY2L5=#`^z~JIAzb61;wPv6 zGyeV0`S-t|?|xtXMIa3R0*d3OfKUEQ1*2;`*^3AE<4sJ_c5kHqx98}qeA_+xF21i_*mpV{iLI6x`}l?g_4;Ln#usxOF1;#$F-A_H(KO5 zWe9)$+ca#gUsFF}&8Gf+6sZ4z2LBPKi>beEL+}}b;In5SQ1?*Ohwp{_cIp?<_%-(O zV<1*!j}a~VMHD<9t8d`vhe!oe-$eiO62N%M6y6l&Xj^!F!FcpRS*+ia#0TPCIdcF0 z-Bm@7+!e!-;N5%oqjl+iIibF^CmY?7aAEI+R%bL$o!la1nu0hn!g93o<^C zTQigY`LT~7`6m4MjxT~-nscLs82~4#H+W~Hp7UTIzdT5Ot3pBYZ7ER^avecKU`K)> zvN4qvTE-fpcDtGKr2!pj9sm-0a0n^bdpo3HBJwr_a>Of+6x!FL4ArSK4JNt;``j@`y@bB42EdCqO=bW0^M=X!We*EWghMD}q`)AkuRN#rTvQuk*DsbS` z%&u9zIS<}?Vf^9JZzwFe^t}`=#@glFe zi~JBh#IuqLv7hH7{`}pw7X5v1?(OewLVxTm5^h%h)LH5&B;2fg*Qq&$go~pUXH^$O zX_D$Pt+kBi5T0omTp@AQ8ucUC>WlOl3dnVoAiv#GO`yZ*3V%%!={ywRdTtwa9Sx}g zfr3a%n$j9%vOe~*!<}l-rQ=B zn$s$ofXO%pki)5_a09a1BmnqYX)PRc>d*3a1k!@K#jgPxpKeht@OuJiQQhO$1j1Fd zOxL!m_Yl|We;ptAQvpJNXv z%G_g{;c}IZzpgYhXS~I#o%%NkNuS5pXSq_Rev9z+!F$oLEUn&;+{OFrpvpa1y?6IP zv>v%zi-cCg!PUCF8bP4-{~@&VIP^ck71-LJ;)iH)M1ww|jrufQ8Mx}J-$rFl{SLKU I?$-kU59o08*8l(j diff --git a/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py b/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py index 819f88bc6..964e7cbc5 100644 --- a/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py +++ b/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py @@ -275,8 +275,8 @@ def test_import_result_with_custom_assets(mocker, test_db): prefix="covalent-" ) as srv_dir: manifest = get_mock_result(dispatch_id, sdk_dir) - manifest.lattice.custom_assets = {"custom_lattice_asset": AssetSchema(size=0)} - manifest.lattice.transport_graph.nodes[0].custom_assets = { + manifest.lattice.assets._custom = {"custom_lattice_asset": AssetSchema(size=0)} + manifest.lattice.transport_graph.nodes[0].assets._custom = { "custom_electron_asset": AssetSchema(size=0) } filtered_res = import_result(manifest, srv_dir, None) diff --git a/tests/covalent_dispatcher_tests/_dal/lattice_test.py b/tests/covalent_dispatcher_tests/_dal/lattice_test.py index 7a55ac23f..f0f2d9a1a 100644 --- a/tests/covalent_dispatcher_tests/_dal/lattice_test.py +++ b/tests/covalent_dispatcher_tests/_dal/lattice_test.py @@ -83,9 +83,6 @@ def test_lattice_attributes(test_db, mocker): workflow_function = lat.get_value("workflow_function").get_deserialized() assert workflow_function(42) == 42 - res.lattice.lattice_imports == lat.get_value("lattice_imports") - res.lattice.cova_imports == lat.get_value("cova_imports") - def test_lattice_restricted_attributes(test_db, mocker): res = get_mock_result() diff --git a/tests/covalent_dispatcher_tests/_db/update_test.py b/tests/covalent_dispatcher_tests/_db/update_test.py index 567c83bc9..6e7dfb4c9 100644 --- a/tests/covalent_dispatcher_tests/_db/update_test.py +++ b/tests/covalent_dispatcher_tests/_db/update_test.py @@ -154,19 +154,6 @@ def test_result_persist_workflow_1(test_db, result_1, mocker): assert executor_data["short_name"] == le.short_name() assert executor_data["attributes"] == le.__dict__ - saved_named_args = local_store.load_file( - storage_path=lattice_storage_path, filename=lattice_row.named_args_filename - ) - - saved_named_kwargs = local_store.load_file( - storage_path=lattice_storage_path, filename=lattice_row.named_kwargs_filename - ) - saved_named_args_raw = saved_named_args.get_deserialized() - saved_named_kwargs_raw = saved_named_kwargs.get_deserialized() - - assert saved_named_args_raw == {} - assert saved_named_kwargs_raw == {"a": 1, "b": 2} - # Check that the electron records are as expected assert len(electron_rows) == 6 for electron in electron_rows: diff --git a/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py b/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py index 759dbbe1b..310367df7 100644 --- a/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py +++ b/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py @@ -59,8 +59,6 @@ WORKFLOW_EXECUTOR_DATA_FILENAME = "workflow_executor_data.pkl" ERROR_FILENAME = "error.txt" INPUTS_FILENAME = "inputs.pkl" -NAMED_ARGS_FILENAME = "named_args.pkl" -NAMED_KWARGS_FILENAME = "named_kwargs.pkl" RESULTS_FILENAME = "results.pkl" VALUE_FILENAME = "value.pkl" STDOUT_FILENAME = "stdout.log" @@ -68,8 +66,6 @@ ERROR_FILENAME = "error.log" TRANSPORT_GRAPH_FILENAME = "transport_graph.pkl" HOOKS_FILENAME = "hooks.pkl" -COVA_IMPORTS_FILENAME = "cova_imports.json" -LATTICE_IMPORTS_FILENAME = "lattice_imports.txt" RESULTS_DIR = "/tmp/results" @@ -126,12 +122,8 @@ def get_lattice_kwargs( workflow_executor_data=json.dumps({}), error_filename=ERROR_FILENAME, inputs_filename=INPUTS_FILENAME, - named_args_filename=NAMED_ARGS_FILENAME, - named_kwargs_filename=NAMED_KWARGS_FILENAME, results_filename=RESULTS_FILENAME, hooks_filename=HOOKS_FILENAME, - cova_imports_filename=COVA_IMPORTS_FILENAME, - lattice_imports_filename=LATTICE_IMPORTS_FILENAME, results_dir=RESULTS_DIR, root_dispatch_id="dispatch_1", created_at=None, @@ -159,12 +151,8 @@ def get_lattice_kwargs( "workflow_executor_data": workflow_executor_data, "error_filename": error_filename, "inputs_filename": inputs_filename, - "named_args_filename": named_args_filename, - "named_kwargs_filename": named_kwargs_filename, "results_filename": results_filename, "hooks_filename": hooks_filename, - "cova_imports_filename": cova_imports_filename, - "lattice_imports_filename": lattice_imports_filename, "results_dir": results_dir, "root_dispatch_id": root_dispatch_id, "created_at": created_at, @@ -286,12 +274,8 @@ def test_insert_lattices_data(test_db, mocker): assert lattice.workflow_executor == "dask" assert lattice.error_filename == ERROR_FILENAME assert lattice.inputs_filename == INPUTS_FILENAME - assert lattice.named_args_filename == NAMED_ARGS_FILENAME - assert lattice.named_kwargs_filename == NAMED_KWARGS_FILENAME assert lattice.results_filename == RESULTS_FILENAME assert lattice.hooks_filename == HOOKS_FILENAME - assert lattice.cova_imports_filename == COVA_IMPORTS_FILENAME - assert lattice.lattice_imports_filename == LATTICE_IMPORTS_FILENAME assert lattice.results_dir == RESULTS_DIR assert lattice.root_dispatch_id == f"dispatch_{i + 1}" assert ( diff --git a/tests/covalent_dispatcher_tests/_service/app_test.py b/tests/covalent_dispatcher_tests/_service/app_test.py index 7877fe673..4615e35c5 100644 --- a/tests/covalent_dispatcher_tests/_service/app_test.py +++ b/tests/covalent_dispatcher_tests/_service/app_test.py @@ -206,7 +206,7 @@ def test_register_sublattice(mocker, app, client, mock_manifest): ) mocker.patch("covalent_dispatcher._service.app.cancel_all_with_status") resp = client.post( - "/api/v2/dispatches/parent_dispatch/subdispatches", + "/api/v2/dispatches/parent_dispatch/sublattices", data=mock_manifest.json(), ) diff --git a/tests/covalent_tests/serialize/lattice_serialization_test.py b/tests/covalent_tests/serialize/lattice_serialization_test.py index 4247b6230..709041bde 100644 --- a/tests/covalent_tests/serialize/lattice_serialization_test.py +++ b/tests/covalent_tests/serialize/lattice_serialization_test.py @@ -89,10 +89,10 @@ def workflow(x, y): with tempfile.TemporaryDirectory() as d: manifest = serialize_lattice(workflow, d) - assert ["custom_lat_asset"] == list(manifest.custom_assets.keys()) + assert ["custom_lat_asset"] == list(manifest.assets._custom.keys()) node_0 = manifest.transport_graph.nodes[0] - assert "custom_electron_asset" in node_0.custom_assets + assert "custom_electron_asset" in node_0.assets._custom node_1 = manifest.transport_graph.nodes[1] - assert not node_1.custom_assets + assert not node_1.assets._custom diff --git a/tests/covalent_tests/triggers/database_trigger_test.py b/tests/covalent_tests/triggers/database_trigger_test.py index 734eb2e06..8927417be 100644 --- a/tests/covalent_tests/triggers/database_trigger_test.py +++ b/tests/covalent_tests/triggers/database_trigger_test.py @@ -61,8 +61,8 @@ def test_database_trigger_observe(mocker, where_clauses, database_trigger): database_trigger.where_clauses = where_clauses database_trigger.trigger = mocker.MagicMock() - mock_db_engine = mocker.patch("covalent.triggers.database_trigger.create_engine") - mock_session = mocker.patch("covalent.triggers.database_trigger.Session") + mock_db_engine = mocker.patch("sqlalchemy.create_engine") + mock_session = mocker.patch("sqlalchemy.orm.Session") mock_event = mocker.patch("covalent.triggers.database_trigger.Event") mock_sleep = mocker.patch("covalent.triggers.database_trigger.time.sleep") diff --git a/tests/covalent_tests/workflow/dispatch_source_test.py b/tests/covalent_tests/workflow/dispatch_source_test.py deleted file mode 100644 index 94d588b6b..000000000 --- a/tests/covalent_tests/workflow/dispatch_source_test.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2021 Agnostiq Inc. -# -# This file is part of Covalent. -# -# Licensed under the Apache License 2.0 (the "License"). A copy of the -# License may be obtained with this software package or at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Use of this file is prohibited except in compliance with the License. -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for writing the dispatch_source.py file""" - -import pytest - -from covalent._results_manager.result import _filter_cova_decorators - -COVA_IMPORTS = {"covalent", "lattice", "electron", "ct", "cova", "etron"} - - -INPUT1 = "\n".join( - [ - "@covalent.electron(", - ' executor="local"', - ")", - "def identity(x):", - " return x", - "", - "@covalent.electron", - "@covalent.lattice", - "@covalent.electron(", - ' executor="local"', - ")", - "def double(x):", - " return 2*x", - ] -) - -INPUT2 = INPUT1.replace("covalent", "ct") -INPUT3 = INPUT1.replace("covalent", "cova") -INPUT4 = INPUT1.replace("ct.electron", "electron") -INPUT5 = INPUT1.replace("ct.electron", "etron") -INPUT6 = INPUT1.replace("ct.lattice", "lattice") - -OUTPUT1 = "\n".join( - [ - "# @covalent.electron(", - '# executor="local"', - "# )", - "def identity(x):", - " return x", - "", - "# @covalent.electron", - "# @covalent.lattice", - "# @covalent.electron(", - '# executor="local"', - "# )", - "def double(x):", - " return 2*x", - ] -) - -OUTPUT2 = OUTPUT1.replace("covalent", "ct") -OUTPUT3 = OUTPUT1.replace("covalent", "cova") -OUTPUT4 = OUTPUT1.replace("ct.electron", "electron") -OUTPUT5 = OUTPUT1.replace("ct.electron", "etron") -OUTPUT6 = OUTPUT1.replace("ct.lattice", "lattice") - - -@pytest.mark.parametrize( - "input_str, expected_str", - [ - (INPUT1, OUTPUT1), - (INPUT2, OUTPUT2), - (INPUT3, OUTPUT3), - (INPUT4, OUTPUT4), - (INPUT5, OUTPUT5), - (INPUT6, OUTPUT6), - ], -) -def test_filter_cova_decorators( - input_str, - expected_str, -): - """Test the filtering out of Covalent-related decorators.""" - - output_str = _filter_cova_decorators(input_str, COVA_IMPORTS) - - assert output_str == expected_str diff --git a/tests/covalent_tests/workflow/electron_test.py b/tests/covalent_tests/workflow/electron_test.py index 2d5936ed3..327673b6f 100644 --- a/tests/covalent_tests/workflow/electron_test.py +++ b/tests/covalent_tests/workflow/electron_test.py @@ -33,7 +33,6 @@ _build_sublattice_graph, filter_null_metadata, get_serialized_function_str, - to_decoded_electron_collection, ) from covalent._workflow.lattice import Lattice from covalent._workflow.transport import TransportableObject, encode_metadata @@ -238,20 +237,6 @@ def test_wait_for_post_processing_when_returning_waiting_electron(): assert workflow_2.workflow_function.get_deserialized()() == 64 -def test_collection_node_helper_electron(): - """Unit test for `to_decoded_electron_collection`""" - - list_collection = [ - TransportableObject.make_transportable(1), - TransportableObject.make_transportable(2), - ] - - dict_collection = {"a": list_collection[0], "b": list_collection[1]} - assert to_decoded_electron_collection(x=list_collection) == [1, 2] - - assert to_decoded_electron_collection(x=dict_collection) == {"a": 1, "b": 2} - - def test_injected_inputs_are_not_in_tg(): """Test that arguments to electrons injected by calldeps aren't added to the transport graph""" diff --git a/tests/covalent_tests/workflow/lattice_serialization_test.py b/tests/covalent_tests/workflow/lattice_serialization_test.py index 72d962d0d..4be41091a 100644 --- a/tests/covalent_tests/workflow/lattice_serialization_test.py +++ b/tests/covalent_tests/workflow/lattice_serialization_test.py @@ -55,7 +55,6 @@ def workflow(x): return f(x) workflow.build_graph(5) - workflow.cova_imports = ["dummy_module"] json_workflow = workflow.serialize_to_json() diff --git a/tests/covalent_ui_backend_tests/functional_tests/webhook_test.py b/tests/covalent_ui_backend_tests/functional_tests/webhook_test.py index 25b696f35..50e34e23d 100644 --- a/tests/covalent_ui_backend_tests/functional_tests/webhook_test.py +++ b/tests/covalent_ui_backend_tests/functional_tests/webhook_test.py @@ -70,10 +70,9 @@ async def test_send_update(): assert response is None -@pytest.mark.skip(reason="Test is breaking, need to fix see PR #1728") def test_send_draw_request(): """Test draw request""" - workflow = get_mock_simple_workflow() - lattice = Lattice.deserialize_from_json(workflow.serialize_to_json()) + lattice = get_mock_simple_workflow() + lattice.build_graph(3) response = send_draw_request(lattice) assert response is None From c08d2a9b24d3a663b7c91adbd042a7fa4aa1cca7 Mon Sep 17 00:00:00 2001 From: CovalentOpsBot Date: Wed, 29 May 2024 23:15:28 +0000 Subject: [PATCH 06/18] The new version will be 0.235.0-rc.0 --- CHANGELOG.md | 8 ++++++++ VERSION | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9411e401d..e4dc7a665 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +## [0.235.0-rc.0] - 2024-05-29 + +### Authors + +- Ara Ghukasyan <38226926+araghukas@users.noreply.github.com> +- Casey Jao + + ### Changed - Updated Slurm plugin docs to note possible SSH limitation diff --git a/VERSION b/VERSION index 9d818f2d5..abb9ba616 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.234.1-rc.0 \ No newline at end of file +0.235.0-rc.0 \ No newline at end of file From 4ab39bf09122331f2ae02e4230fb4e0fd91e94b1 Mon Sep 17 00:00:00 2001 From: Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:22:31 -0400 Subject: [PATCH 07/18] Making qelectron import within try except. (#1954) * moving supression of import error to try to deal with other errors until we update pennylane for qelectron * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * updated changelog --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: kessler-frost --- CHANGELOG.md | 4 ++++ covalent/__init__.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4dc7a665..982fa3bd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Fixed + +- Ignoring all errors when importing qelectrons instead of only `ImportError` + ## [0.235.0-rc.0] - 2024-05-29 ### Authors diff --git a/covalent/__init__.py b/covalent/__init__.py index 0cfc7bbe7..d644a57d6 100644 --- a/covalent/__init__.py +++ b/covalent/__init__.py @@ -51,7 +51,8 @@ from ._workflow.electron import wait # nopycln: import from .executor.utils import get_context # nopycln: import -with contextlib.suppress(ImportError): +with contextlib.suppress(Exception): + # try to load qelectron modules from ._workflow.qelectron import qelectron # nopycln: import from .quantum import QCluster # nopycln: import From 18fb20bf3919a88b44aa21995c552127cceb17c6 Mon Sep 17 00:00:00 2001 From: CovalentOpsBot Date: Mon, 10 Jun 2024 17:25:34 +0000 Subject: [PATCH 08/18] The new version will be 0.235.1-rc.0 --- CHANGELOG.md | 9 +++++++++ VERSION | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 982fa3bd3..2f670f14b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +## [0.235.1-rc.0] - 2024-06-10 + +### Authors + +- Santosh kumar <29346072+santoshkumarradha@users.noreply.github.com> +- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> +- Co-authored-by: kessler-frost + + ### Fixed - Ignoring all errors when importing qelectrons instead of only `ImportError` diff --git a/VERSION b/VERSION index abb9ba616..abee87d30 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.235.0-rc.0 \ No newline at end of file +0.235.1-rc.0 \ No newline at end of file From bde09e587efefd4d5003f11659b701c9d82f5c6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:16:08 -0400 Subject: [PATCH 09/18] Bump transformers (#1892) Bumps [transformers](https://github.com/huggingface/transformers) from 4.31.0 to 4.36.0. - [Release notes](https://github.com/huggingface/transformers/releases) - [Commits](https://github.com/huggingface/transformers/compare/v4.31.0...v4.36.0) --- updated-dependencies: - dependency-name: transformers dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham --- .../tutorials/0_ClassicalMachineLearning/genai/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt b/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt index d036f6ae7..1162956d1 100644 --- a/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt +++ b/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt @@ -7,5 +7,5 @@ Pillow==9.5.0 sentencepiece==0.1.99 streamlit==1.25.0 torch==2.0.1 -transformers==4.31.0 +transformers==4.36.0 xformers==0.0.21 From 2f86b14fd20b61ab4b528f63848a90f1d53a63c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:22:22 -0400 Subject: [PATCH 10/18] Bump tj-actions/changed-files from 31 to 41 in /.github/workflows (#1897) Bumps [tj-actions/changed-files](https://github.com/tj-actions/changed-files) from 31 to 41. - [Release notes](https://github.com/tj-actions/changed-files/releases) - [Changelog](https://github.com/tj-actions/changed-files/blob/main/HISTORY.md) - [Commits](https://github.com/tj-actions/changed-files/compare/v31...v41) --- updated-dependencies: - dependency-name: tj-actions/changed-files dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham --- .github/workflows/boilerplate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/boilerplate.yml b/.github/workflows/boilerplate.yml index 862b33f87..8b2732fd3 100644 --- a/.github/workflows/boilerplate.yml +++ b/.github/workflows/boilerplate.yml @@ -28,7 +28,7 @@ jobs: fetch-depth: 0 - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v31 + uses: tj-actions/changed-files@v41 with: files: | **/*.py From 484afbbfda8e5234fc8e47387d483b6fda7919ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:24:09 -0400 Subject: [PATCH 11/18] Bump transformers in /doc/source/tutorials/voice_cloning (#1904) Bumps [transformers](https://github.com/huggingface/transformers) from 4.33.3 to 4.36.0. - [Release notes](https://github.com/huggingface/transformers/releases) - [Commits](https://github.com/huggingface/transformers/compare/v4.33.3...v4.36.0) --- updated-dependencies: - dependency-name: transformers dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham --- doc/source/tutorials/voice_cloning/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tutorials/voice_cloning/requirements.txt b/doc/source/tutorials/voice_cloning/requirements.txt index 534f3b8e0..46354ffe1 100644 --- a/doc/source/tutorials/voice_cloning/requirements.txt +++ b/doc/source/tutorials/voice_cloning/requirements.txt @@ -8,5 +8,5 @@ soundfile==0.12.1 streamlit==1.28.1 torch==2.1.0 torchaudio==2.1.0 -transformers==4.33.3 +transformers==4.36.0 TTS==0.19.1 From 33a516c652a7c8dbdd3c60e4f2083948a3d8a1c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:27:57 -0400 Subject: [PATCH 12/18] Bump streamlit in /doc/source/tutorials/0_ClassicalMachineLearning/genai (#1905) Bumps [streamlit](https://github.com/streamlit/streamlit) from 1.25.0 to 1.30.0. - [Release notes](https://github.com/streamlit/streamlit/releases) - [Commits](https://github.com/streamlit/streamlit/compare/1.25.0...1.30.0) --- updated-dependencies: - dependency-name: streamlit dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham --- .../tutorials/0_ClassicalMachineLearning/genai/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt b/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt index 1162956d1..ad025cedc 100644 --- a/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt +++ b/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt @@ -5,7 +5,7 @@ diffusers==0.19.3 emoji==2.8.0 Pillow==9.5.0 sentencepiece==0.1.99 -streamlit==1.25.0 +streamlit==1.30.0 torch==2.0.1 transformers==4.36.0 xformers==0.0.21 From 9f3e32040766ee88bdc199cfdabbf7568e5aa5fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:28:47 -0400 Subject: [PATCH 13/18] Bump streamlit in /doc/source/tutorials/voice_cloning (#1906) Bumps [streamlit](https://github.com/streamlit/streamlit) from 1.28.1 to 1.30.0. - [Release notes](https://github.com/streamlit/streamlit/releases) - [Commits](https://github.com/streamlit/streamlit/compare/1.28.1...1.30.0) --- updated-dependencies: - dependency-name: streamlit dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham --- doc/source/tutorials/voice_cloning/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tutorials/voice_cloning/requirements.txt b/doc/source/tutorials/voice_cloning/requirements.txt index 46354ffe1..d9422ce39 100644 --- a/doc/source/tutorials/voice_cloning/requirements.txt +++ b/doc/source/tutorials/voice_cloning/requirements.txt @@ -5,7 +5,7 @@ pydub==0.25.1 pytube==15.0.0 scipy==1.11.3 soundfile==0.12.1 -streamlit==1.28.1 +streamlit==1.30.0 torch==2.1.0 torchaudio==2.1.0 transformers==4.36.0 From 5957b41cdf9a165ecf55915407db0da1079ba37a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:29:52 -0400 Subject: [PATCH 14/18] Bump pillow in /doc/source/tutorials/0_ClassicalMachineLearning/genai (#1958) Bumps [pillow](https://github.com/python-pillow/Pillow) from 9.5.0 to 10.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/9.5.0...10.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham --- .../tutorials/0_ClassicalMachineLearning/genai/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt b/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt index ad025cedc..2843addc7 100644 --- a/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt +++ b/doc/source/tutorials/0_ClassicalMachineLearning/genai/requirements.txt @@ -3,7 +3,7 @@ bs4==0.0.1 covalent-azurebatch-plugin==0.12.0 diffusers==0.19.3 emoji==2.8.0 -Pillow==9.5.0 +Pillow==10.3.0 sentencepiece==0.1.99 streamlit==1.30.0 torch==2.0.1 From 25a112789dac48860dca045261d6cc8814c5c8b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:30:55 -0400 Subject: [PATCH 15/18] Bump tar from 6.1.11 to 6.2.1 in /covalent_ui/webapp (#1957) Bumps [tar](https://github.com/isaacs/node-tar) from 6.1.11 to 6.2.1. - [Release notes](https://github.com/isaacs/node-tar/releases) - [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md) - [Commits](https://github.com/isaacs/node-tar/compare/v6.1.11...v6.2.1) --- updated-dependencies: - dependency-name: tar dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham --- covalent_ui/webapp/yarn.lock | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/covalent_ui/webapp/yarn.lock b/covalent_ui/webapp/yarn.lock index 9bd5de6ca..88dbe692d 100644 --- a/covalent_ui/webapp/yarn.lock +++ b/covalent_ui/webapp/yarn.lock @@ -8057,6 +8057,11 @@ minipass@^3.0.0, minipass@^3.1.1: dependencies: yallist "^4.0.0" +minipass@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-5.0.0.tgz#3e9788ffb90b694a5d0ec94479a45b5d8738133d" + integrity sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ== + minizlib@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" @@ -11379,13 +11384,13 @@ tapable@^1.0.0, tapable@^1.1.3: integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== tar@^6.0.2: - version "6.1.11" - resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621" - integrity sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA== + version "6.2.1" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" + integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A== dependencies: chownr "^2.0.0" fs-minipass "^2.0.0" - minipass "^3.0.0" + minipass "^5.0.0" minizlib "^2.1.1" mkdirp "^1.0.3" yallist "^4.0.0" From 4570455b8f5e790d33ca1fe583ede9b73e335600 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:32:28 -0400 Subject: [PATCH 16/18] Bump express from 4.17.3 to 4.19.2 in /covalent_ui/webapp (#1956) Bumps [express](https://github.com/expressjs/express) from 4.17.3 to 4.19.2. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/master/History.md) - [Commits](https://github.com/expressjs/express/compare/4.17.3...4.19.2) --- updated-dependencies: - dependency-name: express dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham --- covalent_ui/webapp/yarn.lock | 160 ++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 70 deletions(-) diff --git a/covalent_ui/webapp/yarn.lock b/covalent_ui/webapp/yarn.lock index 88dbe692d..c89ce4f22 100644 --- a/covalent_ui/webapp/yarn.lock +++ b/covalent_ui/webapp/yarn.lock @@ -3259,21 +3259,23 @@ bn.js@^5.2.1: resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.2.1.tgz#0bc527a6a0d18d0aa8d5b0538ce4a77dccfa7b70" integrity sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ== -body-parser@1.19.2: - version "1.19.2" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.2.tgz#4714ccd9c157d44797b8b5607d72c0b89952f26e" - integrity sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw== +body-parser@1.20.2: + version "1.20.2" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" + integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== dependencies: bytes "3.1.2" - content-type "~1.0.4" + content-type "~1.0.5" debug "2.6.9" - depd "~1.1.2" - http-errors "1.8.1" + depd "2.0.0" + destroy "1.2.0" + http-errors "2.0.0" iconv-lite "0.4.24" - on-finished "~2.3.0" - qs "6.9.7" - raw-body "2.4.3" + on-finished "2.4.1" + qs "6.11.0" + raw-body "2.5.2" type-is "~1.6.18" + unpipe "1.0.0" bonjour@^3.5.0: version "3.5.0" @@ -3958,6 +3960,11 @@ content-type@~1.0.4: resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== +content-type@~1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918" + integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== + convert-source-map@1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.7.0.tgz#17a2cb882d7f77d3490585e2ce6c524424a3a442" @@ -3982,10 +3989,10 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw= -cookie@0.4.2: - version "0.4.2" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432" - integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA== +cookie@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" + integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== copy-concurrently@^1.0.0: version "1.0.5" @@ -4613,6 +4620,11 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= +depd@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" + integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== + depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -4626,10 +4638,10 @@ des.js@^1.0.0: inherits "^2.0.1" minimalistic-assert "^1.0.0" -destroy@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" - integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= +destroy@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" + integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== detect-newline@^3.0.0: version "3.1.0" @@ -5426,37 +5438,38 @@ expect@^26.6.0, expect@^26.6.2: jest-regex-util "^26.0.0" express@^4.17.1: - version "4.17.3" - resolved "https://registry.yarnpkg.com/express/-/express-4.17.3.tgz#f6c7302194a4fb54271b73a1fe7a06478c8f85a1" - integrity sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg== + version "4.19.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" + integrity sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q== dependencies: accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.19.2" + body-parser "1.20.2" content-disposition "0.5.4" content-type "~1.0.4" - cookie "0.4.2" + cookie "0.6.0" cookie-signature "1.0.6" debug "2.6.9" - depd "~1.1.2" + depd "2.0.0" encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "~1.1.2" + finalhandler "1.2.0" fresh "0.5.2" + http-errors "2.0.0" merge-descriptors "1.0.1" methods "~1.1.2" - on-finished "~2.3.0" + on-finished "2.4.1" parseurl "~1.3.3" path-to-regexp "0.1.7" proxy-addr "~2.0.7" - qs "6.9.7" + qs "6.11.0" range-parser "~1.2.1" safe-buffer "5.2.1" - send "0.17.2" - serve-static "1.14.2" + send "0.18.0" + serve-static "1.15.0" setprototypeof "1.2.0" - statuses "~1.5.0" + statuses "2.0.1" type-is "~1.6.18" utils-merge "1.0.1" vary "~1.1.2" @@ -5598,17 +5611,17 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -finalhandler@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d" - integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== +finalhandler@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" + integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== dependencies: debug "2.6.9" encodeurl "~1.0.2" escape-html "~1.0.3" - on-finished "~2.3.0" + on-finished "2.4.1" parseurl "~1.3.3" - statuses "~1.5.0" + statuses "2.0.1" unpipe "~1.0.0" find-cache-dir@^2.1.0: @@ -6263,15 +6276,15 @@ http-deceiver@^1.2.7: resolved "https://registry.yarnpkg.com/http-deceiver/-/http-deceiver-1.2.7.tgz#fa7168944ab9a519d337cb0bec7284dc3e723d87" integrity sha1-+nFolEq5pRnTN8sL7HKE3D5yPYc= -http-errors@1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c" - integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g== +http-errors@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" + integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== dependencies: - depd "~1.1.2" + depd "2.0.0" inherits "2.0.4" setprototypeof "1.2.0" - statuses ">= 1.5.0 < 2" + statuses "2.0.1" toidentifier "1.0.1" http-errors@~1.6.2: @@ -8459,10 +8472,10 @@ obuf@^1.0.0, obuf@^1.1.2: resolved "https://registry.yarnpkg.com/obuf/-/obuf-1.1.2.tgz#09bea3343d41859ebd446292d11c9d4db619084e" integrity sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg== -on-finished@~2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" - integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= +on-finished@2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" + integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== dependencies: ee-first "1.1.1" @@ -9738,10 +9751,12 @@ q@^1.1.2: resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= -qs@6.9.7: - version "6.9.7" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.7.tgz#4610846871485e1e048f44ae3b94033f0e675afe" - integrity sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw== +qs@6.11.0: + version "6.11.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" + integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== + dependencies: + side-channel "^1.0.4" query-string@^4.1.0: version "4.3.4" @@ -9803,13 +9818,13 @@ range-parser@^1.2.1, range-parser@~1.2.1: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.4.3: - version "2.4.3" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.3.tgz#8f80305d11c2a0a545c2d9d89d7a0286fcead43c" - integrity sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g== +raw-body@2.5.2: + version "2.5.2" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a" + integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== dependencies: bytes "3.1.2" - http-errors "1.8.1" + http-errors "2.0.0" iconv-lite "0.4.24" unpipe "1.0.0" @@ -10664,24 +10679,24 @@ semver@^7.2.1, semver@^7.3.2, semver@^7.3.5: dependencies: lru-cache "^6.0.0" -send@0.17.2: - version "0.17.2" - resolved "https://registry.yarnpkg.com/send/-/send-0.17.2.tgz#926622f76601c41808012c8bf1688fe3906f7820" - integrity sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww== +send@0.18.0: + version "0.18.0" + resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" + integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== dependencies: debug "2.6.9" - depd "~1.1.2" - destroy "~1.0.4" + depd "2.0.0" + destroy "1.2.0" encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" fresh "0.5.2" - http-errors "1.8.1" + http-errors "2.0.0" mime "1.6.0" ms "2.1.3" - on-finished "~2.3.0" + on-finished "2.4.1" range-parser "~1.2.1" - statuses "~1.5.0" + statuses "2.0.1" serialize-javascript@^4.0.0: version "4.0.0" @@ -10710,15 +10725,15 @@ serve-index@^1.9.1: mime-types "~2.1.17" parseurl "~1.3.2" -serve-static@1.14.2: - version "1.14.2" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.2.tgz#722d6294b1d62626d41b43a013ece4598d292bfa" - integrity sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ== +serve-static@1.15.0: + version "1.15.0" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" + integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== dependencies: encodeurl "~1.0.2" escape-html "~1.0.3" parseurl "~1.3.3" - send "0.17.2" + send "0.18.0" set-blocking@^2.0.0: version "2.0.0" @@ -11081,7 +11096,12 @@ static-extend@^0.1.1: define-property "^0.2.5" object-copy "^0.1.0" -"statuses@>= 1.4.0 < 2", "statuses@>= 1.5.0 < 2", statuses@~1.5.0: +statuses@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" + integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== + +"statuses@>= 1.4.0 < 2": version "1.5.0" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= From 99e9c7f6bf61694c853ce837c8650d33890ae0d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:35:07 -0400 Subject: [PATCH 17/18] Bump transformers from 4.33.3 to 4.38.0 in /doc/source/tutorials/voice_cloning (#1955) * Bump transformers in /doc/source/tutorials/voice_cloning Bumps [transformers](https://github.com/huggingface/transformers) from 4.33.3 to 4.38.0. - [Release notes](https://github.com/huggingface/transformers/releases) - [Commits](https://github.com/huggingface/transformers/compare/v4.33.3...v4.38.0) --- updated-dependencies: - dependency-name: transformers dependency-type: direct:production ... Signed-off-by: dependabot[bot] * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- doc/source/tutorials/voice_cloning/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tutorials/voice_cloning/requirements.txt b/doc/source/tutorials/voice_cloning/requirements.txt index d9422ce39..0ef381da8 100644 --- a/doc/source/tutorials/voice_cloning/requirements.txt +++ b/doc/source/tutorials/voice_cloning/requirements.txt @@ -8,5 +8,5 @@ soundfile==0.12.1 streamlit==1.30.0 torch==2.1.0 torchaudio==2.1.0 -transformers==4.36.0 +transformers==4.38.0 TTS==0.19.1 From 50440bc6460e3d39f1462e8d3b464bf4622fdddb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:38:01 -0400 Subject: [PATCH 18/18] Bump follow-redirects from 1.15.3 to 1.15.6 in /covalent_ui/webapp (#1943) Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.3 to 1.15.6. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.3...v1.15.6) --- updated-dependencies: - dependency-name: follow-redirects dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Will Cunningham --- covalent_ui/webapp/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/covalent_ui/webapp/yarn.lock b/covalent_ui/webapp/yarn.lock index c89ce4f22..d373df4e8 100644 --- a/covalent_ui/webapp/yarn.lock +++ b/covalent_ui/webapp/yarn.lock @@ -5696,9 +5696,9 @@ flush-write-stream@^1.0.0: readable-stream "^2.3.6" follow-redirects@^1.0.0, follow-redirects@^1.15.0: - version "1.15.3" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.3.tgz#fe2f3ef2690afce7e82ed0b44db08165b207123a" - integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q== + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== for-in@^1.0.2: version "1.0.2"