Merge branch 'develop' into arosen93-patch-1

.github/workflows/tests.yml

@@ -164,6 +164,12 @@ jobs:
           pip install --no-cache-dir -r ./requirements.txt
           pip install --no-cache-dir -r ./tests/requirements.txt
 
+      - name: Set up Node
+        if: env.NEED_FRONTEND || env.BUILD_AND_RUN_ALL
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: 'covalent_ui/webapp/.nvmrc'
+
       - name: Build webapp
         if: env.NEED_FRONTEND || env.BUILD_AND_RUN_ALL
         uses: nick-fields/retry@v2

.pre-commit-config.yaml

@@ -40,35 +40,33 @@ repos:
         args: ["--profile", "black"]
 
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.7.0
     hooks:
       - id: black
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
         args:
           - "--max-complexity=50"
-        exclude: |
-          (?x)^(
-              tests/example_dispatch.py
-          )$
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.0
+    rev: v3.0.1
     hooks:
       - id: prettier
         types: [yaml, markdown]
         exclude: |
           (?x)^(
               meta.yaml|
           )$
+
   - repo: https://github.com/hadialqattan/pycln
-    rev: v2.1.5
+    rev: v2.2.1
     hooks:
       - id: pycln
         args: [--config=pyproject.toml]
+
   - repo: https://github.com/Yelp/detect-secrets
     rev: v1.4.0
     hooks:

CHANGELOG.md

@@ -9,12 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Operations
 
+
 - Added Python 3.11 to test suite
+- Respecting node version as specified in `.nvmrc` file for testworkflow
 - Bumped versions in pre-commit config
 - Added prettier for markdown files.
 - Reduce the number of pinned version numbers in the `setup.py`, `requirements.txt`, and `requirements-client.txt`
 - Updated the `wci.yml` file with new features
-- Bumped pre-commit prettier version
+- Bumped pre-commit versions
 
 ### Added
 
@@ -50,9 +52,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed test cases to adapt changes to SQLAlchemy version 1.4.49
 - Ignored remote file transfer how-to functional tests.
 - Skipping a UI backend test for now
+- Fixed `test_decorated_function` test case in functional tests
 
 ### Fixed
 
+- Using `filelock` package now for platform independent file locking of config file. This should fix the failing tests as well as improve compatibility with Windows.
 - When stopping the server, we send the proper `SIGINT` signal to uvicorn instead of `SIGKILL` which allows the second part of the FastAPI `lifespan` to execute properly.
 - Fixed the outstanding incompatibities between front-end data layer and a postgres database
 - Reverted file-lock changes

covalent/_shared_files/config.py

@@ -18,7 +18,7 @@
 #
 # Relief from the License may be granted by purchasing a commercial license.
 
-import fcntl
+
 import os
 import shutil
 from dataclasses import asdict
@@ -27,6 +27,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
+import filelock
 import toml
 
 """Configuration manager."""
@@ -109,16 +110,16 @@ def update_nested_dict(old_dict, new_dict, override_existing: bool = True):
                         else:
                             old_dict.setdefault(key, value)
 
-        with open(self.config_file, "r+") as f:
-            fcntl.lockf(f, fcntl.LOCK_EX)
-            file_config = toml.load(f)
+        with filelock.FileLock(f"{self.config_file}.lock", timeout=1):
+            with open(self.config_file, "r+") as f:
+                file_config = toml.load(f)
 
-            update_nested_dict(self.config_data, file_config)
-            if new_entries:
-                update_nested_dict(self.config_data, new_entries, override_existing)
+                update_nested_dict(self.config_data, file_config)
+                if new_entries:
+                    update_nested_dict(self.config_data, new_entries, override_existing)
 
-            # Writing it back to the file
-            self.write_config()
+                # Writing it back to the file
+                self.write_config()
 
     def read_config(self) -> None:
         """
@@ -143,8 +144,8 @@ def write_config(self) -> None:
         Returns:
             None
         """
+
         with open(self.config_file, "w") as f:
-            fcntl.lockf(f, fcntl.LOCK_EX)
             toml.dump(self.config_data, f)
 
     def purge_config(self) -> None:

requirements-client.txt

@@ -2,6 +2,7 @@ aiofiles>=0.8.0
 aiohttp>=3.8.1
 cloudpickle>=2.0.0
 dask[distributed]>=2022.6.0
+filelock>=3.12.2
 furl>=2.1.3
 networkx>=2.8.6
 requests>=2.24.0

requirements.txt

@@ -5,6 +5,7 @@ click>=8.1.3
 cloudpickle>=2.0.0
 dask[distributed]>=2022.6.0
 fastapi>=0.93.0
+filelock>=3.12.2
 furl>=2.1.3
 natsort>=8.4.0
 networkx>=2.8.6

tests/covalent_tests/shared_files/config_test.py

@@ -219,12 +219,34 @@ def test_write_config(mocker):
     cm = ConfigManager()
     toml_dump_mock = mocker.patch("covalent._shared_files.config.toml.dump")
     open_mock = mocker.patch("covalent._shared_files.config.open")
-    lock_mock = mocker.patch("fcntl.lockf")
     mock_file = open_mock.return_value.__enter__.return_value
     cm.write_config()
     toml_dump_mock.assert_called_once_with(cm.config_data, mock_file)
     open_mock.assert_called_once_with(cm.config_file, "w")
-    lock_mock.assert_called_once()
+
+
+def test_update_config(mocker):
+    """Test the update_config method for config manager."""
+
+    cm = ConfigManager()
+
+    cm.config_file = "mock_config_file"
+    cm.config_data = {"mock_section": {"mock_dir": "initial_value"}}
+    # Cannot mock `update_nested_dict`` since it's defined within the function
+
+    mock_filelock = mocker.patch("covalent._shared_files.config.filelock.FileLock")
+    mock_open = mocker.patch("covalent._shared_files.config.open")
+    mock_toml_load = mocker.patch("covalent._shared_files.config.toml.load")
+
+    cm.write_config = mocker.Mock()
+
+    cm.update_config()
+
+    mock_filelock.assert_called_once_with("mock_config_file.lock", timeout=1)
+    mock_open.assert_called_once_with("mock_config_file", "r+")
+    mock_toml_load.assert_called_once_with(mock_open.return_value.__enter__.return_value)
+
+    cm.write_config.assert_called_once()
 
 
 def test_config_manager_set(mocker):

tests/functional_tests/docs_how_to_test.py

@@ -46,6 +46,8 @@
     "creating_custom_executors.ipynb",
     "file_transfers_to_from_azure_blob.ipynb",
     "file_transfers_to_from_gcp_storage.ipynb",
+    "file_transfers_for_workflows_to_remote.ipynb",
+    "file_transfers_to_remote.ipynb",
 ]
 
 

tests/functional_tests/workflow_stack_test.py

@@ -474,22 +474,20 @@ def test_decorated_function():
     Test whether covalent works as intended on an already decorated function.
     """
 
-    import pennylane as qml
+    def wrapper(func):
+        def inner(*args, **kwargs):
+            return func(*args, **kwargs)
 
-    import covalent as ct
-
-    dev1 = qml.device("default.qubit", wires=1)
+        return inner
 
     @ct.electron
-    @qml.qnode(dev1)
-    def circuit(params):
-        qml.RX(params[0], wires=0)
-        qml.RY(params[1], wires=0)
-        return qml.expval(qml.PauliZ(0))
+    @wrapper
+    def task(x):
+        return x**2
 
     @ct.lattice
     def workflow():
-        return circuit([0.54, 0.12])
+        return task(x=5)
 
     dispatch_id = ct.dispatch(workflow)()
     workflow_result = rm.get_result(dispatch_id, wait=True)

tests/requirements.txt

@@ -1,18 +1,17 @@
-boto3==1.26.110
-detect-secrets==1.3.0
-flake8==5.0.4
-httpx==0.24.1
-isort==5.10.1
-locust==2.11.0
-mock==4.0.3
-nbconvert==6.5.1
-pennylane==0.25.1
-pre-commit==2.20.0
-pytest==7.1.3
-pytest-asyncio==0.21.0
-pytest-cov==3.0.0
-pytest-mock==3.8.2
-pytest-rerunfailures==10.2
-pytest_asyncio==0.21.0
-scikit-image==0.19.1
-scikit-learn==1.2.2
+boto3>=1.26.110
+detect-secrets>=1.3.0
+flake8>=5.0.4
+httpx>=0.24.1
+isort>=5.10.1
+locust>=2.11.0
+mock>=4.0.3
+nbconvert>=6.5.1
+pennylane>=0.25.1
+pre-commit>=2.20.0
+pytest>=7.1.3
+pytest-asyncio>=0.21.0
+pytest-cov>=3.0.0
+pytest-mock>=3.8.2
+pytest-rerunfailures>=10.2
+scikit-image>=0.19.1
+scikit-learn>=1.2.2