diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7674fd121..d18a6c170 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -164,6 +164,12 @@ jobs: pip install --no-cache-dir -r ./requirements.txt pip install --no-cache-dir -r ./tests/requirements.txt + - name: Set up Node + if: env.NEED_FRONTEND || env.BUILD_AND_RUN_ALL + uses: actions/setup-node@v3 + with: + node-version-file: 'covalent_ui/webapp/.nvmrc' + - name: Build webapp if: env.NEED_FRONTEND || env.BUILD_AND_RUN_ALL uses: nick-fields/retry@v2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fce65a588..7d50c7623 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,23 +40,19 @@ repos: args: ["--profile", "black"] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 args: - "--max-complexity=50" - exclude: | - (?x)^( - tests/example_dispatch.py - )$ - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0 + rev: v3.0.1 hooks: - id: prettier types: [yaml, markdown] @@ -64,11 +60,13 @@ repos: (?x)^( meta.yaml| )$ + - repo: https://github.com/hadialqattan/pycln - rev: v2.1.5 + rev: v2.2.1 hooks: - id: pycln args: [--config=pyproject.toml] + - repo: https://github.com/Yelp/detect-secrets rev: v1.4.0 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index f76f0cf16..21ed37d34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Operations + - Added Python 3.11 to test suite +- Respecting node version as specified in `.nvmrc` file for testworkflow - Bumped versions in pre-commit config - Added prettier for markdown files. - Reduce the number of pinned version numbers in the `setup.py`, `requirements.txt`, and `requirements-client.txt` - Updated the `wci.yml` file with new features -- Bumped pre-commit prettier version +- Bumped pre-commit versions ### Added @@ -50,9 +52,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed test cases to adapt changes to SQLAlchemy version 1.4.49 - Ignored remote file transfer how-to functional tests. - Skipping a UI backend test for now +- Fixed `test_decorated_function` test case in functional tests ### Fixed +- Using `filelock` package now for platform independent file locking of config file. This should fix the failing tests as well as improve compatibility with Windows. - When stopping the server, we send the proper `SIGINT` signal to uvicorn instead of `SIGKILL` which allows the second part of the FastAPI `lifespan` to execute properly. - Fixed the outstanding incompatibities between front-end data layer and a postgres database - Reverted file-lock changes diff --git a/covalent/_shared_files/config.py b/covalent/_shared_files/config.py index 636903053..3bc9c16bb 100644 --- a/covalent/_shared_files/config.py +++ b/covalent/_shared_files/config.py @@ -18,7 +18,7 @@ # # Relief from the License may be granted by purchasing a commercial license. -import fcntl + import os import shutil from dataclasses import asdict @@ -27,6 +27,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union +import filelock import toml """Configuration manager.""" @@ -109,16 +110,16 @@ def update_nested_dict(old_dict, new_dict, override_existing: bool = True): else: old_dict.setdefault(key, value) - with open(self.config_file, "r+") as f: - fcntl.lockf(f, fcntl.LOCK_EX) - file_config = toml.load(f) + with filelock.FileLock(f"{self.config_file}.lock", timeout=1): + with open(self.config_file, "r+") as f: + file_config = toml.load(f) - update_nested_dict(self.config_data, file_config) - if new_entries: - update_nested_dict(self.config_data, new_entries, override_existing) + update_nested_dict(self.config_data, file_config) + if new_entries: + update_nested_dict(self.config_data, new_entries, override_existing) - # Writing it back to the file - self.write_config() + # Writing it back to the file + self.write_config() def read_config(self) -> None: """ @@ -143,8 +144,8 @@ def write_config(self) -> None: Returns: None """ + with open(self.config_file, "w") as f: - fcntl.lockf(f, fcntl.LOCK_EX) toml.dump(self.config_data, f) def purge_config(self) -> None: diff --git a/requirements-client.txt b/requirements-client.txt index 65c6547fd..2ad354644 100644 --- a/requirements-client.txt +++ b/requirements-client.txt @@ -2,6 +2,7 @@ aiofiles>=0.8.0 aiohttp>=3.8.1 cloudpickle>=2.0.0 dask[distributed]>=2022.6.0 +filelock>=3.12.2 furl>=2.1.3 networkx>=2.8.6 requests>=2.24.0 diff --git a/requirements.txt b/requirements.txt index 8c233258f..c7338385b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ click>=8.1.3 cloudpickle>=2.0.0 dask[distributed]>=2022.6.0 fastapi>=0.93.0 +filelock>=3.12.2 furl>=2.1.3 natsort>=8.4.0 networkx>=2.8.6 diff --git a/tests/covalent_tests/shared_files/config_test.py b/tests/covalent_tests/shared_files/config_test.py index cd772109a..e29cef0b9 100644 --- a/tests/covalent_tests/shared_files/config_test.py +++ b/tests/covalent_tests/shared_files/config_test.py @@ -219,12 +219,34 @@ def test_write_config(mocker): cm = ConfigManager() toml_dump_mock = mocker.patch("covalent._shared_files.config.toml.dump") open_mock = mocker.patch("covalent._shared_files.config.open") - lock_mock = mocker.patch("fcntl.lockf") mock_file = open_mock.return_value.__enter__.return_value cm.write_config() toml_dump_mock.assert_called_once_with(cm.config_data, mock_file) open_mock.assert_called_once_with(cm.config_file, "w") - lock_mock.assert_called_once() + + +def test_update_config(mocker): + """Test the update_config method for config manager.""" + + cm = ConfigManager() + + cm.config_file = "mock_config_file" + cm.config_data = {"mock_section": {"mock_dir": "initial_value"}} + # Cannot mock `update_nested_dict`` since it's defined within the function + + mock_filelock = mocker.patch("covalent._shared_files.config.filelock.FileLock") + mock_open = mocker.patch("covalent._shared_files.config.open") + mock_toml_load = mocker.patch("covalent._shared_files.config.toml.load") + + cm.write_config = mocker.Mock() + + cm.update_config() + + mock_filelock.assert_called_once_with("mock_config_file.lock", timeout=1) + mock_open.assert_called_once_with("mock_config_file", "r+") + mock_toml_load.assert_called_once_with(mock_open.return_value.__enter__.return_value) + + cm.write_config.assert_called_once() def test_config_manager_set(mocker): diff --git a/tests/functional_tests/docs_how_to_test.py b/tests/functional_tests/docs_how_to_test.py index 7ce8f2171..572156508 100644 --- a/tests/functional_tests/docs_how_to_test.py +++ b/tests/functional_tests/docs_how_to_test.py @@ -46,6 +46,8 @@ "creating_custom_executors.ipynb", "file_transfers_to_from_azure_blob.ipynb", "file_transfers_to_from_gcp_storage.ipynb", + "file_transfers_for_workflows_to_remote.ipynb", + "file_transfers_to_remote.ipynb", ] diff --git a/tests/functional_tests/workflow_stack_test.py b/tests/functional_tests/workflow_stack_test.py index fd2b0c438..433e1efce 100644 --- a/tests/functional_tests/workflow_stack_test.py +++ b/tests/functional_tests/workflow_stack_test.py @@ -474,22 +474,20 @@ def test_decorated_function(): Test whether covalent works as intended on an already decorated function. """ - import pennylane as qml + def wrapper(func): + def inner(*args, **kwargs): + return func(*args, **kwargs) - import covalent as ct - - dev1 = qml.device("default.qubit", wires=1) + return inner @ct.electron - @qml.qnode(dev1) - def circuit(params): - qml.RX(params[0], wires=0) - qml.RY(params[1], wires=0) - return qml.expval(qml.PauliZ(0)) + @wrapper + def task(x): + return x**2 @ct.lattice def workflow(): - return circuit([0.54, 0.12]) + return task(x=5) dispatch_id = ct.dispatch(workflow)() workflow_result = rm.get_result(dispatch_id, wait=True) diff --git a/tests/requirements.txt b/tests/requirements.txt index fe4fa8d11..29415a9c8 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,18 +1,17 @@ -boto3==1.26.110 -detect-secrets==1.3.0 -flake8==5.0.4 -httpx==0.24.1 -isort==5.10.1 -locust==2.11.0 -mock==4.0.3 -nbconvert==6.5.1 -pennylane==0.25.1 -pre-commit==2.20.0 -pytest==7.1.3 -pytest-asyncio==0.21.0 -pytest-cov==3.0.0 -pytest-mock==3.8.2 -pytest-rerunfailures==10.2 -pytest_asyncio==0.21.0 -scikit-image==0.19.1 -scikit-learn==1.2.2 +boto3>=1.26.110 +detect-secrets>=1.3.0 +flake8>=5.0.4 +httpx>=0.24.1 +isort>=5.10.1 +locust>=2.11.0 +mock>=4.0.3 +nbconvert>=6.5.1 +pennylane>=0.25.1 +pre-commit>=2.20.0 +pytest>=7.1.3 +pytest-asyncio>=0.21.0 +pytest-cov>=3.0.0 +pytest-mock>=3.8.2 +pytest-rerunfailures>=10.2 +scikit-image>=0.19.1 +scikit-learn>=1.2.2