From e50003008ee95db051272a0a51507a44ff2133b3 Mon Sep 17 00:00:00 2001 From: Will Cunningham Date: Tue, 8 Aug 2023 08:36:28 -0400 Subject: [PATCH 1/4] updating pre-commit packages (#1743) --- .pre-commit-config.yaml | 14 ++++++-------- CHANGELOG.md | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fce65a588..7d50c7623 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,23 +40,19 @@ repos: args: ["--profile", "black"] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 args: - "--max-complexity=50" - exclude: | - (?x)^( - tests/example_dispatch.py - )$ - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0 + rev: v3.0.1 hooks: - id: prettier types: [yaml, markdown] @@ -64,11 +60,13 @@ repos: (?x)^( meta.yaml| )$ + - repo: https://github.com/hadialqattan/pycln - rev: v2.1.5 + rev: v2.2.1 hooks: - id: pycln args: [--config=pyproject.toml] + - repo: https://github.com/Yelp/detect-secrets rev: v1.4.0 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index 044980d4f..e01a8a710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added prettier for markdown files. - Reduce the number of pinned version numbers in the `setup.py`, `requirements.txt`, and `requirements-client.txt` - Updated the `wci.yml` file with new features -- Bumped pre-commit prettier version +- Bumped pre-commit versions ### Added From cdd3022ea6211e9fac3499c0ba1035380396fc1a Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Tue, 8 Aug 2023 09:01:50 -0400 Subject: [PATCH 2/4] Fix for config file locking (#1719) * using filelock package for platform independency in locking the config file * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * updated changelog * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * trying to prevent the deadlock * going backwards to check failure point * fixing test * retracing * retracing * disabling test * trying to prevent the deadlock * disabling some tests temporarily * disabling some tests temporarily * disabling some tests temporarily * enabling the tests * cleaning up * added filelock to requirements * Update CHANGELOG.md * retracted tests/requirements.txt changes * fixed test and using >= in tests/requirements now * updated changelog and skipping remote fs test * fixing test * adding test * adding test --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 1 + covalent/_shared_files/config.py | 21 +++++------ requirements-client.txt | 1 + requirements.txt | 1 + .../shared_files/config_test.py | 26 ++++++++++++-- tests/functional_tests/docs_how_to_test.py | 2 ++ tests/requirements.txt | 35 +++++++++---------- 7 files changed, 57 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e01a8a710..06610d0aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Using `filelock` package now for platform independent file locking of config file. This should fix the failing tests as well as improve compatibility with Windows. - When stopping the server, we send the proper `SIGINT` signal to uvicorn instead of `SIGKILL` which allows the second part of the FastAPI `lifespan` to execute properly. - Fixed the outstanding incompatibities between front-end data layer and a postgres database - Reverted file-lock changes diff --git a/covalent/_shared_files/config.py b/covalent/_shared_files/config.py index 636903053..3bc9c16bb 100644 --- a/covalent/_shared_files/config.py +++ b/covalent/_shared_files/config.py @@ -18,7 +18,7 @@ # # Relief from the License may be granted by purchasing a commercial license. -import fcntl + import os import shutil from dataclasses import asdict @@ -27,6 +27,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union +import filelock import toml """Configuration manager.""" @@ -109,16 +110,16 @@ def update_nested_dict(old_dict, new_dict, override_existing: bool = True): else: old_dict.setdefault(key, value) - with open(self.config_file, "r+") as f: - fcntl.lockf(f, fcntl.LOCK_EX) - file_config = toml.load(f) + with filelock.FileLock(f"{self.config_file}.lock", timeout=1): + with open(self.config_file, "r+") as f: + file_config = toml.load(f) - update_nested_dict(self.config_data, file_config) - if new_entries: - update_nested_dict(self.config_data, new_entries, override_existing) + update_nested_dict(self.config_data, file_config) + if new_entries: + update_nested_dict(self.config_data, new_entries, override_existing) - # Writing it back to the file - self.write_config() + # Writing it back to the file + self.write_config() def read_config(self) -> None: """ @@ -143,8 +144,8 @@ def write_config(self) -> None: Returns: None """ + with open(self.config_file, "w") as f: - fcntl.lockf(f, fcntl.LOCK_EX) toml.dump(self.config_data, f) def purge_config(self) -> None: diff --git a/requirements-client.txt b/requirements-client.txt index 65c6547fd..2ad354644 100644 --- a/requirements-client.txt +++ b/requirements-client.txt @@ -2,6 +2,7 @@ aiofiles>=0.8.0 aiohttp>=3.8.1 cloudpickle>=2.0.0 dask[distributed]>=2022.6.0 +filelock>=3.12.2 furl>=2.1.3 networkx>=2.8.6 requests>=2.24.0 diff --git a/requirements.txt b/requirements.txt index 8c233258f..c7338385b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ click>=8.1.3 cloudpickle>=2.0.0 dask[distributed]>=2022.6.0 fastapi>=0.93.0 +filelock>=3.12.2 furl>=2.1.3 natsort>=8.4.0 networkx>=2.8.6 diff --git a/tests/covalent_tests/shared_files/config_test.py b/tests/covalent_tests/shared_files/config_test.py index cd772109a..e29cef0b9 100644 --- a/tests/covalent_tests/shared_files/config_test.py +++ b/tests/covalent_tests/shared_files/config_test.py @@ -219,12 +219,34 @@ def test_write_config(mocker): cm = ConfigManager() toml_dump_mock = mocker.patch("covalent._shared_files.config.toml.dump") open_mock = mocker.patch("covalent._shared_files.config.open") - lock_mock = mocker.patch("fcntl.lockf") mock_file = open_mock.return_value.__enter__.return_value cm.write_config() toml_dump_mock.assert_called_once_with(cm.config_data, mock_file) open_mock.assert_called_once_with(cm.config_file, "w") - lock_mock.assert_called_once() + + +def test_update_config(mocker): + """Test the update_config method for config manager.""" + + cm = ConfigManager() + + cm.config_file = "mock_config_file" + cm.config_data = {"mock_section": {"mock_dir": "initial_value"}} + # Cannot mock `update_nested_dict`` since it's defined within the function + + mock_filelock = mocker.patch("covalent._shared_files.config.filelock.FileLock") + mock_open = mocker.patch("covalent._shared_files.config.open") + mock_toml_load = mocker.patch("covalent._shared_files.config.toml.load") + + cm.write_config = mocker.Mock() + + cm.update_config() + + mock_filelock.assert_called_once_with("mock_config_file.lock", timeout=1) + mock_open.assert_called_once_with("mock_config_file", "r+") + mock_toml_load.assert_called_once_with(mock_open.return_value.__enter__.return_value) + + cm.write_config.assert_called_once() def test_config_manager_set(mocker): diff --git a/tests/functional_tests/docs_how_to_test.py b/tests/functional_tests/docs_how_to_test.py index 7ce8f2171..572156508 100644 --- a/tests/functional_tests/docs_how_to_test.py +++ b/tests/functional_tests/docs_how_to_test.py @@ -46,6 +46,8 @@ "creating_custom_executors.ipynb", "file_transfers_to_from_azure_blob.ipynb", "file_transfers_to_from_gcp_storage.ipynb", + "file_transfers_for_workflows_to_remote.ipynb", + "file_transfers_to_remote.ipynb", ] diff --git a/tests/requirements.txt b/tests/requirements.txt index fe4fa8d11..29415a9c8 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,18 +1,17 @@ -boto3==1.26.110 -detect-secrets==1.3.0 -flake8==5.0.4 -httpx==0.24.1 -isort==5.10.1 -locust==2.11.0 -mock==4.0.3 -nbconvert==6.5.1 -pennylane==0.25.1 -pre-commit==2.20.0 -pytest==7.1.3 -pytest-asyncio==0.21.0 -pytest-cov==3.0.0 -pytest-mock==3.8.2 -pytest-rerunfailures==10.2 -pytest_asyncio==0.21.0 -scikit-image==0.19.1 -scikit-learn==1.2.2 +boto3>=1.26.110 +detect-secrets>=1.3.0 +flake8>=5.0.4 +httpx>=0.24.1 +isort>=5.10.1 +locust>=2.11.0 +mock>=4.0.3 +nbconvert>=6.5.1 +pennylane>=0.25.1 +pre-commit>=2.20.0 +pytest>=7.1.3 +pytest-asyncio>=0.21.0 +pytest-cov>=3.0.0 +pytest-mock>=3.8.2 +pytest-rerunfailures>=10.2 +scikit-image>=0.19.1 +scikit-learn>=1.2.2 From a5f5cf925c105477963a56373fa05b2da82eab63 Mon Sep 17 00:00:00 2001 From: Alejandro Esquivel Date: Thu, 10 Aug 2023 06:45:07 -0700 Subject: [PATCH 3/4] Using nvmrc for node version in tests workflow (#1744) --- .github/workflows/tests.yml | 6 ++++++ CHANGELOG.md | 1 + 2 files changed, 7 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7674fd121..d18a6c170 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -164,6 +164,12 @@ jobs: pip install --no-cache-dir -r ./requirements.txt pip install --no-cache-dir -r ./tests/requirements.txt + - name: Set up Node + if: env.NEED_FRONTEND || env.BUILD_AND_RUN_ALL + uses: actions/setup-node@v3 + with: + node-version-file: 'covalent_ui/webapp/.nvmrc' + - name: Build webapp if: env.NEED_FRONTEND || env.BUILD_AND_RUN_ALL uses: nick-fields/retry@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 06610d0aa..c8636c956 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Operations +- Respecting node version as specified in `.nvmrc` file for tests workflow - Bumped versions in pre-commit config - Added prettier for markdown files. - Reduce the number of pinned version numbers in the `setup.py`, `requirements.txt`, and `requirements-client.txt` From eec8d7752f219649ae83789159e596f02a0fe92c Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Wed, 16 Aug 2023 01:16:29 -0400 Subject: [PATCH 4/4] Fixed decorator functional test failing for nightly (#1747) * fixed decorator functional test * updated changelog --- CHANGELOG.md | 1 + tests/functional_tests/workflow_stack_test.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8636c956..fdf31f80b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed test cases to adapt changes to SQLAlchemy version 1.4.49 - Ignored remote file transfer how-to functional tests. - Skipping a UI backend test for now +- Fixed `test_decorated_function` test case in functional tests ### Fixed diff --git a/tests/functional_tests/workflow_stack_test.py b/tests/functional_tests/workflow_stack_test.py index fd2b0c438..433e1efce 100644 --- a/tests/functional_tests/workflow_stack_test.py +++ b/tests/functional_tests/workflow_stack_test.py @@ -474,22 +474,20 @@ def test_decorated_function(): Test whether covalent works as intended on an already decorated function. """ - import pennylane as qml + def wrapper(func): + def inner(*args, **kwargs): + return func(*args, **kwargs) - import covalent as ct - - dev1 = qml.device("default.qubit", wires=1) + return inner @ct.electron - @qml.qnode(dev1) - def circuit(params): - qml.RX(params[0], wires=0) - qml.RY(params[1], wires=0) - return qml.expval(qml.PauliZ(0)) + @wrapper + def task(x): + return x**2 @ct.lattice def workflow(): - return circuit([0.54, 0.12]) + return task(x=5) dispatch_id = ct.dispatch(workflow)() workflow_result = rm.get_result(dispatch_id, wait=True)