diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index fec7a6302..a1c0f9995 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -68,8 +68,8 @@ jobs: fail: "Copyleft,Error,Other" exclude: "^(pylint|aio[-_]*).*" exclude-license: 'Mozilla Public License 2.0 \(MPL 2.0\)' - with-totals: true - table-headers: true + totals: true + headers: true - name: Print report if: ${{ always() }} run: echo "${{ steps.license_check_report.outputs.report }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index d33a8db9c..0988d7d0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Tests + +- Move QA scripts from QA repo to Covalent functional tests. + ### Docs - Update requirements file for the tutorials: `1_QuantumMachineLearning/pennylane_kernel/source.ipynb` and `machine_learning/dnn_comparison.ipynb`. diff --git a/tests/functional_tests/qa_basic_workflow_test.py b/tests/functional_tests/qa_basic_workflow_test.py new file mode 100644 index 000000000..ec5de8e8e --- /dev/null +++ b/tests/functional_tests/qa_basic_workflow_test.py @@ -0,0 +1,46 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + + +"""QA script to test basic workflow functionality.""" + +import covalent as ct + + +def test_basic_workflow(): + """Test the basic workflow functionality.""" + + @ct.electron + def join_words(a, b): + return ", ".join([a, b]) + + @ct.electron + def excitement(a): + return f"{a}!" + + @ct.lattice + def simple_workflow(a, b): + phrase = join_words(a, b) + return excitement(phrase) + + dispatch_id = ct.dispatch(simple_workflow)("Hello", "World") + res = ct.get_result(dispatch_id, wait=True) + assert res.result == "Hello, World!" + assert res.status == "COMPLETED" diff --git a/tests/functional_tests/qa_call_deps_retval_test.py b/tests/functional_tests/qa_call_deps_retval_test.py new file mode 100644 index 000000000..bcdb8b2e4 --- /dev/null +++ b/tests/functional_tests/qa_call_deps_retval_test.py @@ -0,0 +1,66 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +"""QA script to test the call deps return value functionality.""" + + +import covalent as ct + + +def test_call_deps_retval(): + """Test the call deps return value functionality.""" + + def greeting_one(): + return "hello" + + def greeting_two(): + return "howdy" + + def greeting_three(): + return "hey" + + @ct.electron( + call_before=[ + ct.DepsCall(greeting_one, retval_keyword="greetings"), + ct.DepsCall(greeting_two, retval_keyword="greetings"), + ] + ) + def get_all_greetings(greetings=[]): + return greetings + + @ct.electron( + call_before=[ + ct.DepsCall(greeting_three, retval_keyword="greeting"), + ] + ) + def get_one_greeting(greeting=None): + return f"{greeting}, whats up?" + + @ct.lattice + def workflow(): + greetings = get_all_greetings() + greeting = get_one_greeting() + return [greeting] + greetings + + # Dispatch the workflow + dispatch_id = ct.dispatch(workflow)() + res = ct.get_result(dispatch_id, wait=True) + assert res.result == ["hey, whats up?", "hello", "howdy"] + assert res.status == "COMPLETED" diff --git a/tests/functional_tests/qa_cpu_stress_test.py b/tests/functional_tests/qa_cpu_stress_test.py new file mode 100644 index 000000000..b32014fe5 --- /dev/null +++ b/tests/functional_tests/qa_cpu_stress_test.py @@ -0,0 +1,54 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +"""CPU stress test.""" + +import covalent as ct + +N = 8 +STALL = 1e6 +X = 50 + + +def test_cpu_stress(): + """Stress test the CPU.""" + + @ct.electron + def cpu_workload(stall, x): + i = 0 + while i < stall: + x * x + i += 1 + + @ct.lattice + def workflow(iterations, stall, x): + for _ in range(iterations): + cpu_workload(stall, x) + + n_electrons = [2**i for i in range(N)] + execution_time_taken = [] + dispatch_ids = [ct.dispatch(workflow)(it, STALL, X) for it in n_electrons] + + for d_id in dispatch_ids: + result = ct.get_result(d_id, wait=True) + execution_time_taken.append((result.end_time - result.start_time).total_seconds()) + + for time_taken in execution_time_taken: + assert time_taken < 60 diff --git a/tests/functional_tests/qa_deps_workflow_test.py b/tests/functional_tests/qa_deps_workflow_test.py new file mode 100644 index 000000000..1cb8476e4 --- /dev/null +++ b/tests/functional_tests/qa_deps_workflow_test.py @@ -0,0 +1,59 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +"""Test the deps functionality.""" + +from pathlib import Path + +import numpy + +import covalent as ct +from covalent import DepsBash, DepsCall, DepsPip + + +def test_deps_workflow(): + """Test the deps functionality.""" + + deps_pip = DepsPip(packages=["numpy==1.23.1"]) + deps_bash = DepsBash(commands=["echo $HOME >> /tmp/deps_bash_test.txt"]) + + def deps_call(): + Path("/tmp/deps_bash_test.txt").unlink() + + @ct.electron( + call_before=[deps_pip, deps_bash], + call_after=[DepsCall(deps_call)], + ) + def get_deps_results(): + results = [] + with open("/tmp/deps_bash_test.txt", "r") as f: + results.append(f.read()) + results.append(numpy.sum(numpy.identity(3))) + return results + + @ct.lattice + def workflow(): + return get_deps_results() + + # Dispatch the workflow + dispatch_id = ct.dispatch(workflow)() + res = ct.get_result(dispatch_id, wait=True) + assert int(res.result[1]) == 3 + assert res.status == "COMPLETED" diff --git a/tests/functional_tests/qa_heterogenous_workflows_test.py b/tests/functional_tests/qa_heterogenous_workflows_test.py new file mode 100644 index 000000000..c89e352c2 --- /dev/null +++ b/tests/functional_tests/qa_heterogenous_workflows_test.py @@ -0,0 +1,89 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +"""Test different types of workflows.""" + +import enum + +import pytest + +import covalent as ct + + +class WORKFLOW_TYPES(str, enum.Enum): + HORIZONTAL = "HORIZONTAL" + VERTICAL = "VERTICAL" + HAGRID = "HAGRID" + + +@pytest.mark.parametrize( + "workflow_type, parallel, serial, n", + [ + (WORKFLOW_TYPES.HORIZONTAL, True, False, 8), + (WORKFLOW_TYPES.VERTICAL, False, True, 10), + (WORKFLOW_TYPES.HAGRID, True, True, 10), + ], +) +def test_heterogenous_workflows(workflow_type, parallel, serial, n): + """Test different types of workflows.""" + + @ct.electron + def identity(x): + return x + + @ct.electron + def combine(x): + return sum(x) + + @ct.lattice + def workflow(n, parallel=False, serial=False): + vals = [] + result = 186282 + nodes = range(n) + + if parallel and not serial: + for _ in nodes: + vals.append(identity(1)) + result = combine(vals) + + elif serial and not parallel: + for _ in nodes: + result = identity(result) + elif serial and parallel: + for i in nodes: + for _ in nodes: + if i == 0: + vals.append(identity(1)) + else: + vals.append(identity(result)) + result = combine(vals) + return result + + dispatch_id = ct.dispatch(workflow)(n, parallel, serial) + res = ct.get_result(dispatch_id, wait=True) + + if workflow_type == WORKFLOW_TYPES.HORIZONTAL: + assert res.result == 8 + elif workflow_type == WORKFLOW_TYPES.VERTICAL: + assert res.result == 186282 + elif workflow_type == WORKFLOW_TYPES.HAGRID: + assert res.result == 23579476910 + + assert res.status == "COMPLETED" diff --git a/tests/functional_tests/qa_http_file_transfer_test.py b/tests/functional_tests/qa_http_file_transfer_test.py new file mode 100644 index 000000000..b3f4e00f7 --- /dev/null +++ b/tests/functional_tests/qa_http_file_transfer_test.py @@ -0,0 +1,52 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +"""Test http file transfer.""" + +import covalent as ct + + +def test_http_file_transfer(): + """Test http file transfer.""" + + @ct.electron( + files=[ + ct.fs.TransferFromRemote( + "https://raw.githubusercontent.com/curran/data/gh-pages/dataSoup/datasets.csv" + ), + ] + ) + def get_remote_file(files=[]): + source_path, dest_path = files[0] + with open(dest_path, "r") as f: + return f.read() + + @ct.lattice + def workflow(): + return get_remote_file() + + dispatch_id = ct.dispatch(workflow)() + res = ct.get_result(dispatch_id, wait=True) + result_string = res.result + + assert result_string[:12] == "Dataset Name" + assert result_string[-12:] == "05,,JSON,,,," + assert len(result_string) == 9248 + assert res.status == "COMPLETED" diff --git a/tests/functional_tests/qa_sublattice_stress_test.py b/tests/functional_tests/qa_sublattice_stress_test.py new file mode 100644 index 000000000..a2ab31ae6 --- /dev/null +++ b/tests/functional_tests/qa_sublattice_stress_test.py @@ -0,0 +1,118 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +"""Sublattice stress test.""" + +import numpy as np + +import covalent as ct + +N = 5 +STALL = 1e5 +X = 20 + + +def test_sublattice_stress(): + """Stress test the sublattice.""" + + @ct.electron + def matrix_workload(mat_1, mat_2, stall=STALL, x=X): + i = 0 + while i < stall: + x * x + i += 1 + + return mat_1 + + @ct.electron + def matrix_flatten(mat_1): + return np.ravel(mat_1) + + @ct.electron + def split_in_half(mat_1): + return np.array_split(mat_1, 2) + + @ct.lattice + def inflate(mat_1, dim=3): + mat_2 = np.random.default_rng().integers(3, size=(dim, dim)) + + mat_3 = matrix_workload(mat_1, mat_2) + + mat_4 = matrix_flatten(mat_3) + + mat_5 = matrix_workload(mat_1, mat_3) + + return matrix_workload(mat_4, mat_5) + + @ct.lattice + def deflate(mat_1, dim=3): + mat_2 = np.random.default_rng().integers(3, size=(dim, dim)) + + mat_3, mat_4 = split_in_half(mat_1) + + mat_5 = matrix_workload(mat_1, mat_3) + + mat_6 = matrix_workload(mat_2, mat_4) + + mat_7 = matrix_flatten(mat_5) + + mat_8, mat_9 = split_in_half(mat_6) + + return mat_8, matrix_workload(mat_7, mat_9) + + @ct.electron + @ct.lattice + def idi(mat_1): + mat_2 = inflate(mat_1) + mat_3, mat_4 = deflate(mat_2) + mat_5 = inflate(mat_3) + + return matrix_workload(mat_4, mat_5) + + @ct.lattice + def workflow(dim=3): + mat_1 = np.random.default_rng().integers(10, size=(dim, dim)) + + mat_2, mat_3 = split_in_half(mat_1) + + mat_4 = matrix_workload(mat_2, mat_3) + + mat_5 = inflate(mat_4, dim) + + mat_6 = matrix_flatten(mat_5) + + mat_7 = inflate(mat_5) + + mat_8, mat_9 = deflate(mat_6) + + mat_10 = idi(mat_9) + + return matrix_workload(matrix_workload(mat_7, mat_10), mat_8) + + iterations = list(range(N)) + execution_time_taken = [] + dispatch_ids = [ct.dispatch(workflow)() for _ in iterations] + + for d_id in dispatch_ids: + result = ct.get_result(d_id, wait=True) + execution_time_taken.append((result.end_time - result.start_time).total_seconds()) + + for time_taken in execution_time_taken: + assert time_taken < 30 diff --git a/tests/functional_tests/qa_svm_workflow_test.py b/tests/functional_tests/qa_svm_workflow_test.py new file mode 100644 index 000000000..7bdc26f78 --- /dev/null +++ b/tests/functional_tests/qa_svm_workflow_test.py @@ -0,0 +1,64 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +"""SVM workflow test.""" + +import pytest +from numpy.random import permutation +from sklearn import datasets, svm + +import covalent as ct + + +def test_svm_workflow(): + """Test the SVM workflow.""" + + @ct.electron + def load_data(): + iris = datasets.load_iris() + perm = permutation(iris.target.size) + iris.data = iris.data[perm] + iris.target = iris.target[perm] + return iris.data, iris.target + + @ct.electron + def train_svm(data, C, gamma): + X, y = data + clf = svm.SVC(C=C, gamma=gamma) + clf.fit(X[90:], y[90:]) + return clf + + @ct.electron + def score_svm(data, clf): + X_test, y_test = data + return clf.score(X_test[:90], y_test[:90]) + + @ct.lattice + def run_experiment(C=1.0, gamma=0.7): + data = load_data() + clf = train_svm(data=data, C=C, gamma=gamma) + score = score_svm(data=data, clf=clf) + return score + + dispatchable_func = ct.dispatch(run_experiment) + + dispatch_id = dispatchable_func(C=1.0, gamma=0.7) + res = ct.get_result(dispatch_id, wait=True) + assert pytest.approx(res.result, 0.1) == 0.97 diff --git a/tests/functional_tests/qa_wait_for_file_transfer.py b/tests/functional_tests/qa_wait_for_file_transfer.py new file mode 100644 index 000000000..7d3854037 --- /dev/null +++ b/tests/functional_tests/qa_wait_for_file_transfer.py @@ -0,0 +1,65 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +"""Test wait for file transfer.""" + + +from pathlib import Path + +import covalent as ct + + +def test_wait_for_file_transfer(): + """Test wait for file transfer.""" + + dest_filepath = str(Path("/tmp/dest.csv").resolve()) + + @ct.electron( + files=[ + ct.fs.TransferFromRemote( + "https://raw.githubusercontent.com/curran/data/gh-pages/dataSoup/datasets.csv", + dest_filepath, + ), + ] + ) + def get_remote_file(files=[]): + pass + + @ct.electron() + def read_file(): + with open(dest_filepath, "r") as f: + return f.read() + + @ct.lattice + def workflow(): + get_remote_file_task = get_remote_file() + read_file_task = read_file() + ct.wait(read_file_task, [get_remote_file_task]) + + return read_file_task + + dispatch_id = ct.dispatch(workflow)() + res = ct.get_result(dispatch_id, wait=True) + result_string = res.result + + assert result_string[:12] == "Dataset Name" + assert result_string[-12:] == "05,,JSON,,,," + assert len(result_string) == 9248 + assert res.status == "COMPLETED" diff --git a/tests/requirements.txt b/tests/requirements.txt index c93e853e3..a7a0a1764 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -13,3 +13,4 @@ pytest-cov==3.0.0 pytest-mock==3.8.2 pytest-rerunfailures==10.2 scikit-image==0.19.1 +scikit-learn==1.2.2