Moved QA scripts to functional tests (#1583)

* Moved basic workflow to functional tests

* Add heterogenous workflows

* Add qa script

* Add CPU stress test script

* Finish adding QA scripts

* Update stress test baselines

* Update license checker syntax

.github/workflows/license.yml

@@ -68,8 +68,8 @@ jobs:
           fail: "Copyleft,Error,Other"
           exclude: "^(pylint|aio[-_]*).*"
           exclude-license: 'Mozilla Public License 2.0 \(MPL 2.0\)'
-          with-totals: true
-          table-headers: true
+          totals: true
+          headers: true
       - name: Print report
         if: ${{ always() }}
         run: echo "${{ steps.license_check_report.outputs.report }}"

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Tests
+
+- Move QA scripts from QA repo to Covalent functional tests.
+
 ### Docs
 
 - Update requirements file for the tutorials: `1_QuantumMachineLearning/pennylane_kernel/source.ipynb` and `machine_learning/dnn_comparison.ipynb`.

tests/functional_tests/qa_basic_workflow_test.py

@@ -0,0 +1,46 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the GNU Affero General Public License 3.0 (the "License").
+# A copy of the License may be obtained with this software package or at
+#
+#      https://www.gnu.org/licenses/agpl-3.0.en.html
+#
+# Use of this file is prohibited except in compliance with the License. Any
+# modifications or derivative works of this file must retain this copyright
+# notice, and modified files must contain a notice indicating that they have
+# been altered from the originals.
+#
+# Covalent is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
+#
+# Relief from the License may be granted by purchasing a commercial license.
+
+
+"""QA script to test basic workflow functionality."""
+
+import covalent as ct
+
+
+def test_basic_workflow():
+    """Test the basic workflow functionality."""
+
+    @ct.electron
+    def join_words(a, b):
+        return ", ".join([a, b])
+
+    @ct.electron
+    def excitement(a):
+        return f"{a}!"
+
+    @ct.lattice
+    def simple_workflow(a, b):
+        phrase = join_words(a, b)
+        return excitement(phrase)
+
+    dispatch_id = ct.dispatch(simple_workflow)("Hello", "World")
+    res = ct.get_result(dispatch_id, wait=True)
+    assert res.result == "Hello, World!"
+    assert res.status == "COMPLETED"

tests/functional_tests/qa_call_deps_retval_test.py

@@ -0,0 +1,66 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the GNU Affero General Public License 3.0 (the "License").
+# A copy of the License may be obtained with this software package or at
+#
+#      https://www.gnu.org/licenses/agpl-3.0.en.html
+#
+# Use of this file is prohibited except in compliance with the License. Any
+# modifications or derivative works of this file must retain this copyright
+# notice, and modified files must contain a notice indicating that they have
+# been altered from the originals.
+#
+# Covalent is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
+#
+# Relief from the License may be granted by purchasing a commercial license.
+
+"""QA script to test the call deps return value functionality."""
+
+
+import covalent as ct
+
+
+def test_call_deps_retval():
+    """Test the call deps return value functionality."""
+
+    def greeting_one():
+        return "hello"
+
+    def greeting_two():
+        return "howdy"
+
+    def greeting_three():
+        return "hey"
+
+    @ct.electron(
+        call_before=[
+            ct.DepsCall(greeting_one, retval_keyword="greetings"),
+            ct.DepsCall(greeting_two, retval_keyword="greetings"),
+        ]
+    )
+    def get_all_greetings(greetings=[]):
+        return greetings
+
+    @ct.electron(
+        call_before=[
+            ct.DepsCall(greeting_three, retval_keyword="greeting"),
+        ]
+    )
+    def get_one_greeting(greeting=None):
+        return f"{greeting}, whats up?"
+
+    @ct.lattice
+    def workflow():
+        greetings = get_all_greetings()
+        greeting = get_one_greeting()
+        return [greeting] + greetings
+
+    # Dispatch the workflow
+    dispatch_id = ct.dispatch(workflow)()
+    res = ct.get_result(dispatch_id, wait=True)
+    assert res.result == ["hey, whats up?", "hello", "howdy"]
+    assert res.status == "COMPLETED"

tests/functional_tests/qa_cpu_stress_test.py

@@ -0,0 +1,54 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the GNU Affero General Public License 3.0 (the "License").
+# A copy of the License may be obtained with this software package or at
+#
+#      https://www.gnu.org/licenses/agpl-3.0.en.html
+#
+# Use of this file is prohibited except in compliance with the License. Any
+# modifications or derivative works of this file must retain this copyright
+# notice, and modified files must contain a notice indicating that they have
+# been altered from the originals.
+#
+# Covalent is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
+#
+# Relief from the License may be granted by purchasing a commercial license.
+
+"""CPU stress test."""
+
+import covalent as ct
+
+N = 8
+STALL = 1e6
+X = 50
+
+
+def test_cpu_stress():
+    """Stress test the CPU."""
+
+    @ct.electron
+    def cpu_workload(stall, x):
+        i = 0
+        while i < stall:
+            x * x
+            i += 1
+
+    @ct.lattice
+    def workflow(iterations, stall, x):
+        for _ in range(iterations):
+            cpu_workload(stall, x)
+
+    n_electrons = [2**i for i in range(N)]
+    execution_time_taken = []
+    dispatch_ids = [ct.dispatch(workflow)(it, STALL, X) for it in n_electrons]
+
+    for d_id in dispatch_ids:
+        result = ct.get_result(d_id, wait=True)
+        execution_time_taken.append((result.end_time - result.start_time).total_seconds())
+
+    for time_taken in execution_time_taken:
+        assert time_taken < 60

tests/functional_tests/qa_deps_workflow_test.py

@@ -0,0 +1,59 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the GNU Affero General Public License 3.0 (the "License").
+# A copy of the License may be obtained with this software package or at
+#
+#      https://www.gnu.org/licenses/agpl-3.0.en.html
+#
+# Use of this file is prohibited except in compliance with the License. Any
+# modifications or derivative works of this file must retain this copyright
+# notice, and modified files must contain a notice indicating that they have
+# been altered from the originals.
+#
+# Covalent is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
+#
+# Relief from the License may be granted by purchasing a commercial license.
+
+"""Test the deps functionality."""
+
+from pathlib import Path
+
+import numpy
+
+import covalent as ct
+from covalent import DepsBash, DepsCall, DepsPip
+
+
+def test_deps_workflow():
+    """Test the deps functionality."""
+
+    deps_pip = DepsPip(packages=["numpy==1.23.1"])
+    deps_bash = DepsBash(commands=["echo $HOME >> /tmp/deps_bash_test.txt"])
+
+    def deps_call():
+        Path("/tmp/deps_bash_test.txt").unlink()
+
+    @ct.electron(
+        call_before=[deps_pip, deps_bash],
+        call_after=[DepsCall(deps_call)],
+    )
+    def get_deps_results():
+        results = []
+        with open("/tmp/deps_bash_test.txt", "r") as f:
+            results.append(f.read())
+        results.append(numpy.sum(numpy.identity(3)))
+        return results
+
+    @ct.lattice
+    def workflow():
+        return get_deps_results()
+
+    # Dispatch the workflow
+    dispatch_id = ct.dispatch(workflow)()
+    res = ct.get_result(dispatch_id, wait=True)
+    assert int(res.result[1]) == 3
+    assert res.status == "COMPLETED"

tests/functional_tests/qa_heterogenous_workflows_test.py

@@ -0,0 +1,89 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the GNU Affero General Public License 3.0 (the "License").
+# A copy of the License may be obtained with this software package or at
+#
+#      https://www.gnu.org/licenses/agpl-3.0.en.html
+#
+# Use of this file is prohibited except in compliance with the License. Any
+# modifications or derivative works of this file must retain this copyright
+# notice, and modified files must contain a notice indicating that they have
+# been altered from the originals.
+#
+# Covalent is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
+#
+# Relief from the License may be granted by purchasing a commercial license.
+
+"""Test different types of workflows."""
+
+import enum
+
+import pytest
+
+import covalent as ct
+
+
+class WORKFLOW_TYPES(str, enum.Enum):
+    HORIZONTAL = "HORIZONTAL"
+    VERTICAL = "VERTICAL"
+    HAGRID = "HAGRID"
+
+
+@pytest.mark.parametrize(
+    "workflow_type, parallel, serial, n",
+    [
+        (WORKFLOW_TYPES.HORIZONTAL, True, False, 8),
+        (WORKFLOW_TYPES.VERTICAL, False, True, 10),
+        (WORKFLOW_TYPES.HAGRID, True, True, 10),
+    ],
+)
+def test_heterogenous_workflows(workflow_type, parallel, serial, n):
+    """Test different types of workflows."""
+
+    @ct.electron
+    def identity(x):
+        return x
+
+    @ct.electron
+    def combine(x):
+        return sum(x)
+
+    @ct.lattice
+    def workflow(n, parallel=False, serial=False):
+        vals = []
+        result = 186282
+        nodes = range(n)
+
+        if parallel and not serial:
+            for _ in nodes:
+                vals.append(identity(1))
+            result = combine(vals)
+
+        elif serial and not parallel:
+            for _ in nodes:
+                result = identity(result)
+        elif serial and parallel:
+            for i in nodes:
+                for _ in nodes:
+                    if i == 0:
+                        vals.append(identity(1))
+                    else:
+                        vals.append(identity(result))
+                result = combine(vals)
+        return result
+
+    dispatch_id = ct.dispatch(workflow)(n, parallel, serial)
+    res = ct.get_result(dispatch_id, wait=True)
+
+    if workflow_type == WORKFLOW_TYPES.HORIZONTAL:
+        assert res.result == 8
+    elif workflow_type == WORKFLOW_TYPES.VERTICAL:
+        assert res.result == 186282
+    elif workflow_type == WORKFLOW_TYPES.HAGRID:
+        assert res.result == 23579476910
+
+    assert res.status == "COMPLETED"

tests/functional_tests/qa_http_file_transfer_test.py

@@ -0,0 +1,52 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the GNU Affero General Public License 3.0 (the "License").
+# A copy of the License may be obtained with this software package or at
+#
+#      https://www.gnu.org/licenses/agpl-3.0.en.html
+#
+# Use of this file is prohibited except in compliance with the License. Any
+# modifications or derivative works of this file must retain this copyright
+# notice, and modified files must contain a notice indicating that they have
+# been altered from the originals.
+#
+# Covalent is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
+#
+# Relief from the License may be granted by purchasing a commercial license.
+
+"""Test http file transfer."""
+
+import covalent as ct
+
+
+def test_http_file_transfer():
+    """Test http file transfer."""
+
+    @ct.electron(
+        files=[
+            ct.fs.TransferFromRemote(
+                "https://raw.githubusercontent.com/curran/data/gh-pages/dataSoup/datasets.csv"
+            ),
+        ]
+    )
+    def get_remote_file(files=[]):
+        source_path, dest_path = files[0]
+        with open(dest_path, "r") as f:
+            return f.read()
+
+    @ct.lattice
+    def workflow():
+        return get_remote_file()
+
+    dispatch_id = ct.dispatch(workflow)()
+    res = ct.get_result(dispatch_id, wait=True)
+    result_string = res.result
+
+    assert result_string[:12] == "Dataset Name"
+    assert result_string[-12:] == "05,,JSON,,,,"
+    assert len(result_string) == 9248
+    assert res.status == "COMPLETED"