Changed functional tests to use pytest, using .env config for executor (

#50)

* Changed functional tests to use pytest, using .env config for executor instance

* updated functional test script names to have suffix _test

* updated ecs executor import in functional test

* added .env.example

* updated functional test assertion

* added functional tests readme

* added AWS_PROFILE to .env.example

* added pytest-xdist as test requirement

* not running functional tests in test workflow

* Updated changelog

* moved ft test imports to inside test

* Updated ft README

.env.example

@@ -0,0 +1,10 @@
+AWS_PROFILE=default
+
+executor_ecs_cluster_name=
+executor_ecs_task_execution_role_name=
+executor_ecs_task_family_name=
+executor_ecs_task_log_group_name=
+executor_ecs_task_role_name=
+executor_ecs_task_security_group_id=
+executor_ecs_task_subnet_id=
+executor_s3_bucket_name=

.github/workflows/tests.yml

@@ -93,7 +93,7 @@ jobs:
         shell: python
 
       - name: Run tests
-        run: PYTHONPATH=$PWD/tests pytest -vv tests/ --cov=covalent_ecs_plugin
+        run: PYTHONPATH=$PWD/tests pytest -m "not functional_tests" -vv tests/ --cov=covalent_ecs_plugin
 
       - name: Generate coverage report
         run: coverage xml -o coverage.xml

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Changed
+
+- Functional tests using pytest and .env file configuration
+
 ## [0.22.0] - 2022-11-22
 
 ### Changed

pyproject.toml

@@ -37,3 +37,8 @@ multi_line_output = 3
 include_trailing_comma = true
 profile = 'black'
 skip_gitignore = true
+
+[tool.pytest.ini_options]
+markers = [
+    "functional_tests: marks tests that are to be run in the functional tests ci pipeline"
+]

tests/functional_tests/README.md

@@ -0,0 +1,25 @@
+## Functional Test Instructions
+
+### 1.Setup
+
+In the project root run the following:
+
+```sh
+pip install -r ./tests/requirements.txt
+pip install -r ./tests/functional_tests/requirements.txt
+export PYTHONPATH=$(pwd)
+```
+
+Copy create `.env` file:
+
+```sh
+cp .env.example .env
+```
+
+Fill in the configuration values either manually or from terraform output.
+
+### 2. Run Functional Tests
+
+```sh
+pytest -vvs -m functional_tests
+```

tests/functional_tests/basic_workflow.py → tests/functional_tests/basic_workflow_test.py

@@ -18,40 +18,34 @@
 #
 # Relief from the License may be granted by purchasing a commercial license.
 
-
-import sys
-
 import covalent as ct
+import pytest
 
-# Extract terraform outputs & instantiate executor
-import executor_instance
+from tests.functional_tests.fixtures.executor import executor
 
 # Basic Workflow
 
 
-@ct.electron(executor=executor_instance.executor)
-def join_words(a, b):
-    return ", ".join([a, b])
-
-
-@ct.electron
-def excitement(a):
-    return f"{a}!"
-
+@pytest.mark.functional_tests
+def test_basic_workflow():
+    @ct.electron(executor=executor)
+    def join_words(a, b):
+        return ", ".join([a, b])
 
-@ct.lattice
-def basic_workflow(a, b):
-    phrase = join_words(a, b)
-    return excitement(phrase)
+    @ct.electron
+    def excitement(a):
+        return f"{a}!"
 
+    @ct.lattice
+    def basic_workflow(a, b):
+        phrase = join_words(a, b)
+        return excitement(phrase)
 
-# Dispatch the workflow
-dispatch_id = ct.dispatch(basic_workflow)("Hello", "World")
-result = ct.get_result(dispatch_id=dispatch_id, wait=True)
-status = str(result.status)
+    # Dispatch the workflow
+    dispatch_id = ct.dispatch(basic_workflow)("Hello", "World")
+    result = ct.get_result(dispatch_id=dispatch_id, wait=True)
+    status = str(result.status)
 
-print(result)
+    print(result)
 
-if status == str(ct.status.FAILED):
-    print("Basic Workflow failed to run.")
-    sys.exit(1)
+    assert status == str(ct.status.COMPLETED)

tests/functional_tests/fixtures/executor.py

@@ -0,0 +1,27 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+
+from covalent_ecs_plugin.ecs import ECSExecutor
+
+executor_config = {
+    "s3_bucket_name": os.getenv("executor_s3_bucket_name"),
+    "ecr_repo_name": os.getenv("executor_ecr_repo_name"),
+    "ecs_cluster_name": os.getenv("executor_ecs_cluster_name"),
+    "ecs_task_family_name": os.getenv("executor_ecs_task_family_name"),
+    "ecs_task_execution_role_name": os.getenv("executor_ecs_task_execution_role_name"),
+    "ecs_task_role_name": os.getenv("executor_ecs_task_role_name"),
+    "ecs_task_log_group_name": os.getenv("executor_ecs_task_log_group_name"),
+    "ecs_task_subnet_id": os.getenv("executor_ecs_task_subnet_id"),
+    "ecs_task_security_group_id": os.getenv("executor_ecs_task_security_group_id"),
+    "vcpu": os.getenv("executor_vcpu", 0.25),
+    "memory": os.getenv("executor_memory", 0.5),
+    "cache_dir": "/tmp/covalent",
+}
+
+print("Using Executor Configuration:")
+print(executor_config)
+
+executor = ECSExecutor(**executor_config)

tests/functional_tests/requirements.txt

@@ -1,2 +1,3 @@
 numpy==1.23.2
+python-dotenv==0.21.0
 scikit-learn==1.1.2

tests/functional_tests/svm_workflow_test.py

@@ -0,0 +1,70 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the GNU Affero General Public License 3.0 (the "License").
+# A copy of the License may be obtained with this software package or at
+#
+#      https://www.gnu.org/licenses/agpl-3.0.en.html
+#
+# Use of this file is prohibited except in compliance with the License. Any
+# modifications or derivative works of this file must retain this copyright
+# notice, and modified files must contain a notice indicating that they have
+# been altered from the originals.
+#
+# Covalent is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
+#
+# Relief from the License may be granted by purchasing a commercial license.
+
+
+import covalent as ct
+import pytest
+
+from tests.functional_tests.fixtures.executor import executor
+
+deps_pip = ct.DepsPip(packages=["numpy==1.22.4", "scikit-learn==1.1.2"])
+
+
+@pytest.mark.functional_tests
+def test_svm_worklow():
+    from numpy.random import permutation
+    from sklearn import datasets, svm
+
+    @ct.electron
+    def load_data():
+        iris = datasets.load_iris()
+        perm = permutation(iris.target.size)
+        iris.data = iris.data[perm]
+        iris.target = iris.target[perm]
+        return iris.data, iris.target
+
+    @ct.electron(executor=executor, deps_pip=deps_pip)
+    def train_svm(data, C, gamma):
+        X, y = data
+        clf = svm.SVC(C=C, gamma=gamma)
+        clf.fit(X[90:], y[90:])
+        return clf
+
+    @ct.electron
+    def score_svm(data, clf):
+        X_test, y_test = data
+        return clf.score(X_test[:90], y_test[:90])
+
+    @ct.lattice
+    def run_experiment(C=1.0, gamma=0.7):
+        data = load_data()
+        clf = train_svm(data=data, C=C, gamma=gamma)
+        score = score_svm(data=data, clf=clf)
+        return score
+
+    dispatchable_func = ct.dispatch(run_experiment)
+
+    dispatch_id = dispatchable_func(C=1.0, gamma=0.7)
+    result = ct.get_result(dispatch_id=dispatch_id, wait=True)
+    status = str(result.status)
+
+    print(result)
+
+    assert status == str(ct.status.COMPLETED)