From 7024f09b6fb76067520df2edae6842b820f74a67 Mon Sep 17 00:00:00 2001 From: Alejandro Esquivel Date: Tue, 22 Nov 2022 13:41:33 -0800 Subject: [PATCH] Changed functional tests to use pytest, using .env config for executor (#50) * Changed functional tests to use pytest, using .env config for executor instance * updated functional test script names to have suffix _test * updated ecs executor import in functional test * added .env.example * updated functional test assertion * added functional tests readme * added AWS_PROFILE to .env.example * added pytest-xdist as test requirement * not running functional tests in test workflow * Updated changelog * moved ft test imports to inside test * Updated ft README --- .env.example | 10 +++ .github/workflows/tests.yml | 2 +- CHANGELOG.md | 4 + pyproject.toml | 5 ++ tests/functional_tests/README.md | 25 +++++++ ...sic_workflow.py => basic_workflow_test.py} | 46 +++++------- tests/functional_tests/executor_instance.py | 69 ------------------ tests/functional_tests/fixtures/__init__.py | 0 tests/functional_tests/fixtures/executor.py | 27 +++++++ tests/functional_tests/requirements.txt | 1 + tests/functional_tests/svm_workflow.py | 73 ------------------- tests/functional_tests/svm_workflow_test.py | 70 ++++++++++++++++++ tests/functional_tests/terraform_output.py | 50 ------------- tests/requirements.txt | 1 + 14 files changed, 164 insertions(+), 219 deletions(-) create mode 100644 .env.example create mode 100644 tests/functional_tests/README.md rename tests/functional_tests/{basic_workflow.py => basic_workflow_test.py} (57%) delete mode 100644 tests/functional_tests/executor_instance.py create mode 100644 tests/functional_tests/fixtures/__init__.py create mode 100644 tests/functional_tests/fixtures/executor.py delete mode 100644 tests/functional_tests/svm_workflow.py create mode 100644 tests/functional_tests/svm_workflow_test.py delete mode 100644 tests/functional_tests/terraform_output.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..861f5ff --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +AWS_PROFILE=default + +executor_ecs_cluster_name= +executor_ecs_task_execution_role_name= +executor_ecs_task_family_name= +executor_ecs_task_log_group_name= +executor_ecs_task_role_name= +executor_ecs_task_security_group_id= +executor_ecs_task_subnet_id= +executor_s3_bucket_name= diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0ea0cc2..0b30d9d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -93,7 +93,7 @@ jobs: shell: python - name: Run tests - run: PYTHONPATH=$PWD/tests pytest -vv tests/ --cov=covalent_ecs_plugin + run: PYTHONPATH=$PWD/tests pytest -m "not functional_tests" -vv tests/ --cov=covalent_ecs_plugin - name: Generate coverage report run: coverage xml -o coverage.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d2de48..1779a76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Changed + +- Functional tests using pytest and .env file configuration + ## [0.22.0] - 2022-11-22 ### Changed diff --git a/pyproject.toml b/pyproject.toml index 0c31a1e..a23f117 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,3 +37,8 @@ multi_line_output = 3 include_trailing_comma = true profile = 'black' skip_gitignore = true + +[tool.pytest.ini_options] +markers = [ + "functional_tests: marks tests that are to be run in the functional tests ci pipeline" +] diff --git a/tests/functional_tests/README.md b/tests/functional_tests/README.md new file mode 100644 index 0000000..05c15f7 --- /dev/null +++ b/tests/functional_tests/README.md @@ -0,0 +1,25 @@ +## Functional Test Instructions + +### 1.Setup + +In the project root run the following: + +```sh +pip install -r ./tests/requirements.txt +pip install -r ./tests/functional_tests/requirements.txt +export PYTHONPATH=$(pwd) +``` + +Copy create `.env` file: + +```sh +cp .env.example .env +``` + +Fill in the configuration values either manually or from terraform output. + +### 2. Run Functional Tests + +```sh +pytest -vvs -m functional_tests +``` diff --git a/tests/functional_tests/basic_workflow.py b/tests/functional_tests/basic_workflow_test.py similarity index 57% rename from tests/functional_tests/basic_workflow.py rename to tests/functional_tests/basic_workflow_test.py index fd14091..e5d262f 100644 --- a/tests/functional_tests/basic_workflow.py +++ b/tests/functional_tests/basic_workflow_test.py @@ -18,40 +18,34 @@ # # Relief from the License may be granted by purchasing a commercial license. - -import sys - import covalent as ct +import pytest -# Extract terraform outputs & instantiate executor -import executor_instance +from tests.functional_tests.fixtures.executor import executor # Basic Workflow -@ct.electron(executor=executor_instance.executor) -def join_words(a, b): - return ", ".join([a, b]) - - -@ct.electron -def excitement(a): - return f"{a}!" - +@pytest.mark.functional_tests +def test_basic_workflow(): + @ct.electron(executor=executor) + def join_words(a, b): + return ", ".join([a, b]) -@ct.lattice -def basic_workflow(a, b): - phrase = join_words(a, b) - return excitement(phrase) + @ct.electron + def excitement(a): + return f"{a}!" + @ct.lattice + def basic_workflow(a, b): + phrase = join_words(a, b) + return excitement(phrase) -# Dispatch the workflow -dispatch_id = ct.dispatch(basic_workflow)("Hello", "World") -result = ct.get_result(dispatch_id=dispatch_id, wait=True) -status = str(result.status) + # Dispatch the workflow + dispatch_id = ct.dispatch(basic_workflow)("Hello", "World") + result = ct.get_result(dispatch_id=dispatch_id, wait=True) + status = str(result.status) -print(result) + print(result) -if status == str(ct.status.FAILED): - print("Basic Workflow failed to run.") - sys.exit(1) + assert status == str(ct.status.COMPLETED) diff --git a/tests/functional_tests/executor_instance.py b/tests/functional_tests/executor_instance.py deleted file mode 100644 index 27a3b20..0000000 --- a/tests/functional_tests/executor_instance.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2021 Agnostiq Inc. -# -# This file is part of Covalent. -# -# Licensed under the GNU Affero General Public License 3.0 (the "License"). -# A copy of the License may be obtained with this software package or at -# -# https://www.gnu.org/licenses/agpl-3.0.en.html -# -# Use of this file is prohibited except in compliance with the License. Any -# modifications or derivative works of this file must retain this copyright -# notice, and modified files must contain a notice indicating that they have -# been altered from the originals. -# -# Covalent is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. -# -# Relief from the License may be granted by purchasing a commercial license. - - -import os - -import covalent as ct -import terraform_output - -ECS_EXECUTOR_S3_BUCKET = terraform_output.get("s3_bucket_name", "covalent-fargate-task-resources") -ECS_EXECUTOR_ECR_REPO = terraform_output.get("ecr_repo_name", "covalent-fargate-task-images") -ECS_EXECUTOR_ECS_CLUSTER = terraform_output.get("ecs_cluster_name", "covalent-fargate-cluster") -ECS_EXECUTOR_TASK_FAMILY_NAME = terraform_output.get( - "ecs_task_family_name", "covalent-fargate-tasks" -) -ECS_EXECUTOR_EXECUTION_ROLE_NAME = terraform_output.get( - "ecs_task_execution_role_name", "ecsTaskExecutionRole" -) -ECS_EXECUTOR_TASK_ROLE_NAME = terraform_output.get("ecs_task_role_name", "CovalentFargateTaskRole") -ECS_EXECUTOR_TASK_LOG_GROUP_NAME = terraform_output.get( - "ecs_task_log_group_name", "covalent-fargate-task-logs" -) -ECS_EXECUTOR_TASK_SUBNET_ID = terraform_output.get("ecs_task_subnet_id", "") -ECS_EXECUTOR_TASK_SECURITY_GROUP_NAME = terraform_output.get("ecs_task_security_group_id", "") - -executor_config = { - "s3_bucket_name": os.getenv("ECS_EXECUTOR_S3_BUCKET", ECS_EXECUTOR_S3_BUCKET), - "ecr_repo_name": os.getenv("ECS_EXECUTOR_ECR_REPO", ECS_EXECUTOR_ECR_REPO), - "ecs_cluster_name": os.getenv("ECS_EXECUTOR_ECS_CLUSTER", ECS_EXECUTOR_ECS_CLUSTER), - "ecs_task_family_name": os.getenv( - "ECS_EXECUTOR_TASK_FAMILY_NAME", ECS_EXECUTOR_TASK_FAMILY_NAME - ), - "ecs_task_execution_role_name": os.getenv( - "ECS_EXECUTOR_EXECUTION_ROLE_NAME", ECS_EXECUTOR_EXECUTION_ROLE_NAME - ), - "ecs_task_role_name": os.getenv("ECS_EXECUTOR_TASK_ROLE_NAME", ECS_EXECUTOR_TASK_ROLE_NAME), - "ecs_task_log_group_name": os.getenv( - "ECS_EXECUTOR_TASK_LOG_GROUP_NAME", ECS_EXECUTOR_TASK_LOG_GROUP_NAME - ), - "ecs_task_subnet_id": os.getenv("ECS_EXECUTOR_TASK_SUBNET_ID", ECS_EXECUTOR_TASK_SUBNET_ID), - "ecs_task_security_group_id": os.getenv( - "ECS_EXECUTOR_TASK_SECURITY_GROUP_NAME", ECS_EXECUTOR_TASK_SECURITY_GROUP_NAME - ), - "vcpu": os.getenv("ECS_EXECUTOR_VCPU", 0.25), - "memory": os.getenv("ECS_EXECUTOR_MEMORY", 0.5), - "cache_dir": "/tmp/covalent", -} - -print("Using Executor Config:") -print(executor_config) - -executor = ct.executor.ECSExecutor(**executor_config) diff --git a/tests/functional_tests/fixtures/__init__.py b/tests/functional_tests/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional_tests/fixtures/executor.py b/tests/functional_tests/fixtures/executor.py new file mode 100644 index 0000000..884a7c7 --- /dev/null +++ b/tests/functional_tests/fixtures/executor.py @@ -0,0 +1,27 @@ +from dotenv import load_dotenv + +load_dotenv() + +import os + +from covalent_ecs_plugin.ecs import ECSExecutor + +executor_config = { + "s3_bucket_name": os.getenv("executor_s3_bucket_name"), + "ecr_repo_name": os.getenv("executor_ecr_repo_name"), + "ecs_cluster_name": os.getenv("executor_ecs_cluster_name"), + "ecs_task_family_name": os.getenv("executor_ecs_task_family_name"), + "ecs_task_execution_role_name": os.getenv("executor_ecs_task_execution_role_name"), + "ecs_task_role_name": os.getenv("executor_ecs_task_role_name"), + "ecs_task_log_group_name": os.getenv("executor_ecs_task_log_group_name"), + "ecs_task_subnet_id": os.getenv("executor_ecs_task_subnet_id"), + "ecs_task_security_group_id": os.getenv("executor_ecs_task_security_group_id"), + "vcpu": os.getenv("executor_vcpu", 0.25), + "memory": os.getenv("executor_memory", 0.5), + "cache_dir": "/tmp/covalent", +} + +print("Using Executor Configuration:") +print(executor_config) + +executor = ECSExecutor(**executor_config) diff --git a/tests/functional_tests/requirements.txt b/tests/functional_tests/requirements.txt index 66a7502..ccb9b2b 100644 --- a/tests/functional_tests/requirements.txt +++ b/tests/functional_tests/requirements.txt @@ -1,2 +1,3 @@ numpy==1.23.2 +python-dotenv==0.21.0 scikit-learn==1.1.2 diff --git a/tests/functional_tests/svm_workflow.py b/tests/functional_tests/svm_workflow.py deleted file mode 100644 index 8f78c17..0000000 --- a/tests/functional_tests/svm_workflow.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2021 Agnostiq Inc. -# -# This file is part of Covalent. -# -# Licensed under the GNU Affero General Public License 3.0 (the "License"). -# A copy of the License may be obtained with this software package or at -# -# https://www.gnu.org/licenses/agpl-3.0.en.html -# -# Use of this file is prohibited except in compliance with the License. Any -# modifications or derivative works of this file must retain this copyright -# notice, and modified files must contain a notice indicating that they have -# been altered from the originals. -# -# Covalent is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. -# -# Relief from the License may be granted by purchasing a commercial license. - - -import sys - -import covalent as ct -import executor_instance -from numpy.random import permutation -from sklearn import datasets, svm - -deps_pip = ct.DepsPip(packages=["numpy==1.22.4", "scikit-learn==1.1.2"]) - - -@ct.electron -def load_data(): - iris = datasets.load_iris() - perm = permutation(iris.target.size) - iris.data = iris.data[perm] - iris.target = iris.target[perm] - return iris.data, iris.target - - -@ct.electron(executor=executor_instance.executor, deps_pip=deps_pip) -def train_svm(data, C, gamma): - X, y = data - clf = svm.SVC(C=C, gamma=gamma) - clf.fit(X[90:], y[90:]) - return clf - - -@ct.electron -def score_svm(data, clf): - X_test, y_test = data - return clf.score(X_test[:90], y_test[:90]) - - -@ct.lattice -def run_experiment(C=1.0, gamma=0.7): - data = load_data() - clf = train_svm(data=data, C=C, gamma=gamma) - score = score_svm(data=data, clf=clf) - return score - - -dispatchable_func = ct.dispatch(run_experiment) - -dispatch_id = dispatchable_func(C=1.0, gamma=0.7) -result = ct.get_result(dispatch_id=dispatch_id, wait=True) -status = str(result.status) - -print(result) - -if status == str(ct.status.FAILED): - print("Basic Workflow failed to run.") - sys.exit(1) diff --git a/tests/functional_tests/svm_workflow_test.py b/tests/functional_tests/svm_workflow_test.py new file mode 100644 index 0000000..9588534 --- /dev/null +++ b/tests/functional_tests/svm_workflow_test.py @@ -0,0 +1,70 @@ +# Copyright 2021 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + + +import covalent as ct +import pytest + +from tests.functional_tests.fixtures.executor import executor + +deps_pip = ct.DepsPip(packages=["numpy==1.22.4", "scikit-learn==1.1.2"]) + + +@pytest.mark.functional_tests +def test_svm_worklow(): + from numpy.random import permutation + from sklearn import datasets, svm + + @ct.electron + def load_data(): + iris = datasets.load_iris() + perm = permutation(iris.target.size) + iris.data = iris.data[perm] + iris.target = iris.target[perm] + return iris.data, iris.target + + @ct.electron(executor=executor, deps_pip=deps_pip) + def train_svm(data, C, gamma): + X, y = data + clf = svm.SVC(C=C, gamma=gamma) + clf.fit(X[90:], y[90:]) + return clf + + @ct.electron + def score_svm(data, clf): + X_test, y_test = data + return clf.score(X_test[:90], y_test[:90]) + + @ct.lattice + def run_experiment(C=1.0, gamma=0.7): + data = load_data() + clf = train_svm(data=data, C=C, gamma=gamma) + score = score_svm(data=data, clf=clf) + return score + + dispatchable_func = ct.dispatch(run_experiment) + + dispatch_id = dispatchable_func(C=1.0, gamma=0.7) + result = ct.get_result(dispatch_id=dispatch_id, wait=True) + status = str(result.status) + + print(result) + + assert status == str(ct.status.COMPLETED) diff --git a/tests/functional_tests/terraform_output.py b/tests/functional_tests/terraform_output.py deleted file mode 100644 index d2cdca1..0000000 --- a/tests/functional_tests/terraform_output.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2021 Agnostiq Inc. -# -# This file is part of Covalent. -# -# Licensed under the GNU Affero General Public License 3.0 (the "License"). -# A copy of the License may be obtained with this software package or at -# -# https://www.gnu.org/licenses/agpl-3.0.en.html -# -# Use of this file is prohibited except in compliance with the License. Any -# modifications or derivative works of this file must retain this copyright -# notice, and modified files must contain a notice indicating that they have -# been altered from the originals. -# -# Covalent is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. -# -# Relief from the License may be granted by purchasing a commercial license. - - -import json -import os -import subprocess -import sys - -TERRAFORM_OUTPUTS = {} - -try: - terraform_dir = os.getenv("TF_DIR") - proc = subprocess.run( - [ - "terraform", - f"-chdir={terraform_dir}", - "output", - "-json", - ], - check=True, - capture_output=True, - ) - TERRAFORM_OUTPUTS = json.loads(proc.stdout.decode()) -except Exception as e: - pass - - -def get(key: str, default): - try: - return TERRAFORM_OUTPUTS[key]["value"] - except KeyError: - return default diff --git a/tests/requirements.txt b/tests/requirements.txt index 8f99aa6..78a7b3b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,3 +2,4 @@ pytest==6.2.5 pytest-asyncio==0.19.0 pytest-cov==2.12.0 pytest-mock==3.6.1 +pytest-xdist