diff --git a/ci-isolation/src/main/resources/s3-access.json b/ci-isolation/src/main/resources/s3-access.json index cf74a53e..3342664a 100644 --- a/ci-isolation/src/main/resources/s3-access.json +++ b/ci-isolation/src/main/resources/s3-access.json @@ -13,6 +13,7 @@ "Action": [ "iam:CreatePolicy", "iam:CreateRole", + "iam:GetRole", "iam:AttachRolePolicy", "iam:DeleteRole", "iam:DeletePolicy", @@ -21,4 +22,4 @@ "Resource": "*" } ] -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 3ca5ad59..5bc53625 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ botocore = "1.29.163" protobuf = ">=3.1,<=3.20.0" sagemaker = "^2.59.1" pyexasol = "^0.24.0" -localstack-client = "^1.25" importlib-resources = "^5.2.0" click = "^8.0.3" typeguard = "^2.11.1" @@ -40,6 +39,7 @@ coverage = "^6.3" exasol-udf-mock-python = { git = "https://github.com/exasol/udf-mock-python.git", branch = "main" } exasol-bucketfs = "^0.6.0" poethepoet = "^0.13.1" +localstack-client = "^1.25" boto3 = "^1.20.40" [build-system] diff --git a/scripts/setup_integration_test.sh b/scripts/setup_integration_test.sh index 08191df5..1ea2a2c6 100755 --- a/scripts/setup_integration_test.sh +++ b/scripts/setup_integration_test.sh @@ -13,8 +13,8 @@ cnt_func=0 function install_localstack { cnt_func=$((cnt_func+1)) echo -e "${YEL} Step-$cnt_func: ${GRA} Install Localstack packages${NC}" - pip install localstack=='0.12.18' - pip install localstack-client=="1.25" + pip install localstack + pip install localstack-client } function checkout_exasol_test_container { diff --git a/tests/ci_tests/fixtures/build_language_container_fixture.py b/tests/ci_tests/fixtures/build_language_container_fixture.py index dcf50786..a0f19f89 100644 --- a/tests/ci_tests/fixtures/build_language_container_fixture.py +++ b/tests/ci_tests/fixtures/build_language_container_fixture.py @@ -30,9 +30,9 @@ def language_container(): completed_process = subprocess.run( [script_dir], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = completed_process.stdout.decode("UTF-8") + print(output) completed_process.check_returncode() - print(output) lines = output.splitlines() alter_session_selector = "ALTER SYSTEM SET SCRIPT_LANGUAGES='" diff --git a/tests/ci_tests/fixtures/prepare_environment_fixture.py b/tests/ci_tests/fixtures/prepare_environment_fixture.py index 5cc7d4b5..4930e6d1 100644 --- a/tests/ci_tests/fixtures/prepare_environment_fixture.py +++ b/tests/ci_tests/fixtures/prepare_environment_fixture.py @@ -1,10 +1,14 @@ +import dataclasses import os +from inspect import cleandoc + import boto3 +import pyexasol import pytest from click.testing import CliRunner from exasol_sagemaker_extension.deployment import deploy_cli -from tests.ci_tests.utils.parameters import db_params, aws_params, \ +from tests.ci_tests.utils.parameters import db_params, \ reg_model_setup_params, cls_model_setup_params @@ -50,40 +54,153 @@ def _setup_database(db_conn): __insert_into_tables(db_conn, model_setup) -def _create_aws_connection(conn): +@pytest.fixture(scope="session") +def connection_object_for_aws_credentials(db_conn, aws_s3_bucket): + aws_conn_name = "test_aws_credentials_connection_name" + aws_region = os.environ["AWS_DEFAULT_REGION"] + aws_s3_uri = f"https://{aws_s3_bucket}.s3.{aws_region}.amazonaws.com" query = "CREATE OR REPLACE CONNECTION {aws_conn_name} " \ "TO '{aws_s3_uri}' " \ - "USER '{aws_key_id}' IDENTIFIED BY '{aws_access_key}'"\ - .format(aws_conn_name=aws_params.aws_conn_name, - aws_s3_uri=aws_params.aws_s3_uri, - aws_key_id=os.environ["AWS_ACCESS_KEY_ID"], - aws_access_key=os.environ["AWS_SECRET_ACCESS_KEY"]) - conn.execute(query) + "USER '{aws_access_key_id}' IDENTIFIED BY '{aws_secret_access_key}'" \ + .format(aws_conn_name=aws_conn_name, + aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + aws_s3_uri=aws_s3_uri) + db_conn.execute(query) + print(query) + yield aws_conn_name + db_conn.execute(f"DROP CONNECTION {aws_conn_name};") def _create_aws_s3_bucket(): s3_client = boto3.client('s3') + bucket_name = "ci-exasol-sagemaker-extension-bucket" try: s3_client.create_bucket( - Bucket=aws_params.aws_bucket, + Bucket=bucket_name, CreateBucketConfiguration={ 'LocationConstraint': os.environ["AWS_DEFAULT_REGION"]} ) except s3_client.exceptions.BucketAlreadyOwnedByYou as ex: print("'BucketAlreadyOwnedByYou' exception is handled") + return bucket_name -def _remove_aws_s3_bucket(): +def _remove_aws_s3_bucket_content(bucket_name: str): s3_client = boto3.resource('s3') - - bucket = s3_client.Bucket(aws_params.aws_bucket) + bucket = s3_client.Bucket(bucket_name) bucket.objects.all().delete() @pytest.fixture(scope="session") -def prepare_ci_test_environment(db_conn): +def aws_s3_bucket(): + bucket_name = _create_aws_s3_bucket() + yield bucket_name + _remove_aws_s3_bucket_content(bucket_name) + + +@pytest.fixture(scope="session") +def aws_sagemaker_role() -> str: + iam_client = boto3.client('iam') + role_name = _create_sagemaker_role(iam_client) + policy_arn = _create_sagemaker_policy(iam_client) + _attach_policy_to_role(iam_client, + policy_arn=policy_arn, + role_name=role_name) + _attach_policy_to_role(iam_client, + policy_arn="arn:aws:iam::aws:policy/AmazonSageMakerFullAccess", + role_name=role_name) + return role_name + + +def _attach_policy_to_role(iam_client, policy_arn, role_name): + response = iam_client.attach_role_policy( + PolicyArn=policy_arn, + RoleName=role_name, + ) + + +def _create_sagemaker_role(iam_client): + role_name = "ci-exasol-sagemaker-extension-role" + try: + assume_policy_document = cleandoc(""" + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "sagemaker.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + } + """) + response = iam_client.create_role( + RoleName=role_name, + AssumeRolePolicyDocument=assume_policy_document, + Description='This role is used for the CI Tests of the exasol.sagemaker-extension', + ) + except iam_client.exceptions.EntityAlreadyExistsException as ex: + print("'EntityAlreadyExistsException' exception is handled") + return role_name + + +def _create_sagemaker_policy(iam_client) -> str: + policy_name = "ci-exasol-sagemaker-extension-policy" + try: + policy_document = cleandoc(""" + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": "*" + } + ] + } + """) + response = iam_client.create_policy( + PolicyName=policy_name, + PolicyDocument=policy_document, + Description='This policy is used for the CI Tests of the exasol.sagemaker-extension', + + ) + print(response) + return response["Policy"]["ARN"] # FIXME got key error in this line + except iam_client.exceptions.EntityAlreadyExistsException as ex: + print("'EntityAlreadyExistsException' exception is handled") + sts_client = boto3.client('sts') + account_id = sts_client.get_caller_identity()['Account'] + policy_arn = f'arn:aws:iam::{account_id}:policy/{policy_name}' + return policy_arn + + +@dataclasses.dataclass +class CITestEnvironment: + db_conn: pyexasol.ExaConnection + aws_s3_bucket: str + aws_sagemaker_role: str + connection_object_for_aws_credentials: str + aws_region: str = os.environ["AWS_DEFAULT_REGION"] + + @property + def aws_bucket_uri(self) -> str: + aws_bucket_uri = f"s3://{self.aws_s3_bucket}" + return aws_bucket_uri + + +@pytest.fixture(scope="session") +def prepare_ci_test_environment(db_conn, + aws_s3_bucket, + connection_object_for_aws_credentials, + aws_sagemaker_role) -> CITestEnvironment: _setup_database(db_conn) - _create_aws_connection(db_conn) - _create_aws_s3_bucket() - yield db_conn - _remove_aws_s3_bucket() + yield CITestEnvironment(db_conn=db_conn, + aws_s3_bucket=aws_s3_bucket, + connection_object_for_aws_credentials=connection_object_for_aws_credentials, + aws_sagemaker_role=aws_sagemaker_role) diff --git a/tests/ci_tests/fixtures/setup_ci_test_environment.py b/tests/ci_tests/fixtures/setup_ci_test_environment.py index a787bd4e..14d60b4a 100644 --- a/tests/ci_tests/fixtures/setup_ci_test_environment.py +++ b/tests/ci_tests/fixtures/setup_ci_test_environment.py @@ -1,8 +1,9 @@ import pytest +from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment + @pytest.fixture(scope="session") def setup_ci_test_environment(register_language_container, - prepare_ci_test_environment): - db_conn = prepare_ci_test_environment - return db_conn + prepare_ci_test_environment) -> CITestEnvironment: + return prepare_ci_test_environment diff --git a/tests/ci_tests/test_deploying_autopilot.py b/tests/ci_tests/test_deploying_autopilot.py index 8a5ce2f5..fdf9aa00 100644 --- a/tests/ci_tests/test_deploying_autopilot.py +++ b/tests/ci_tests/test_deploying_autopilot.py @@ -1,14 +1,16 @@ import time -import pytest from datetime import datetime + +import pytest + +from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment +from tests.ci_tests.utils import parameters from tests.ci_tests.utils.autopilot_deployment import AutopilotTestDeployment from tests.ci_tests.utils.autopilot_polling import AutopilotTestPolling from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining from tests.ci_tests.utils.cleanup import cleanup -from tests.ci_tests.utils.queries import DatabaseQueries -from tests.ci_tests.utils.checkers import is_aws_credentials_not_set from tests.ci_tests.utils.parameters import cls_model_setup_params -from tests.ci_tests.utils import parameters +from tests.ci_tests.utils.queries import DatabaseQueries def _is_training_completed(status): @@ -19,14 +21,14 @@ def _is_training_completed(status): @cleanup -def _deploy_endpoint(job_name, endpoint_name, model_setup_params, db_conn): +def _deploy_endpoint(job_name, endpoint_name, model_setup_params, ci_test_env: CITestEnvironment, ): # poll until the training is completed timeout_time = time.time() + parameters.TIMEOUT while True: status = AutopilotTestPolling.poll_autopilot_job( job_name, model_setup_params.schema_name, - db_conn) + ci_test_env) print(status) if _is_training_completed(status): @@ -42,12 +44,12 @@ def _deploy_endpoint(job_name, endpoint_name, model_setup_params, db_conn): job_name, endpoint_name, model_setup_params, - db_conn + ci_test_env ) # assertion all_scripts = DatabaseQueries.get_all_scripts( - model_setup_params, db_conn) + model_setup_params, ci_test_env.db_conn) assert endpoint_name in list(map(lambda x: x[0], all_scripts)) diff --git a/tests/ci_tests/test_polling_autopilot.py b/tests/ci_tests/test_polling_autopilot.py index 8def87af..ccac8e85 100644 --- a/tests/ci_tests/test_polling_autopilot.py +++ b/tests/ci_tests/test_polling_autopilot.py @@ -1,8 +1,9 @@ -import pytest from datetime import datetime + +import pytest + from tests.ci_tests.utils.autopilot_polling import AutopilotTestPolling from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining -from tests.ci_tests.utils.checkers import is_aws_credentials_not_set from tests.ci_tests.utils.parameters import cls_model_setup_params diff --git a/tests/ci_tests/test_predicting_autopilot.py b/tests/ci_tests/test_predicting_autopilot.py index b00f5317..7093c801 100644 --- a/tests/ci_tests/test_predicting_autopilot.py +++ b/tests/ci_tests/test_predicting_autopilot.py @@ -1,12 +1,14 @@ import time -import pytest from datetime import datetime + +import pytest + +from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment from tests.ci_tests.utils import parameters from tests.ci_tests.utils.autopilot_deployment import AutopilotTestDeployment from tests.ci_tests.utils.autopilot_polling import AutopilotTestPolling from tests.ci_tests.utils.autopilot_prediction import AutopilotTestPrediction from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining -from tests.ci_tests.utils.checkers import is_aws_credentials_not_set from tests.ci_tests.utils.cleanup import cleanup from tests.ci_tests.utils.parameters import cls_model_setup_params, \ reg_model_setup_params @@ -20,14 +22,14 @@ def _is_training_completed(status): @cleanup -def _make_prediction(job_name, endpoint_name, model_setup_params, db_conn): +def _make_prediction(job_name, endpoint_name, model_setup_params, ci_test_env: CITestEnvironment): # poll until the training is completed timeout_time = time.time() + parameters.TIMEOUT while True: status = AutopilotTestPolling.poll_autopilot_job( job_name, model_setup_params.schema_name, - db_conn) + ci_test_env) print(status) if _is_training_completed(status): @@ -43,12 +45,12 @@ def _make_prediction(job_name, endpoint_name, model_setup_params, db_conn): job_name, endpoint_name, model_setup_params, - db_conn + ci_test_env ) # assertion predictions = AutopilotTestPrediction.predict( - endpoint_name, model_setup_params.schema_name, db_conn) + endpoint_name, model_setup_params.schema_name, ci_test_env) assert predictions diff --git a/tests/ci_tests/test_training_autopilot.py b/tests/ci_tests/test_training_autopilot.py index 5c8b31a3..324f546c 100644 --- a/tests/ci_tests/test_training_autopilot.py +++ b/tests/ci_tests/test_training_autopilot.py @@ -1,13 +1,16 @@ -import pytest from datetime import datetime + +import pytest + +from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining -from tests.ci_tests.utils.queries import DatabaseQueries from tests.ci_tests.utils.checkers import is_aws_credentials_not_set from tests.ci_tests.utils.parameters import reg_model_setup_params, \ cls_model_setup_params +from tests.ci_tests.utils.queries import DatabaseQueries -@pytest.mark.skipif("is_aws_credentials_not_set() == True", +@pytest.mark.skipif(is_aws_credentials_not_set() == True, reason="AWS credentials are not set") def test_train_autopilot_regression_job(setup_ci_test_environment): curr_datetime = datetime.now().strftime("%y%m%d%H%M%S") @@ -22,7 +25,7 @@ def test_train_autopilot_regression_job(setup_ci_test_environment): job_name, reg_model_setup_params, setup_ci_test_environment) -@pytest.mark.skipif("is_aws_credentials_not_set() == True", +@pytest.mark.skipif(is_aws_credentials_not_set() == True, reason="AWS credentials are not set") def test_train_autopilot_classification_job(setup_ci_test_environment): curr_datetime = datetime.now().strftime("%y%m%d%H%M%S") @@ -38,7 +41,7 @@ def test_train_autopilot_classification_job(setup_ci_test_environment): job_name, cls_model_setup_params, setup_ci_test_environment) -def _assert_training_job(job_name, model_setup_params, db_conn): +def _assert_training_job(job_name, model_setup_params, ci_test_env: CITestEnvironment): all_jobs = DatabaseQueries.get_all_jobs( - model_setup_params, db_conn) + model_setup_params, ci_test_env.db_conn) assert job_name in list(map(lambda x: x[0], all_jobs)) diff --git a/tests/ci_tests/utils/autopilot_deletion.py b/tests/ci_tests/utils/autopilot_deletion.py index fe6f039d..9bf44a5a 100644 --- a/tests/ci_tests/utils/autopilot_deletion.py +++ b/tests/ci_tests/utils/autopilot_deletion.py @@ -1,17 +1,15 @@ import os from sagemaker import Predictor -from tests.ci_tests.utils.parameters import aws_params - class AutopilotTestDeletion: @staticmethod - def delete_endpoint_via_database(endpoint_name, setup_params, db_conn): + def delete_endpoint_via_database(endpoint_name, setup_params, db_conn, aws_conn_name): query_deletion = "EXECUTE SCRIPT " \ "{schema}.SME_DELETE_SAGEMAKER_AUTOPILOT_ENDPOINT(" \ "'{endpoint_name}', '{aws_conn_name}', '{aws_region}')". \ format(schema=setup_params.schema_name, endpoint_name=endpoint_name, - aws_conn_name=aws_params.aws_conn_name, + aws_conn_name=aws_conn_name, aws_region=os.environ["AWS_REGION"]) db_conn.execute(query_deletion) diff --git a/tests/ci_tests/utils/autopilot_deployment.py b/tests/ci_tests/utils/autopilot_deployment.py index b977fbd1..ea046024 100644 --- a/tests/ci_tests/utils/autopilot_deployment.py +++ b/tests/ci_tests/utils/autopilot_deployment.py @@ -1,6 +1,6 @@ import os -from tests.ci_tests.utils.parameters import aws_params +from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment INSTANCE_TYPE = "ml.c5.large" INSTANCE_COUNT = 1 @@ -8,7 +8,7 @@ class AutopilotTestDeployment: @staticmethod - def deploy_endpoint(job_name, endpoint_name, setup_params, db_conn): + def deploy_endpoint(job_name, endpoint_name, setup_params, ci_test_env: CITestEnvironment): query_deployment = "EXECUTE SCRIPT " \ "{schema}.SME_DEPLOY_SAGEMAKER_AUTOPILOT_ENDPOINT(" \ "'{job_name}', '{endpoint_name}', '{schema}', " \ @@ -19,7 +19,7 @@ def deploy_endpoint(job_name, endpoint_name, setup_params, db_conn): endpoint_name=endpoint_name, instance_type=INSTANCE_TYPE, instance_count=INSTANCE_COUNT, - aws_conn_name=aws_params.aws_conn_name, + aws_conn_name=ci_test_env.connection_object_for_aws_credentials, aws_region=os.environ["AWS_DEFAULT_REGION"]) - db_conn.execute(query_deployment) + ci_test_env.db_conn.execute(query_deployment) diff --git a/tests/ci_tests/utils/autopilot_polling.py b/tests/ci_tests/utils/autopilot_polling.py index 80fcfc64..5c0be684 100644 --- a/tests/ci_tests/utils/autopilot_polling.py +++ b/tests/ci_tests/utils/autopilot_polling.py @@ -1,18 +1,18 @@ import os -from tests.ci_tests.utils.parameters import aws_params +from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment class AutopilotTestPolling: @staticmethod - def poll_autopilot_job(job_name, schema_name, db_conn) -> str: + def poll_autopilot_job(job_name, schema_name, ci_test_env: CITestEnvironment) -> str: query_polling = "EXECUTE SCRIPT " \ "{schema}.SME_POLL_SAGEMAKER_AUTOPILOT_JOB_STATUS(" \ "'{job_name}', '{aws_conn_name}', '{aws_region}')". \ format(schema=schema_name, job_name=job_name, - aws_conn_name=aws_params.aws_conn_name, + aws_conn_name=ci_test_env.connection_object_for_aws_credentials, aws_region=os.environ["AWS_DEFAULT_REGION"]) - status = db_conn.execute(query_polling).fetchall() + status = ci_test_env.db_conn.execute(query_polling).fetchall() return status diff --git a/tests/ci_tests/utils/autopilot_prediction.py b/tests/ci_tests/utils/autopilot_prediction.py index dd031856..157a8f75 100644 --- a/tests/ci_tests/utils/autopilot_prediction.py +++ b/tests/ci_tests/utils/autopilot_prediction.py @@ -1,14 +1,14 @@ +from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment + class AutopilotTestPrediction: @staticmethod - def predict(endpoint_name, schema_name, db_conn) -> list: + def predict(endpoint_name, schema_name, ci_test_env: CITestEnvironment) -> list: prediction_udf_name = endpoint_name query = """SELECT "{schema}"."{udf_name}"(3,4)""" query_prediction = query.format( schema=schema_name.upper(), udf_name=prediction_udf_name) - prediction = db_conn.execute(query_prediction).fetchall() - - print(prediction) + prediction = ci_test_env.db_conn.execute(query_prediction).fetchall() return prediction diff --git a/tests/ci_tests/utils/autopilot_training.py b/tests/ci_tests/utils/autopilot_training.py index 442c5017..9b29dfd4 100644 --- a/tests/ci_tests/utils/autopilot_training.py +++ b/tests/ci_tests/utils/autopilot_training.py @@ -1,7 +1,8 @@ -import os import json + +from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment from tests.ci_tests.utils.parameters import reg_model_setup_params, \ - cls_model_setup_params, aws_params + cls_model_setup_params class AutopilotTestTraining: @@ -10,40 +11,44 @@ def __init__(self, job_name, db_conn): self._db_conn = db_conn @classmethod - def train_autopilot_regression_job(cls, job_name, db_conn): - autopilot_regression_trainer = cls(job_name, db_conn) + def train_autopilot_regression_job(cls, job_name, ci_test_env: CITestEnvironment, ): + autopilot_regression_trainer = cls(job_name, ci_test_env.db_conn) problem_params = { "problem_type": "Regression", "objective": '{"MetricName":"MSE"}'} autopilot_regression_trainer.__train( reg_model_setup_params, - problem_params) + problem_params, + ci_test_env + ) @classmethod - def train_autopilot_classification_job(cls, job_name, db_conn, objective: str = '{"MetricName":"F1"}'): - autopilot_classification_trainer = cls(job_name, db_conn) + def train_autopilot_classification_job(cls, job_name, ci_test_env: CITestEnvironment, + objective: str = '{"MetricName":"F1"}'): + autopilot_classification_trainer = cls(job_name, ci_test_env.db_conn) problem_params = { "problem_type": "BinaryClassification", "objective": objective} - + autopilot_classification_trainer.__train( cls_model_setup_params, - problem_params) + problem_params, + ci_test_env) - def __train(self, setup_params, problem_params): + def __train(self, setup_params, problem_params, ci_test_env: CITestEnvironment): params_dict = { - "job_name" : self._job_name, - "aws_credentials_connection_name" : aws_params.aws_conn_name, - "aws_region" : os.environ["AWS_DEFAULT_REGION"], - "iam_sagemaker_role" : os.environ["AWS_ROLE"], - "s3_bucket_uri" : aws_params.aws_bucket_uri, - "s3_output_path" : setup_params.aws_output_path, - "input_schema_name" : setup_params.schema_name, - "input_table_or_view_name" : setup_params.table_name, - "target_attribute_name" : setup_params.target_col, - "max_candidates" : 2, + "job_name": self._job_name, + "aws_credentials_connection_name": ci_test_env.connection_object_for_aws_credentials, + "aws_region": ci_test_env.aws_region, + "iam_sagemaker_role": ci_test_env.aws_sagemaker_role, + "s3_bucket_uri": ci_test_env.aws_bucket_uri, + "s3_output_path": setup_params.aws_output_path, + "input_schema_name": setup_params.schema_name, + "input_table_or_view_name": setup_params.table_name, + "target_attribute_name": setup_params.target_col, + "max_candidates": 2, } params_dict = {**params_dict, **problem_params} @@ -52,5 +57,3 @@ def __train(self, setup_params, problem_params): "{schema}.SME_TRAIN_WITH_SAGEMAKER_AUTOPILOT('{params}')".format( schema=setup_params.schema_name, params=json.dumps(params_dict)) self._db_conn.execute(query_training) - - diff --git a/tests/ci_tests/utils/checkers.py b/tests/ci_tests/utils/checkers.py index 895f2ffb..2b66816e 100644 --- a/tests/ci_tests/utils/checkers.py +++ b/tests/ci_tests/utils/checkers.py @@ -5,6 +5,5 @@ def is_aws_credentials_not_set(): return not ( "AWS_ACCESS_KEY_ID" in os.environ and "AWS_SECRET_ACCESS_KEY" in os.environ and - "AWS_DEFAULT_REGION" in os.environ and - "AWS_ROLE" in os.environ + "AWS_DEFAULT_REGION" in os.environ ) diff --git a/tests/ci_tests/utils/parameters.py b/tests/ci_tests/utils/parameters.py index 048bcf08..c50ec736 100644 --- a/tests/ci_tests/utils/parameters.py +++ b/tests/ci_tests/utils/parameters.py @@ -1,13 +1,11 @@ from collections import namedtuple - -POLLING_INTERVAL = 5*60 # seconds -TIMEOUT = 90*60 # seconds - +POLLING_INTERVAL = 5 * 60 # seconds +TIMEOUT = 90 * 60 # seconds ModelSetupParams = namedtuple("ModelSetupParams", [ "model_type", "schema_name", "table_name", "target_col", - "data", "aws_output_path", "batch_size"]) + "data", "aws_output_path", "batch_size"]) reg_model_setup_params = ModelSetupParams( model_type='reg', @@ -30,19 +28,6 @@ ) -def get_aws_params(): - AWSParams = namedtuple("AWSParams", [ - "aws_bucket", "aws_s3_uri", "aws_bucket_uri", "aws_conn_name"]) - - aws_bucket_name = "persistent-sme-ci-bucket" - return AWSParams( - aws_bucket=aws_bucket_name, - aws_s3_uri=f"https://{aws_bucket_name}.s3.amazonaws.com", - aws_bucket_uri=f"s3://{aws_bucket_name}", - aws_conn_name="aws_connection", - ) - - def get_db_params(): DBParams = namedtuple("DBParams", [ "host", "port", "user", "password"]) @@ -55,7 +40,4 @@ def get_db_params(): ) -aws_params = get_aws_params() db_params = get_db_params() - - diff --git a/tests/integration_tests/fixtures/setup_database_fixture.py b/tests/integration_tests/fixtures/setup_database_fixture.py index 9c80f44e..6ecf4817 100644 --- a/tests/integration_tests/fixtures/setup_database_fixture.py +++ b/tests/integration_tests/fixtures/setup_database_fixture.py @@ -18,8 +18,8 @@ def create_aws_connection(conn): "USER '{aws_key_id}' IDENTIFIED BY '{aws_access_key}'"\ .format(aws_conn_name=aws_params.aws_conn_name, aws_s3_uri=aws_params.aws_s3_uri, - aws_key_id=aws_params.aws_key_id, - aws_access_key=aws_params.aws_access_key) + aws_key_id=aws_params.aws_access_key_id, + aws_access_key=aws_params.aws_secret_access_key) conn.execute(query) diff --git a/tests/integration_tests/test_autopilot_endpoint_deletion_udf_real.py b/tests/integration_tests/test_autopilot_endpoint_deletion_udf_real.py index c9b15179..517e75c2 100644 --- a/tests/integration_tests/test_autopilot_endpoint_deletion_udf_real.py +++ b/tests/integration_tests/test_autopilot_endpoint_deletion_udf_real.py @@ -40,13 +40,13 @@ def get_emitted(self): return self._emitted -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_autopilot_regression_endpoint_deletion_udf_real(): _run_test(reg_setup_params) -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_autopilot_classification_endpoint_deletion_udf_real(): _run_test(cls_setup_params) @@ -61,8 +61,8 @@ def _run_test(setup_params): aws_s3_connection = Connection( address=aws_params.aws_s3_uri, - user=aws_params.aws_key_id, - password=aws_params.aws_access_key) + user=aws_params.aws_access_key_id, + password=aws_params.aws_secret_access_key) exa = ExaEnvironment({aws_params.aws_conn_name: aws_s3_connection}) autopilot_endpoint_deletion_obj = AutopilotEndpointDeletionUDF(exa) autopilot_endpoint_deletion_obj.run(ctx) diff --git a/tests/integration_tests/test_autopilot_endpoint_deployment_udf_real.py b/tests/integration_tests/test_autopilot_endpoint_deployment_udf_real.py index b9e04738..a714c300 100644 --- a/tests/integration_tests/test_autopilot_endpoint_deployment_udf_real.py +++ b/tests/integration_tests/test_autopilot_endpoint_deployment_udf_real.py @@ -49,13 +49,13 @@ def get_emitted(self): return self._emitted -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_autopilot_regression_endpoint_deployment_udf_real(): _run_test(reg_setup_params) -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_autopilot_classification_endpoint_deployment_udf_real(): _run_test(cls_setup_params) @@ -73,8 +73,8 @@ def _run_test(setup_params): aws_s3_connection = Connection( address=aws_params.aws_s3_uri, - user=aws_params.aws_key_id, - password=aws_params.aws_access_key) + user=aws_params.aws_access_key_id, + password=aws_params.aws_secret_access_key) exa = ExaEnvironment({aws_params.aws_conn_name: aws_s3_connection}) autopilot_endpoint_deployment_obj = AutopilotEndpointDeploymentUDF(exa) autopilot_endpoint_deployment_obj.run(ctx) diff --git a/tests/integration_tests/test_autopilot_job_status_polling_udf_real.py b/tests/integration_tests/test_autopilot_job_status_polling_udf_real.py index d816d42c..b1fb25f3 100644 --- a/tests/integration_tests/test_autopilot_job_status_polling_udf_real.py +++ b/tests/integration_tests/test_autopilot_job_status_polling_udf_real.py @@ -40,13 +40,13 @@ def get_emitted(self): return self._emitted -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_poll_autopilot_regression_training_status_udf_real(): _run_test(reg_setup_params) -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_poll_autopilot_classification_training_status_udf_real(): _run_test(cls_setup_params) @@ -61,8 +61,8 @@ def _run_test(setup_params): aws_s3_connection = Connection( address=aws_params.aws_s3_uri, - user=aws_params.aws_key_id, - password=aws_params.aws_access_key) + user=aws_params.aws_access_key_id, + password=aws_params.aws_secret_access_key) exa = ExaEnvironment({aws_params.aws_conn_name: aws_s3_connection}) autopilot_training_status_udf_obj = AutopilotJobStatusPollingUDF(exa) autopilot_training_status_udf_obj.run(ctx) diff --git a/tests/integration_tests/test_autopilot_prediction_udf_real.py b/tests/integration_tests/test_autopilot_prediction_udf_real.py index 1ba56ade..e231e92b 100644 --- a/tests/integration_tests/test_autopilot_prediction_udf_real.py +++ b/tests/integration_tests/test_autopilot_prediction_udf_real.py @@ -62,7 +62,7 @@ def get_dataframe(self, num_rows='all'): return return_df -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_regression_autopilot_prediction_udf_real(): connection_data = { @@ -76,8 +76,8 @@ def test_regression_autopilot_prediction_udf_real(): aws_s3_connection = Connection( address=aws_params.aws_s3_uri, - user=aws_params.aws_key_id, - password=aws_params.aws_access_key) + user=aws_params.aws_access_key_id, + password=aws_params.aws_secret_access_key) model_connection = Connection( address=json.dumps(connection_data)) @@ -101,7 +101,7 @@ def test_regression_autopilot_prediction_udf_real(): assert ctx.get_emitted()[0][0].shape == (3, 3) -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_classification_autopilot_prediction_udf_real(): connection_data = { @@ -115,8 +115,8 @@ def test_classification_autopilot_prediction_udf_real(): aws_s3_connection = Connection( address=aws_params.aws_s3_uri, - user=aws_params.aws_key_id, - password=aws_params.aws_access_key) + user=aws_params.aws_access_key_id, + password=aws_params.aws_secret_access_key) model_connection = Connection( address=json.dumps(connection_data)) diff --git a/tests/integration_tests/test_autopilot_training_udf_real.py b/tests/integration_tests/test_autopilot_training_udf_real.py index 0a467ce4..30e50767 100644 --- a/tests/integration_tests/test_autopilot_training_udf_real.py +++ b/tests/integration_tests/test_autopilot_training_udf_real.py @@ -60,7 +60,7 @@ def get_emitted(self): return self._emitted -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_autopilot_regression_training_udf_real(): params_dict = { @@ -74,7 +74,7 @@ def test_autopilot_regression_training_udf_real(): params_dict['problem_params']) -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_autopilot_classification_training_udf_real(): params_dict = { @@ -88,7 +88,7 @@ def test_autopilot_classification_training_udf_real(): params_dict['problem_params']) -@pytest.mark.skipif(not aws_params.aws_access_key, +@pytest.mark.skipif(not aws_params.aws_secret_access_key, reason="AWS credentials are not set") def test_autopilot_multi_classification_training_udf_real(): params_dict = { @@ -120,8 +120,8 @@ def _run_test(setup_params, problem_params): aws_s3_connection = Connection( address=aws_params.aws_s3_uri, - user=aws_params.aws_key_id, - password=aws_params.aws_access_key) + user=aws_params.aws_access_key_id, + password=aws_params.aws_secret_access_key) exa = ExaEnvironment({aws_params.aws_conn_name: aws_s3_connection}) autopilot_training_udf_obj = AutopilotTrainingUDF(exa) autopilot_training_udf_obj.run(ctx) diff --git a/tests/integration_tests/utils/parameters.py b/tests/integration_tests/utils/parameters.py index 47f91538..efad08c0 100644 --- a/tests/integration_tests/utils/parameters.py +++ b/tests/integration_tests/utils/parameters.py @@ -1,4 +1,5 @@ from collections import namedtuple +import os ''' This module contains namedtuples that store the necessary parameters for the @@ -15,9 +16,9 @@ def get_aws_params(): "aws_region", "aws_s3_uri", "aws_conn_name"]) return AWSParams( - aws_key_id="", - aws_access_key="", - aws_role="", + aws_key_id=os.environ["AWS_ACCESS_KEY_ID"], + aws_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + aws_role="AWS ROLE NOT SET", # FIXME disabled, because failed when we removed Environment Variable in the ci_tests aws_region="eu-central-1", aws_s3_uri="https://sagemaker-extension-bucket.s3.amazonaws.com", aws_conn_name="aws_connection",