From 02c18cce838e21ac03de84ebf3c52780319dc612 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Thu, 1 Feb 2024 16:12:55 +0100 Subject: [PATCH] Fixed version number of VM images etc. (#164) * Fixed version number of VM images etc. --- doc/changes/changelog.md | 1 + doc/changes/changes_0.2.0.md | 20 ++++++++++++ exasol/ds/sandbox/cli/options/id_options.py | 9 +++--- exasol/ds/sandbox/lib/config.py | 3 ++ .../ds/sandbox/lib/dss_docker/create_image.py | 6 ++-- .../lib/release_build/run_release_build.py | 2 +- .../lib/setup_ec2/run_install_dependencies.py | 5 ++- pyproject.toml | 2 +- test/codebuild/test_ci.py | 31 ++++++++++++------- test/unit/test_ansible.py | 27 ++++++++++------ test/unit/test_dss_docker_image.py | 8 ++--- test/unit/test_release_build.py | 11 ++++--- 12 files changed, 85 insertions(+), 40 deletions(-) create mode 100644 doc/changes/changes_0.2.0.md diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 9e62d079..39f13d8b 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,3 +1,4 @@ # Changes +* [0.2.0](changes_0.2.0.md) * [0.1.0](changes_0.1.0.md) diff --git a/doc/changes/changes_0.2.0.md b/doc/changes/changes_0.2.0.md new file mode 100644 index 00000000..2d809681 --- /dev/null +++ b/doc/changes/changes_0.2.0.md @@ -0,0 +1,20 @@ +# ai-lab 0.2.0, released TBD + +Code name: Post release fixes + +## Summary + + +## AI-Lab-Release + +Version: 0.2.0 + +## Features + +## Bug Fixes + +* #163: Fixed version number of VM images etc. + +## Refactoring + +## Documentation diff --git a/exasol/ds/sandbox/cli/options/id_options.py b/exasol/ds/sandbox/cli/options/id_options.py index 06ad4f54..99598c62 100644 --- a/exasol/ds/sandbox/cli/options/id_options.py +++ b/exasol/ds/sandbox/cli/options/id_options.py @@ -1,9 +1,10 @@ import click -from exasol.ds.sandbox.lib.config import SLC_VERSION +from exasol.ds.sandbox.lib.config import AI_LAB_VERSION id_options = [ - click.option('--asset-id', type=str, default=SLC_VERSION, - help="This value will be used in the AMI name, as tag on all AWS resources " - "and in the prefix of the S3 objects.") + click.option( + '--asset-id', type=str, default=AI_LAB_VERSION, + help="This value will be used in the AMI name, as tag on all AWS resources " + "and in the prefix of the S3 objects.") ] diff --git a/exasol/ds/sandbox/lib/config.py b/exasol/ds/sandbox/lib/config.py index 1785955a..9e77940e 100644 --- a/exasol/ds/sandbox/lib/config.py +++ b/exasol/ds/sandbox/lib/config.py @@ -1,5 +1,7 @@ from importlib_metadata import version +# name of the project as specified in file pyproject.toml +AI_LAB_VERSION = version("exasol-ai-lab") SLC_VERSION = version("exasol_script_languages_release") _default_config = { @@ -12,6 +14,7 @@ "state": "available" }, "slc_version": SLC_VERSION, + "ai_lab_version": AI_LAB_VERSION, "waf_region": "us-east-1" } diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index e76eb440..fbded44e 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -17,10 +17,9 @@ from exasol.ds.sandbox.lib.logging import get_status_logger, LogType from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.config import AI_LAB_VERSION DEFAULT_ORG_AND_REPOSITORY = "exasol/ai-lab" -# name of the project as specified in file pyproject.toml -DSS_VERSION = version("exasol-ai-lab") _logger = get_status_logger(LogType.DOCKER_IMAGE) @@ -78,7 +77,7 @@ def __init__( version: str = None, keep_container: bool = False, ): - version = version if version else DSS_VERSION + version = version if version else AI_LAB_VERSION self.container_name = f"ds-sandbox-{DssDockerImage.timestamp()}" self.repository = repository self.version = version @@ -103,6 +102,7 @@ def _ansible_config(self) -> ConfigObject: return ConfigObject( time_to_wait_for_polling=0.1, slc_version=SLC_VERSION, + ai_lab_version=AI_LAB_VERSION, ) def _docker_file(self) -> importlib_resources.abc.Traversable: diff --git a/exasol/ds/sandbox/lib/release_build/run_release_build.py b/exasol/ds/sandbox/lib/release_build/run_release_build.py index 94627525..ebd1fdae 100644 --- a/exasol/ds/sandbox/lib/release_build/run_release_build.py +++ b/exasol/ds/sandbox/lib/release_build/run_release_build.py @@ -75,7 +75,7 @@ def run_start_release_build(aws_access: AwsAccess, config: ConfigObject, upload_url: str, branch: str, gh_token: str) -> None: logging.info(f"run_start_release_build for aws profile {aws_access.aws_profile_for_logging} " f"with upload url: {upload_url}") - _execute_release_build(aws_access, branch, asset_id=config.slc_version, + _execute_release_build(aws_access, branch, asset_id=config.ai_lab_version, release_id=_parse_upload_url(upload_url=upload_url), gh_token=gh_token, make_ami_public=True) diff --git a/exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py b/exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py index 91b65c19..23b88a9e 100644 --- a/exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py +++ b/exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py @@ -24,7 +24,10 @@ def run_install_dependencies(ansible_access: AnsibleAccess, are copied as flat copy to the dynamic working copy, too. The playbook is indicated by variable ansible_run_context, which also might contain additional ansible variables. """ - new_extra_vars = {"slc_version": configuration.slc_version} + new_extra_vars = { + "slc_version": configuration.slc_version, + "ai_lab_version": configuration.ai_lab_version, + } if ansible_run_context.extra_vars is not None: new_extra_vars.update(ansible_run_context.extra_vars) new_ansible_run_context = AnsibleRunContext(ansible_run_context.playbook, new_extra_vars) diff --git a/pyproject.toml b/pyproject.toml index 7efca369..7a649c8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "exasol-ai-lab" -version = "0.1.0" +version = "0.2.0" description = "Provide AI-Lab editions." packages = [ {include = "exasol"}, ] license = "MIT" diff --git a/test/codebuild/test_ci.py b/test/codebuild/test_ci.py index 5a5b2add..e2de2e2a 100644 --- a/test/codebuild/test_ci.py +++ b/test/codebuild/test_ci.py @@ -14,7 +14,7 @@ from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess from exasol.ds.sandbox.lib.asset_id import AssetId from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess -from exasol.ds.sandbox.lib.config import default_config_object, SLC_VERSION +from exasol.ds.sandbox.lib.config import default_config_object, AI_LAB_VERSION from exasol.ds.sandbox.lib.run_create_vm import run_create_vm from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ EC2StackLifecycleContextManager @@ -42,14 +42,25 @@ def change_password(host: str, user: str, curr_pass: str, new_password: str) -> assert res.ok +def _create_asset_id(): + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + return AssetId( + f"ci-test-{AI_LAB_VERSION}-{timestamp}", + stack_prefix="stack", + ami_prefix="ami", + ) + + @pytest.fixture(scope="session") def new_ec2_from_ami(): """ - This fixtures starts the EC-2 instance, runs all setup, exports the AMI, - then starts another EC-2 instance, based on the new AMI, then changes the password (which is expired), - and finally returns that EC-2 name together with the new temporary password. + Start the EC-2 instance, run all setup, export the AMI, then start + another EC-2 instance, based on the new AMI, then change the password + (which is expired), and finally return that EC-2 name together with the + new temporary password. """ - # Create default_password (the one burned into the AMI) and the new password (which will be set during first login) + # Create default_password (the one burned into the AMI) and the new password + # (which will be set during first login) # We use different sizes of both in order to avoid equality of both! default_password = generate_random_password(length=12) new_password = generate_random_password(length=14) @@ -58,10 +69,7 @@ def new_ec2_from_ami(): assert default_password != new_password aws_access = AwsAccess(aws_profile=None) user_name = os.getenv("AWS_USER_NAME") - asset_id = AssetId("ci-test-{suffix}-{now}".format(now=datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), - suffix=SLC_VERSION), - stack_prefix="stack", - ami_prefix="ami") + asset_id = _create_asset_id() run_create_vm(aws_access, None, None, AnsibleAccess(), default_password, tuple(), asset_id, default_config_object, user_name, make_ami_public=False) @@ -71,8 +79,9 @@ def new_ec2_from_ami(): assert len(amis) == 1 ami = amis[0] - lifecycle_generator = run_lifecycle_for_ec2(aws_access, None, None, asset_id=asset_id, - ami_id=ami.id, user_name=user_name) + lifecycle_generator = run_lifecycle_for_ec2( + aws_access, None, None, asset_id=asset_id, + ami_id=ami.id, user_name=user_name) try: with EC2StackLifecycleContextManager(lifecycle_generator, default_config_object) as ec2_data: diff --git a/test/unit/test_ansible.py b/test/unit/test_ansible.py index ea27cfc0..8544f7cd 100644 --- a/test/unit/test_ansible.py +++ b/test/unit/test_ansible.py @@ -34,14 +34,22 @@ def run(self, self.delegate(private_data_dir, run_ctx) +def _extra_vars(config): + return { + "slc_version": config.slc_version, + "ai_lab_version": config.ai_lab_version, + } + def test_run_ansible_default_values(test_config): """ Test which executes run_install_dependencies with default values (default playbook and default ansible variables) """ ansible_access = AnsibleTestAccess() run_install_dependencies(ansible_access, test_config) - expected_ansible_run_context = AnsibleRunContext(playbook="slc_setup.yml", - extra_vars={"slc_version": test_config.slc_version}) + expected_ansible_run_context = AnsibleRunContext( + playbook="slc_setup.yml", + extra_vars=_extra_vars(test_config), + ) assert ansible_access.call_arguments.private_data_dir.startswith("/tmp") assert ansible_access.call_arguments.run_ctx == expected_ansible_run_context @@ -54,8 +62,8 @@ def test_run_ansible_custom_playbook(test_config): ansible_run_context = AnsibleRunContext(playbook="my_playbook.yml", extra_vars=dict()) run_install_dependencies(ansible_access, test_config, host_infos=tuple(), ansible_run_context=ansible_run_context) - expected_ansible_run_context = AnsibleRunContext(playbook="my_playbook.yml", - extra_vars={"slc_version": test_config.slc_version}) + expected_ansible_run_context = AnsibleRunContext( + playbook="my_playbook.yml", extra_vars=_extra_vars(test_config)) assert ansible_access.call_arguments.private_data_dir.startswith("/tmp") assert ansible_access.call_arguments.run_ctx == expected_ansible_run_context @@ -66,11 +74,12 @@ def test_run_ansible_custom_variables(test_config): """ ansible_access = AnsibleTestAccess() ansible_run_context = AnsibleRunContext(playbook="my_playbook.yml", extra_vars={"my_var": True}) - run_install_dependencies(ansible_access, test_config, host_infos=tuple(), ansible_run_context=ansible_run_context) - - expected_ansible_run_context = AnsibleRunContext(playbook="my_playbook.yml", - extra_vars={"slc_version": test_config.slc_version, - "my_var": True}) + run_install_dependencies(ansible_access, test_config, host_infos=tuple(), + ansible_run_context=ansible_run_context) + extra_vars = _extra_vars(test_config) + extra_vars.update({"my_var": True}) + expected_ansible_run_context = AnsibleRunContext( + playbook="my_playbook.yml", extra_vars=extra_vars) assert ansible_access.call_arguments.private_data_dir.startswith("/tmp") assert ansible_access.call_arguments.run_ctx == expected_ansible_run_context diff --git a/test/unit/test_dss_docker_image.py b/test/unit/test_dss_docker_image.py index bdaa7562..550d2e42 100644 --- a/test/unit/test_dss_docker_image.py +++ b/test/unit/test_dss_docker_image.py @@ -3,10 +3,8 @@ from unittest.mock import MagicMock, Mock, create_autospec from datetime import datetime from exasol.ds.sandbox.lib.dss_docker import create_image, DockerRegistry -from exasol.ds.sandbox.lib.dss_docker.create_image import ( - DssDockerImage, - DSS_VERSION, -) +from exasol.ds.sandbox.lib.config import AI_LAB_VERSION +from exasol.ds.sandbox.lib.dss_docker.create_image import DssDockerImage @pytest.fixture @@ -16,7 +14,7 @@ def sample_repo(): def test_constructor_defaults(sample_repo): testee = DssDockerImage(sample_repo) - assert testee.image_name == f"{sample_repo}:{DSS_VERSION}" + assert testee.image_name == f"{sample_repo}:{AI_LAB_VERSION}" assert testee.keep_container == False diff --git a/test/unit/test_release_build.py b/test/unit/test_release_build.py index 8c56852d..8089f4d0 100644 --- a/test/unit/test_release_build.py +++ b/test/unit/test_release_build.py @@ -46,17 +46,18 @@ def test_release_build(test_config): mock_cast(aws_access_mock.get_all_stack_resources).return_value = DUMMY_RESOURCES mock_cast(aws_access_mock.start_codebuild).return_value = (123, create_autospec(CodeBuildWaiter)) run_start_release_build(aws_access_mock, test_config, UPLOAD_URL, BRANCH, GITHUB_TOKEN) - expected_env_variable_overrides = [ + expected_overrides = [ {"name": "RELEASE_ID", "value": "123", "type": "PLAINTEXT"}, - {"name": "ASSET_ID", "value": test_config.slc_version, "type": "PLAINTEXT"}, + {"name": "ASSET_ID", "value": test_config.ai_lab_version, "type": "PLAINTEXT"}, {"name": "GITHUB_TOKEN", "value": GITHUB_TOKEN, "type": "PLAINTEXT"}, {"name": "MAKE_AMI_PUBLIC_OPTION", "value": "--make-ami-public", "type": "PLAINTEXT"} ] mock_cast(aws_access_mock.start_codebuild).\ - assert_called_once_with("codebuild-id-123", - environment_variables_overrides=expected_env_variable_overrides, - branch=BRANCH) + assert_called_once_with( + "codebuild-id-123", + environment_variables_overrides=expected_overrides, + branch=BRANCH) def test_test_release_build(test_config):