From a9caffdbe116c91a1479e4e9201d362d18db3426 Mon Sep 17 00:00:00 2001 From: Ryan McGinty Date: Mon, 8 Jul 2024 19:17:01 -0700 Subject: [PATCH 1/6] wip - updating pipeline stack base class --- pyproject.toml | 5 +- .../cicd/pipeline/base.py | 408 +++++++++++++++++- .../cicd/pipeline/scripts/cicd-release.sh | 119 +++++ 3 files changed, 526 insertions(+), 6 deletions(-) create mode 100755 src/aibs_informatics_cdk_lib/cicd/pipeline/scripts/cicd-release.sh diff --git a/pyproject.toml b/pyproject.toml index 971b44f..7cf1f16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,10 @@ dev = [ version = {attr = "aibs_informatics_cdk_lib._version.__version__"} [tool.setuptools.package-data] -"*" = ['py.typed'] +"*" = [ + 'py.typed', + 'src/aibs_informatics_cdk_lib/cicd/pipeline/scripts/cicd-release.sh', +] [tool.setuptools.packages.find] where = ["src"] diff --git a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py index 125654d..508450a 100644 --- a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py +++ b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py @@ -1,10 +1,26 @@ +import base64 import logging from abc import abstractmethod -from typing import Dict, Generic, List, Mapping, TypeVar +from pathlib import Path +from typing import ( + Callable, + Dict, + Generic, + List, + Mapping, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, + cast, +) import aws_cdk as cdk import constructs from aibs_informatics_core.env import EnvBase +from aws_cdk import aws_codepipeline as aws_codepipeline from aws_cdk import aws_codepipeline_actions from aws_cdk import aws_codestarnotifications as codestarnotifications from aws_cdk import aws_iam as iam @@ -18,6 +34,7 @@ BuildSpec, LinuxBuildImage, ) +from dataclasses_json import global_config from aibs_informatics_cdk_lib.common.aws.core_utils import build_arn from aibs_informatics_cdk_lib.common.aws.iam_utils import ( @@ -46,12 +63,131 @@ STAGE_CONFIG = TypeVar("STAGE_CONFIG", bound=StageConfig) GLOBAL_CONFIG = TypeVar("GLOBAL_CONFIG", bound=GlobalConfig) +PIPELINE_STACK = TypeVar("PIPELINE_STACK", bound="BasePipelineStack") + + +import functools +from dataclasses import dataclass + + +@dataclass +class PipelineStageInfo: + order: int + name: str + pre_steps: Optional[List[pipelines.Step]] = None + post_steps: Optional[List[pipelines.Step]] = None + + +def pipeline_stage( + order: int, + name: str, + pre_steps: Optional[List[pipelines.Step]] = None, + post_steps: Optional[List[pipelines.Step]] = None, +): + """Method decorator for defining a pipeline stage in a BasePipelineStack subclass. + + you can decorate two types of methods: + 1. A method that returns a cdk.Stage + 2. A method that returns a tuple of pre_steps, cdk.Stage, post_steps + where pre_steps and post_steps are lists of pipelines.Step objects. + + Example: + + ```python + class PipelineStack(BasePipelineStack): + + ... + + @pipeline_stage(order=0, name="Source", pre_steps=[...]) + def source_stage(self) -> cdk.Stage: + return SourceStage(self, self.get_construct_id("source-stage")) + + @pipeline_stage(order=1, name="Build") + def build_stage(self) -> cdk.Stage: + return BuildStage(self, self.get_construct_id("build-stage")) + + @pipeline_stage(order=2, name="Deploy") + def deploy_stage(self) -> Tuple[List[pipelines.Step], cdk.Stage, List[pipelines.Step]]: + pre_steps = [...] + post_steps = [...] + stage = DeployStage(self, self.get_construct_id("deploy-stage")) + return pre_steps, stage, post_steps + + ``` + + Args: + order (int): Order of the stage. Lower numbers are executed first. E.g. 1, 2, 3, ... + You can repeat numbers, however, the order will be arbitrary. + name (str): Name of the stage + pre_steps (Optional[List[pipelines.Step]], optional): Optional pre steps to add before the stage. + Defaults to None. + post_steps (Optional[List[pipelines.Step]], optional): Optional post steps to add after the stage. + Defaults to None. + """ + + def decorator_pipeline_stage(func): + @functools.wraps(func) + def wrapper_pipeline_stage(*args, **kwargs): + return func(*args, **kwargs) + + wrapper_pipeline_stage._pipeline_stage_info = PipelineStageInfo( # type: ignore[attr-defined] + order=order, name=name, pre_steps=pre_steps, post_steps=post_steps + ) + return wrapper_pipeline_stage + + return decorator_pipeline_stage + class BasePipelineStack(EnvBaseStack, Generic[STAGE_CONFIG, GLOBAL_CONFIG]): """Defines the CI/CD Pipeline for the an Environment. - https://docs.aws.amazon.com/cdk/api/v1/docs/pipelines-readme.html + This class is meant to be subclassed to define the pipeline for a specific project. + + You are required to implement the `initialize_pipeline` method which should return + a `pipelines.CodePipeline` object. You can then add stages to the pipeline by defining + methods that are decorated with the `@pipeline_stage` decorator. + + Example: + + ```python + + class MyPipelineStack(BasePipelineStack): + + @pipeline_stage(order=1, pre_steps=[...], post_steps=[...]) + def build_stage(self) -> cdk.Stage: + # Define the steps for the build stage + build_steps = [ + pipelines.CodeBuildStep( + "Build", + input=self.get_pipeline_source(self.pipeline_config.source), + commands=[ + "echo 'Building the project'", + "npm install", + "npm run build", + ], + role_policy_statements=[ + self.get_policy_with_secrets(self.pipeline_config.source.oauth_secret_name), + ], + ), + ] + # Create the build stage + build_stage = pipelines.Stage( + self, + "BuildStage", + stage_name="Build", + actions=build_steps, + ) + + return build_stage + + + The following steps are available for pipelines inheriting from this class: + promotion_stage: A stage that is added after all other stages. This stage + is used to promote the deployment to another environment. + notifications: A notification topic that is used to send notifications. You can + enable notifications for pipeline failures and successes. This + can be configured in the `pipeline_config.notifications` attribute. """ def __init__( @@ -68,8 +204,227 @@ def __init__( account=self.stage_config.env.account, region=self.stage_config.env.region ) super().__init__(scope, id, config=config, env=env, **kwargs) + self.build_pipeline() + + @abstractmethod + def initialize_pipeline(self) -> pipelines.CodePipeline: + raise NotImplementedError("Subclasses must implement this method") + + def build_pipeline(self): + """Builds the pipeline. This method should be called after the pipeline is initialized. + + This should not be overridden by subclasses unless you know what you are doing. + + This method will: + 1. Initialize the pipeline + 2. Add stages to the pipeline + 3. Add a promotion stage + 4. Build the pipeline + 5. Setup notifications + + """ + # Initialize Pipeline self.pipeline = self.initialize_pipeline() + # Add Stages + for stage_method in self.get_stage_methods(): + stage_info: PipelineStageInfo = stage_method._pipeline_stage_info # type: ignore[attr-defined] + stage = stage_method() + pre_steps = stage_info.pre_steps + post_steps = stage_info.post_steps + if isinstance(stage, cdk.Stage): + stage = stage + elif ( + isinstance(stage, tuple) + and len(stage) == 3 + and isinstance(stage[0], list) + and isinstance(stage[1], cdk.Stage) + and isinstance(stage[2], list) + ): + pre_steps = [*(pre_steps or []), *(cast(List[pipelines.Step], stage[0]))] + post_steps = [*(post_steps or []), *(cast(List[pipelines.Step], stage[2]))] + stage = stage[1] + else: + raise ValueError( + "Stage must be a cdk.Stage or a tuple of pre_steps, stage, post_steps" + ) + self.pipeline.add_stage(stage, pre=pre_steps, post=post_steps) + + # Add Promotion Stage + self.add_promotion_stage(self.pipeline) + + # Build the pipeline + self.pipeline.build_pipeline() + + # Post Build Setup + self.setup_notifications(self.pipeline) + + def add_promotion_stage(self, pipeline: pipelines.CodePipeline): + """Adds a promotion stage to a CodePipeline + + Promotion stages are used to promote the deployment to another environment. + These promotions are done through github pull requests. This is a major foundation + for the deployment process. + + The environment promotion definitions are defined in the `global_config.stage_promotions`. + This is a mapping of source environment types to target environment types. + + The branch that is used for the promotion is defined in the `pipeline_config.source.branch`. + + Args: + pipeline (pipelines.CodePipeline): Code Pipeline + """ + global_config = self.global_config + pipeline_config = self.pipeline_config + + # In order to add a CodePipeline Stage without stacks, we must use `add_wave` + # https://github.com/aws/aws-cdk/issues/15945#issuecomment-895392052 + promote_wave = pipeline.add_wave("Release") + + # POST Steps + if (source_env_type := self.stage_config.env.env_type) in global_config.stage_promotions: + promotion_target_env_type = global_config.stage_promotions[source_env_type] + promotion_target_pipeline_config = self.project_config.get_stage_config( + promotion_target_env_type + ).pipeline + assert promotion_target_pipeline_config is not None + create_pull_request_step = pipelines.CodeBuildStep( + "CreateReleasePullRequest", + input=self.get_pipeline_source(pipeline_config.source), + # Environment needs to have privelaged access + build_environment=BuildEnvironment(privileged=True), + # By default bin/sh is used, so lets set to bash + # https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#build-spec.shell + partial_build_spec=BuildSpec.from_object( + { + "env": { + "shell": "bash", + "variables": { + "CICD_RELEASE_REVIEWER": "AllenInstitute/marmot", + "CICD_RELEASE_SOURCE_ENV_TYPE": source_env_type, + "CICD_RELEASE_TARGET_ENV_TYPE": promotion_target_env_type, + "CICD_RELEASE_TARGET_BRANCH": promotion_target_pipeline_config.source.branch, + }, + # https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#build-spec.env.secrets-manager + "secrets-manager": { + "GITHUB_TOKEN": pipeline_config.source.oauth_secret_name, + }, + "git-credential-helper": "yes", + }, + } + ), + install_commands=[ + # Installing Github CLI (via https://github.com/cli/cli/blob/trunk/docs/install_linux.md) + # 1. Resolve Download URL via GH API + # 2. Download binary archive + # 3. Unarchive and move binary into /usr/local/bin + # 4. Verify command is available + # Step 1: + 'GH_CLI_DOWNLOAD_LINK=$(curl -H "Authorization:token $GITHUB_TOKEN" -sSL "https://api.github.com/repos/cli/cli/releases/latest" | jq -r \'.assets[] | select(.name|test(".*_linux_amd64.tar.gz")) | .browser_download_url\')', + "GH_CLI_TAR_GZ_PATH=$(basename $GH_CLI_DOWNLOAD_LINK)", + "GH_CLI_DIR=$(basename $GH_CLI_TAR_GZ_PATH .tar.gz)", + # Step 2: + 'curl -H "Authorization:token $GITHUB_TOKEN" -sSL $GH_CLI_DOWNLOAD_LINK -o $GH_CLI_TAR_GZ_PATH', + # Step 3: + "tar -xf $GH_CLI_TAR_GZ_PATH", + "sudo cp $GH_CLI_DIR/bin/gh /usr/local/bin/", + # Step 4: + "gh --version &> /dev/null", + ], + commands=[ + # Setting up repository WITH git metadata + # Why? + # because Github Version 1 CodePipeline Source does not support + # option for including git metadata. Github Version 2 does this, + # but we cannot use this configuration currently. + # What is going on below? + # 1. clone the git repository and work off of that. + # 2. Enable caching and store credentials + # 3. Checkout branch based on source commit + # 4. Run our CI/CD release script + "export REPO_DIR=$(mktemp -d)", + "cd $REPO_DIR", + f"git clone https://${{GITHUB_TOKEN}}@github.com/{pipeline_config.source.repository}.git .", + # Enables credential caching + "git config credential.helper store", + # Supposed to force the caching of the credentials + "git pull", + # Creates a temporary branch using the source commit as its head. + # This ensures that we use the release branch. + "git checkout -b $(basename $REPO_DIR) $CODEBUILD_RESOLVED_SOURCE_VERSION", + ## Step: Download and run release script + # Create a temporary directory and file to store the release script + "export RELEASE_SCRIPT_PATH=$(mktemp -d)/cicd-release.sh", + "mkdir -p $(dirname $RELEASE_SCRIPT_PATH)", + # TODO: Decide which approach is better (prefer 2) + # 1. Download the release script from the source repository (using gh cli) + # - This requires the use of the Github CLI + # - This does not couple changes being deployed with the script in repo + # - This is the most direct approach + # 2. Base64 encode the release script and decode it on the other side + # - This is a bit more complex + # - This couples changes being deployed with the script in repo + ( + # Download the release script from the source repository (using gh cli) + 'gh api repos/AllenInstitute/aibs-informatics-cdk-lib/contents/src/aibs_informatics_cdk_lib/cicd/pipeline/scripts/cicd-release.sh --raw -H "Accept: application/vnd.github.v3.raw" > $RELEASE_SCRIPT_PATH' + if False + else + # Here we are base64 encoding the release script and decoding it on the other side + # Steps: + # 1. Read the release script file + # 2. Base64 encode the file + # 3. Decode the base64 encoded file and write it to the release script path + f"echo {base64.b64encode((Path(__file__).parent / 'scripts' / 'cicd-release.sh').read_text().encode()).decode()} | base64 --decode > $RELEASE_SCRIPT_PATH" + ), + # Run the release script + "bash $RELEASE_SCRIPT_PATH", + ], + role_policy_statements=[ + CODE_BUILD_IAM_POLICY, + self.get_policy_with_secrets(self.pipeline_config.source.oauth_secret_name), + ], + ) + # Add dependencies to all other "post" steps + if promote_wave.post: + for post_step in promote_wave.post: + create_pull_request_step.add_step_dependency(post_step) + + promote_wave.add_post(create_pull_request_step) + + def setup_notifications(self, pipeline: pipelines.CodePipeline): + notifications_config = self.pipeline_config.notifications + if notifications_config.notify_on_any: + sns_notifications_topic = sns.Topic( + self, + self.get_construct_id("sns-notifications"), + display_name=f"Deployment Pipeline Notifications ({self.env_base})", + topic_name=f"{self.env_base}-deployment-pipeline-notifications", + ) + + # Pipeline/Stage/Action Failure Notifications + pipeline.pipeline.notify_on( + self.get_construct_id("any-failures"), + target=sns_notifications_topic, # type: ignore # Topic should match ITopic + enabled=notifications_config.notify_on_failure, + events=[ + aws_codepipeline.PipelineNotificationEvents.PIPELINE_EXECUTION_FAILED, + ], + notification_rule_name=f"{self.env_base}-Deployment-Pipeline-Failures", + detail_type=codestarnotifications.DetailType.FULL, + ) + + # Pipeline Completion Notifications + pipeline.pipeline.notify_on( + self.get_construct_id("pipeline-complete"), + target=sns_notifications_topic, # type: ignore # Topic should match ITopic + enabled=notifications_config.notify_on_success, + events=[ + aws_codepipeline.PipelineNotificationEvents.PIPELINE_EXECUTION_SUCCEEDED, + ], + detail_type=codestarnotifications.DetailType.BASIC, + notification_rule_name=f"{self.env_base}-Deployment-Pipeline-Success", + ) + @property def project_config(self) -> BaseProjectConfig[GLOBAL_CONFIG, STAGE_CONFIG]: return self._project_config @@ -167,6 +522,49 @@ def get_pipeline_source( self.source_cache[source_config.repository] = source return self.source_cache[source_config.repository] - @abstractmethod - def initialize_pipeline(self) -> pipelines.CodePipeline: - raise NotImplementedError("Subclasses must implement this method") + def get_stage_methods( + self, + ) -> List[ + Union[ + Callable[[], cdk.Stage], + Callable[[], Tuple[Sequence[pipelines.Step], cdk.Stage, Sequence[pipelines.Step]]], + ] + ]: + # Get all methods of the instance + methods = [ + getattr(self, method_name) + for method_name in dir(self) + if callable(getattr(self, method_name)) + ] + + # Filter methods that have the _pipeline_stage_info attribute + stage_methods = [method for method in methods if hasattr(method, "_pipeline_stage_info")] + + # Sort methods by their order attribute + stage_methods.sort(key=lambda method: method._pipeline_stage_info.order) # type: ignore[attr-defined] + # Return the sorted methods + + return stage_methods + + @staticmethod + def get_policy_with_secrets(*secret_names: Optional[str]) -> iam.PolicyStatement: + return iam.PolicyStatement( + effect=iam.Effect.ALLOW, + actions=[ + "secretsmanager:GetRandomPassword", + "secretsmanager:GetResourcePolicy", + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + "secretsmanager:ListSecretVersionIds", + ], + resources=[ + build_arn( + service="secretsmanager", + resource_type="secret", + resource_delim=":", + resource_id=f"{secret_name}-??????", + ) + for secret_name in secret_names + if secret_name is not None + ], + ) diff --git a/src/aibs_informatics_cdk_lib/cicd/pipeline/scripts/cicd-release.sh b/src/aibs_informatics_cdk_lib/cicd/pipeline/scripts/cicd-release.sh new file mode 100755 index 0000000..352476e --- /dev/null +++ b/src/aibs_informatics_cdk_lib/cicd/pipeline/scripts/cicd-release.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +################################################################# +# CI/CD Release Script +# Description: +# Purpose of this script is to facilitate submit Pull Requests +# from a source branch/commit to a destination branch. +# +# Input Environment Variables: +# +# CICD_RELEASE_SOURCE_ENV_TYPE: +# Environment Type of source branch +# CICD_RELEASE_TARGET_ENV_TYPE: +# Environment Type of source branch +# CICD_RELEASE_TARGET_BRANCH: +# Target branch to submit pull request into +# CICD_RELEASE_REVIEWER: +# Reviewers for the PR + +################################### + + +export CICD_RELEASE_SOURCE_COMMIT=$CODEBUILD_RESOLVED_SOURCE_VERSION +export CICD_RELEASE_CANDIDATE_BRANCH="candidate/$CICD_RELEASE_TARGET_BRANCH" + +echo "==> CI/CD Release Inputs:" +echo "==> CICD_RELEASE_SOURCE_ENV_TYPE = $CICD_RELEASE_SOURCE_ENV_TYPE" +echo "==> CICD_RELEASE_TARGET_ENV_TYPE = $CICD_RELEASE_TARGET_ENV_TYPE" +echo "==> CICD_RELEASE_SOURCE_COMMIT = $CICD_RELEASE_SOURCE_COMMIT" +echo "==> CICD_RELEASE_CANDIDATE_BRANCH = $CICD_RELEASE_CANDIDATE_BRANCH" +echo "==> CICD_RELEASE_TARGET_BRANCH = $CICD_RELEASE_TARGET_BRANCH" +echo "==> CICD_RELEASE_REVIEWER = $CICD_RELEASE_REVIEWER" + +export CICD_RELEASE_GIT_MESSAGE="$(git log -1 --pretty=%B)" +export CICD_RELEASE_GIT_AUTHOR="$(git log -1 --pretty=%an)" +export CICD_RELEASE_GIT_AUTHOR_EMAIL="$(git log -1 --pretty=%ae)" +export CICD_RELEASE_GIT_COMMIT="$(git log -1 --pretty=%H)" +export CICD_RELEASE_GIT_SHORT_COMMIT="$(git log -1 --pretty=%h)" + +echo "==> CICD_RELEASE_GIT_MESSAGE = $CICD_RELEASE_GIT_MESSAGE" +echo "==> CICD_RELEASE_GIT_AUTHOR = $CICD_RELEASE_GIT_AUTHOR" +echo "==> CICD_RELEASE_GIT_AUTHOR_EMAIL = $CICD_RELEASE_GIT_AUTHOR_EMAIL" +echo "==> CICD_RELEASE_GIT_COMMIT = $CICD_RELEASE_GIT_COMMIT" +echo "==> CICD_RELEASE_GIT_SHORT_COMMIT = $CICD_RELEASE_GIT_SHORT_COMMIT" +echo + + +echo "Verify gh command is on PATH" + +if ! command -v gh &> /dev/null; then + echo "==! Could not find gh command on PATH. EXITING" + exit 1 +fi + +echo +echo "==> Promoting commits up to $CICD_RELEASE_GIT_SHORT_COMMIT to release candidate branch." +echo "==> Release candidate branch: $CICD_RELEASE_CANDIDATE_BRANCH" + +echo "[command] git checkout -B $CICD_RELEASE_CANDIDATE_BRANCH $CICD_RELEASE_SOURCE_COMMIT" +git checkout -B $CICD_RELEASE_CANDIDATE_BRANCH $CICD_RELEASE_SOURCE_COMMIT +echo "[command] git push --set-upstream --force" +git push --set-upstream --force origin $CICD_RELEASE_CANDIDATE_BRANCH + +CICD_RELEASE_DATE=$(date '+%Y-%m-%d') +CICD_RELEASE_PR_TITLE="Release $CICD_RELEASE_SOURCE_ENV_TYPE -> $CICD_RELEASE_TARGET_ENV_TYPE ($CICD_RELEASE_DATE)" + +CICD_RELEASE_PR_MESSAGE_FILE=$(mktemp) + + +cat < $CICD_RELEASE_PR_MESSAGE_FILE +# Release +## Release Summary +| Release Attribute | Value | +| --- | --- | +| Target Branch | $CICD_RELEASE_TARGET_BRANCH | +| Source Branch | $CICD_RELEASE_CANDIDATE_BRANCH ($CICD_RELEASE_GIT_SHORT_COMMIT) | +| Date | $(date '+%Y-%m-%d %H:%M:%S') | + +## Release Notes + +This release includes changes up to $CICD_RELEASE_GIT_SHORT_COMMIT. This includes the following: +- (fill me please) +- (fill me please) +- (fill me please) + +## Checklist +- [ ] All of GCS works impeccably + +EOF + + +echo "==> Checking for open Pull Requests..." +EXISTING_PR_NUMBER=$(gh pr list -B $CICD_RELEASE_TARGET_BRANCH -L 1 | cut -f1) + +if [[ ! -z $EXISTING_PR_NUMBER ]]; then + echo "==> Pull Request already exists ($EXISTING_PR_NUMBER). Updating..." + + # Update the PR message + echo "" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE + echo "---" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE + echo "# Previous Revisions" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE + echo "---" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE + echo "" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE + gh pr view --json body | jq -r '.body' >> $CICD_RELEASE_PR_MESSAGE_FILE + + gh pr edit $EXISTING_PR_NUMBER \ + --title "$CICD_RELEASE_PR_TITLE" \ + --body-file $CICD_RELEASE_PR_MESSAGE_FILE + +else + + echo "==> Creating new Pull Request" + + gh pr create \ + --base $CICD_RELEASE_TARGET_BRANCH \ + --title "$CICD_RELEASE_PR_TITLE" \ + --body-file "$CICD_RELEASE_PR_MESSAGE_FILE" \ + --reviewer "$CICD_RELEASE_REVIEWER" +fi \ No newline at end of file From 31ffa7ee2a848caec01e7fb8482b28b3e1d65292 Mon Sep 17 00:00:00 2001 From: Ryan McGinty Date: Tue, 9 Jul 2024 16:41:43 -0700 Subject: [PATCH 2/6] create base enum for cdk stack target + utils --- .../cicd/pipeline/base.py | 23 ++---- src/aibs_informatics_cdk_lib/cicd/target.py | 79 ++++++++++++++++++- .../project/config.py | 4 +- src/aibs_informatics_cdk_lib/project/utils.py | 41 +++++++--- .../project/test_utils.py | 17 ++++ 5 files changed, 132 insertions(+), 32 deletions(-) diff --git a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py index 508450a..ea1b4d7 100644 --- a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py +++ b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py @@ -24,31 +24,17 @@ from aws_cdk import aws_codepipeline_actions from aws_cdk import aws_codestarnotifications as codestarnotifications from aws_cdk import aws_iam as iam -from aws_cdk import aws_s3 as s3 -from aws_cdk import aws_secretsmanager as secretsmanager from aws_cdk import aws_sns as sns from aws_cdk import pipelines -from aws_cdk.aws_codebuild import ( - BuildEnvironment, - BuildEnvironmentVariable, - BuildSpec, - LinuxBuildImage, -) -from dataclasses_json import global_config +from aws_cdk.aws_codebuild import BuildEnvironment, BuildEnvironmentVariable, BuildSpec from aibs_informatics_cdk_lib.common.aws.core_utils import build_arn -from aibs_informatics_cdk_lib.common.aws.iam_utils import ( - CODE_BUILD_IAM_POLICY, - DYNAMODB_READ_ACTIONS, - S3_FULL_ACCESS_ACTIONS, -) +from aibs_informatics_cdk_lib.common.aws.iam_utils import CODE_BUILD_IAM_POLICY from aibs_informatics_cdk_lib.project.config import ( BaseProjectConfig, CodePipelineSourceConfig, - Env, GlobalConfig, PipelineConfig, - ProjectConfig, StageConfig, ) from aibs_informatics_cdk_lib.stacks.base import EnvBaseStack @@ -200,10 +186,11 @@ def __init__( ) -> None: self.project_config = config self.stage_config = config.get_stage_config(env_base.env_type) + self.stage_config.env.label = env_base.env_label env = cdk.Environment( account=self.stage_config.env.account, region=self.stage_config.env.region ) - super().__init__(scope, id, config=config, env=env, **kwargs) + super().__init__(scope, id, env_base=env_base, env=env, **kwargs) self.build_pipeline() @abstractmethod @@ -228,7 +215,7 @@ def build_pipeline(self): # Add Stages for stage_method in self.get_stage_methods(): - stage_info: PipelineStageInfo = stage_method._pipeline_stage_info # type: ignore[attr-defined] + stage_info: PipelineStageInfo = stage_method._pipeline_stage_info # type: ignore[union-attr] stage = stage_method() pre_steps = stage_info.pre_steps post_steps = stage_info.post_steps diff --git a/src/aibs_informatics_cdk_lib/cicd/target.py b/src/aibs_informatics_cdk_lib/cicd/target.py index b7526b3..4241c46 100644 --- a/src/aibs_informatics_cdk_lib/cicd/target.py +++ b/src/aibs_informatics_cdk_lib/cicd/target.py @@ -1,6 +1,79 @@ from enum import Enum +from typing import Optional, Type, TypeVar, Union +import constructs +from aibs_informatics_core.utils.os_operations import get_env_var -class CDKStackTarget(str, Enum): - PIPELINE = "pipeline" - INFRA = "infra" +from aibs_informatics_cdk_lib.project.utils import _get_from_context + +CDK_STACK_TARGET_ENV_VAR = "CDK_STACK_TARGET" + +T = TypeVar("T", bound="CDKStackTargetBaseEnum") + + +class CDKStackTargetBaseEnum(Enum): + """Base class for CDK stack target types + + Usage: + class MyCDKStackTarget(str, CDKStackTargetBaseEnum): + INFRA = "pipeline" + + """ + + @classmethod + def from_env(cls: Type[T], default: Union[str, T]) -> T: + target = get_env_var(CDK_STACK_TARGET_ENV_VAR) + target = target or default + return cls(target) + + @classmethod + def from_context( + cls: Type[T], + node: constructs.Node, + default: Union[str, T], + context_keys: Optional[list[str]] = None, + ) -> T: + """Resolves the CDK stack target type from context + + Args: + cls (Type[T]): subclassed CDKStackTargetBase + node (constructs.Node): cdk construct node + default (str): default to use. + context_keys (Optional[list[str]], optional): overrides for context names. + Defaults to None. + + Returns: + T: CDKStackTargetBase instance + """ + context_keys = context_keys or ["target", "stack_target"] + + target = _get_from_context(node, context_keys) or default + + return cls(target) + + @classmethod + def from_context_or_env( + cls: Type[T], + node: constructs.Node, + default: Union[str, T], + context_keys: Optional[list[str]] = None, + ) -> T: + """Resolves the CDK stack target type from context or environment + + Order of resolution: + 1. CDK context value (specifying -c K=V) + 2. env variable + 3. default value ("dev") + + Args: + cls (Type[T]): subclassed CDKStackTargetBase + node (constructs.Node): cdk construct node + default (str): default to use. + context_keys (Optional[list[str]], optional): overrides for context names. + Defaults to None. + + """ + + return cls.from_context( + node=node, default=cls.from_env(default), context_keys=context_keys + ) diff --git a/src/aibs_informatics_cdk_lib/project/config.py b/src/aibs_informatics_cdk_lib/project/config.py index 7539362..81d0ba5 100644 --- a/src/aibs_informatics_cdk_lib/project/config.py +++ b/src/aibs_informatics_cdk_lib/project/config.py @@ -85,8 +85,8 @@ class CodePipelineBuildConfig(BaseModel): class CodePipelineSourceConfig(BaseModel): repository: str branch: Annotated[str, PlainValidator(EnvVarStr.validate)] - codestar_connection: Optional[UniqueIDType] - oauth_secret_name: Optional[str] + codestar_connection: Optional[UniqueIDType] = None + oauth_secret_name: Optional[str] = None @model_validator(mode="after") @classmethod diff --git a/src/aibs_informatics_cdk_lib/project/utils.py b/src/aibs_informatics_cdk_lib/project/utils.py index e13cae6..ce270f0 100644 --- a/src/aibs_informatics_cdk_lib/project/utils.py +++ b/src/aibs_informatics_cdk_lib/project/utils.py @@ -6,8 +6,9 @@ ] import logging +import os import pathlib -from typing import List, Optional, Type, Union +from typing import List, Optional, Tuple, Type, Union import constructs from aibs_informatics_core.env import ( @@ -21,7 +22,7 @@ ) from aibs_informatics_core.utils.os_operations import get_env_var, set_env_var -from aibs_informatics_cdk_lib.project.config import BaseProjectConfig, G, ProjectConfig, S +from aibs_informatics_cdk_lib.project.config import BaseProjectConfig, G, P, ProjectConfig, S logger = logging.getLogger(__name__) @@ -120,19 +121,41 @@ def get_env_base(node: constructs.Node) -> EnvBase: return EnvBase.from_type_and_label(env_type=env_type, env_label=env_label) -def get_config( - node: constructs.Node, project_config_cls: Type[BaseProjectConfig[G, S]] = ProjectConfig -) -> S: - env_base = get_env_base(node) +def set_env_base(env_base: EnvBase) -> None: + """Set the environment base + Args: + env_base (EnvBase): environment base + """ set_env_var(EnvBase.ENV_BASE_KEY, env_base) set_env_var(EnvBase.ENV_TYPE_KEY, env_base.env_type) if env_base.env_label: set_env_var(EnvBase.ENV_LABEL_KEY, env_base.env_label) + else: + os.environ.pop(EnvBase.ENV_LABEL_KEY, None) + + +def get_project_config_and_env_base( + node: constructs.Node, project_config_cls: Type[P] = ProjectConfig +) -> Tuple[P, EnvBase]: + env_base = get_env_base(node) + + config = project_config_cls.load_config() + return config, env_base + + +def get_config( + node: constructs.Node, project_config_cls: Type[BaseProjectConfig[G, S]] = ProjectConfig +) -> S: + project_config, env_base = get_project_config_and_env_base( # type: ignore + node, project_config_cls=project_config_cls + ) + set_env_base(env_base) + + stage_config: S = project_config.get_stage_config(env_type=env_base.env_type) + stage_config.env.label = env_base.env_label - config: S = project_config_cls.load_stage_config(env_type=env_base.env_type) - config.env.label = env_base.env_label - return config + return stage_config def _get_from_context( diff --git a/test/aibs_informatics_cdk_lib/project/test_utils.py b/test/aibs_informatics_cdk_lib/project/test_utils.py index ac76403..5da56cd 100644 --- a/test/aibs_informatics_cdk_lib/project/test_utils.py +++ b/test/aibs_informatics_cdk_lib/project/test_utils.py @@ -1,3 +1,5 @@ +import os + import aws_cdk as cdk import constructs import pytest @@ -19,6 +21,7 @@ ENV_LABEL_KEYS, ENV_TYPE_KEYS, get_env_base, + set_env_base, ) USER = "marmotdev" @@ -104,3 +107,17 @@ def test__get_env_base__context_and_env_vars(env_vars, dummy_node): # Base from context supercedes type/label dummy_node.set_context(ENV_BASE_KEY, "dev") assert get_env_base(dummy_node) == EnvBase("dev") + + +def test__set_env_base__env_vars_only(env_vars): + env_base = EnvBase("dev") + set_env_base(env_base) + assert os.environ.get(ENV_BASE_KEY) == "dev" + assert os.environ.get(ENV_TYPE_KEY) == "dev" + assert os.environ.get(ENV_LABEL_KEY) is None + + env_base = EnvBase("prod-marmot") + set_env_base(env_base) + assert os.environ.get(ENV_BASE_KEY) == "prod-marmot" + assert os.environ.get(ENV_TYPE_KEY) == "prod" + assert os.environ.get(ENV_LABEL_KEY) == "marmot" From 9f11bd52d9c957e81ffdc2c4b48ddf0f7622aa2c Mon Sep 17 00:00:00 2001 From: Ryan McGinty Date: Thu, 11 Jul 2024 13:53:54 -0700 Subject: [PATCH 3/6] add sh scripts to package data --- pyproject.toml | 2 +- src/aibs_informatics_cdk_lib/cicd/pipeline/base.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7cf1f16..409b49a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ version = {attr = "aibs_informatics_cdk_lib._version.__version__"} [tool.setuptools.package-data] "*" = [ 'py.typed', - 'src/aibs_informatics_cdk_lib/cicd/pipeline/scripts/cicd-release.sh', + '*.sh', ] [tool.setuptools.packages.find] diff --git a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py index ea1b4d7..e80400c 100644 --- a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py +++ b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py @@ -11,11 +11,11 @@ Optional, Sequence, Tuple, - Type, TypeVar, Union, cast, ) +from importlib.resources import files import aws_cdk as cdk import constructs @@ -361,6 +361,7 @@ def add_promotion_stage(self, pipeline: pipelines.CodePipeline): # 1. Read the release script file # 2. Base64 encode the file # 3. Decode the base64 encoded file and write it to the release script path + # TODO: i think importlib f"echo {base64.b64encode((Path(__file__).parent / 'scripts' / 'cicd-release.sh').read_text().encode()).decode()} | base64 --decode > $RELEASE_SCRIPT_PATH" ), # Run the release script From 4da2eca4989f646c4902fdce087136f19d0fd2f3 Mon Sep 17 00:00:00 2001 From: Ryan McGinty Date: Thu, 11 Jul 2024 15:16:26 -0700 Subject: [PATCH 4/6] updates based on pr comments --- .../cicd/pipeline/base.py | 72 ++++++++++--------- src/aibs_informatics_cdk_lib/project/utils.py | 12 ++++ .../project/test_utils.py | 23 +++--- 3 files changed, 64 insertions(+), 43 deletions(-) diff --git a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py index e80400c..6cef276 100644 --- a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py +++ b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py @@ -1,6 +1,7 @@ import base64 import logging from abc import abstractmethod +from importlib.resources import files from pathlib import Path from typing import ( Callable, @@ -15,7 +16,6 @@ Union, cast, ) -from importlib.resources import files import aws_cdk as cdk import constructs @@ -111,10 +111,33 @@ def deploy_stage(self) -> Tuple[List[pipelines.Step], cdk.Stage, List[pipelines. Defaults to None. """ - def decorator_pipeline_stage(func): + def decorator_pipeline_stage( + func: Callable[[PIPELINE_STACK], Union[cdk.Stage, Tuple[cdk.Stage]]] + ) -> Callable[ + [PIPELINE_STACK], + Tuple[Optional[Sequence[pipelines.Step]], cdk.Stage, Optional[Sequence[pipelines.Step]]], + ]: @functools.wraps(func) - def wrapper_pipeline_stage(*args, **kwargs): - return func(*args, **kwargs) + def wrapper_pipeline_stage( + *args, **kwargs + ) -> Tuple[ + Optional[Sequence[pipelines.Step]], cdk.Stage, Optional[Sequence[pipelines.Step]] + ]: + results = func(*args, **kwargs) + if isinstance(results, cdk.Stage): + return None, results, None + assert isinstance(results, tuple) and len(results) == 3 + assert isinstance(results[0], list) or results[0] is None + assert isinstance(results[1], cdk.Stage) + assert isinstance(results[2], list) or results[2] is None + return cast( + Tuple[ + Optional[Sequence[pipelines.Step]], + cdk.Stage, + Optional[Sequence[pipelines.Step]], + ], + results, + ) wrapper_pipeline_stage._pipeline_stage_info = PipelineStageInfo( # type: ignore[attr-defined] order=order, name=name, pre_steps=pre_steps, post_steps=post_steps @@ -215,26 +238,14 @@ def build_pipeline(self): # Add Stages for stage_method in self.get_stage_methods(): - stage_info: PipelineStageInfo = stage_method._pipeline_stage_info # type: ignore[union-attr] - stage = stage_method() - pre_steps = stage_info.pre_steps - post_steps = stage_info.post_steps - if isinstance(stage, cdk.Stage): - stage = stage - elif ( - isinstance(stage, tuple) - and len(stage) == 3 - and isinstance(stage[0], list) - and isinstance(stage[1], cdk.Stage) - and isinstance(stage[2], list) - ): - pre_steps = [*(pre_steps or []), *(cast(List[pipelines.Step], stage[0]))] - post_steps = [*(post_steps or []), *(cast(List[pipelines.Step], stage[2]))] - stage = stage[1] - else: - raise ValueError( - "Stage must be a cdk.Stage or a tuple of pre_steps, stage, post_steps" - ) + stage_info: PipelineStageInfo = stage_method._pipeline_stage_info # type: ignore[attr-defined] + pre_steps, stage, post_steps = stage_method() + + if stage_info.pre_steps is not None: + pre_steps = [*stage_info.pre_steps, *(pre_steps or [])] + if stage_info.post_steps is not None: + post_steps = [*stage_info.post_steps, *(post_steps or [])] + self.pipeline.add_stage(stage, pre=pre_steps, post=post_steps) # Add Promotion Stage @@ -361,7 +372,9 @@ def add_promotion_stage(self, pipeline: pipelines.CodePipeline): # 1. Read the release script file # 2. Base64 encode the file # 3. Decode the base64 encoded file and write it to the release script path - # TODO: i think importlib + # TODO: i think `importlib.resources.files` is preferred way to go here, but + # it requires specifying the package path. This is a bit more + # difficult to do in this context. So we are using the Path approach. f"echo {base64.b64encode((Path(__file__).parent / 'scripts' / 'cicd-release.sh').read_text().encode()).decode()} | base64 --decode > $RELEASE_SCRIPT_PATH" ), # Run the release script @@ -512,12 +525,7 @@ def get_pipeline_source( def get_stage_methods( self, - ) -> List[ - Union[ - Callable[[], cdk.Stage], - Callable[[], Tuple[Sequence[pipelines.Step], cdk.Stage, Sequence[pipelines.Step]]], - ] - ]: + ) -> List[Callable[[], Tuple[Sequence[pipelines.Step], cdk.Stage, Sequence[pipelines.Step]]]]: # Get all methods of the instance methods = [ getattr(self, method_name) @@ -530,8 +538,8 @@ def get_stage_methods( # Sort methods by their order attribute stage_methods.sort(key=lambda method: method._pipeline_stage_info.order) # type: ignore[attr-defined] - # Return the sorted methods + # Return the sorted methods return stage_methods @staticmethod diff --git a/src/aibs_informatics_cdk_lib/project/utils.py b/src/aibs_informatics_cdk_lib/project/utils.py index ce270f0..86f41e5 100644 --- a/src/aibs_informatics_cdk_lib/project/utils.py +++ b/src/aibs_informatics_cdk_lib/project/utils.py @@ -147,6 +147,18 @@ def get_project_config_and_env_base( def get_config( node: constructs.Node, project_config_cls: Type[BaseProjectConfig[G, S]] = ProjectConfig ) -> S: + """ + Retrieves the stage configuration for a given node. + + Args: + node (constructs.Node): The node for which to retrieve the configuration. + project_config_cls (Type[BaseProjectConfig[G, S]], optional): The project configuration class to use. + Defaults to ProjectConfig. + + Returns: + S: The stage configuration object. + + """ project_config, env_base = get_project_config_and_env_base( # type: ignore node, project_config_cls=project_config_cls ) diff --git a/test/aibs_informatics_cdk_lib/project/test_utils.py b/test/aibs_informatics_cdk_lib/project/test_utils.py index 5da56cd..3c6dd63 100644 --- a/test/aibs_informatics_cdk_lib/project/test_utils.py +++ b/test/aibs_informatics_cdk_lib/project/test_utils.py @@ -1,4 +1,5 @@ import os +from unittest import mock import aws_cdk as cdk import constructs @@ -13,7 +14,6 @@ LABEL_KEY, LABEL_KEY_ALIAS, EnvBase, - EnvType, ) from aibs_informatics_cdk_lib.project.utils import ( @@ -111,13 +111,14 @@ def test__get_env_base__context_and_env_vars(env_vars, dummy_node): def test__set_env_base__env_vars_only(env_vars): env_base = EnvBase("dev") - set_env_base(env_base) - assert os.environ.get(ENV_BASE_KEY) == "dev" - assert os.environ.get(ENV_TYPE_KEY) == "dev" - assert os.environ.get(ENV_LABEL_KEY) is None - - env_base = EnvBase("prod-marmot") - set_env_base(env_base) - assert os.environ.get(ENV_BASE_KEY) == "prod-marmot" - assert os.environ.get(ENV_TYPE_KEY) == "prod" - assert os.environ.get(ENV_LABEL_KEY) == "marmot" + with mock.patch.dict(os.environ, clear=True): + set_env_base(env_base) + assert os.environ.get(ENV_BASE_KEY) == "dev" + assert os.environ.get(ENV_TYPE_KEY) == "dev" + assert os.environ.get(ENV_LABEL_KEY) is None + + env_base = EnvBase("prod-marmot") + set_env_base(env_base) + assert os.environ.get(ENV_BASE_KEY) == "prod-marmot" + assert os.environ.get(ENV_TYPE_KEY) == "prod" + assert os.environ.get(ENV_LABEL_KEY) == "marmot" From 0a399331c18bea7b56da584194fa77ca30ff4b92 Mon Sep 17 00:00:00 2001 From: Ryan McGinty Date: Fri, 12 Jul 2024 09:37:07 -0700 Subject: [PATCH 5/6] expose iam_role_name through compute construct --- src/aibs_informatics_cdk_lib/constructs_/service/compute.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/aibs_informatics_cdk_lib/constructs_/service/compute.py b/src/aibs_informatics_cdk_lib/constructs_/service/compute.py index b0bf225..8ac7759 100644 --- a/src/aibs_informatics_cdk_lib/constructs_/service/compute.py +++ b/src/aibs_informatics_cdk_lib/constructs_/service/compute.py @@ -59,6 +59,7 @@ def __init__( buckets: Optional[Iterable[s3.Bucket]] = None, file_systems: Optional[Iterable[Union[efs.FileSystem, efs.IFileSystem]]] = None, mount_point_configs: Optional[Iterable[MountPointConfiguration]] = None, + instance_role_name: Optional[str] = None, instance_role_policy_statements: Optional[List[iam.PolicyStatement]] = None, **kwargs, ) -> None: @@ -69,6 +70,7 @@ def __init__( batch_name, self.env_base, vpc=vpc, + instance_role_name=instance_role_name, instance_role_policy_statements=instance_role_policy_statements, ) From 9ff88cd934e75c055d82f2b964bbc89bf8e27b6d Mon Sep 17 00:00:00 2001 From: Ryan McGinty Date: Fri, 12 Jul 2024 12:45:31 -0700 Subject: [PATCH 6/6] add more documentation --- .../cicd/pipeline/base.py | 5 +++++ .../constructs_/batch/infrastructure.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py index 6cef276..6a8a3c4 100644 --- a/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py +++ b/src/aibs_informatics_cdk_lib/cicd/pipeline/base.py @@ -354,6 +354,11 @@ def add_promotion_stage(self, pipeline: pipelines.CodePipeline): # Create a temporary directory and file to store the release script "export RELEASE_SCRIPT_PATH=$(mktemp -d)/cicd-release.sh", "mkdir -p $(dirname $RELEASE_SCRIPT_PATH)", + # The release script will not be available to us unless we set up + # a virtual environment and install our source package. This is because the + # release script is in a dependent package (aibs-informatics-cdk-lib) and + # is not included in the source package used as input for this step. + # Assuming we want to avoid having to install the package, We have two options here: # TODO: Decide which approach is better (prefer 2) # 1. Download the release script from the source repository (using gh cli) # - This requires the use of the Github CLI diff --git a/src/aibs_informatics_cdk_lib/constructs_/batch/infrastructure.py b/src/aibs_informatics_cdk_lib/constructs_/batch/infrastructure.py index 2a43760..b816d4d 100644 --- a/src/aibs_informatics_cdk_lib/constructs_/batch/infrastructure.py +++ b/src/aibs_informatics_cdk_lib/constructs_/batch/infrastructure.py @@ -65,6 +65,23 @@ def __init__( instance_role_name: Optional[str] = None, instance_role_policy_statements: Optional[List[iam.PolicyStatement]] = None, ) -> None: + """Batch Infrastructure Construct + + Creates the shared infrastructure for Batch Environments. + Has the ability to create multiple Batch Environments with different configurations. + + + Args: + scope (constructs.Construct): scope + id (str): id + env_base (EnvBase): env base to use + vpc (ec2.IVpc): vpc to use + instance_role_name (Optional[str]): Optionally can specify the name of the instance + role created. Defaults to None (will be auto-generated). + instance_role_policy_statements (Optional[List[iam.PolicyStatement]]): Optionally can + specify additional policy statements to add to the instance role + Defaults to None. + """ super().__init__(scope, id, env_base) self.vpc = vpc