Merge pull request #14 from AllenInstitute/feature/update-cicd-pipeli…

…ne-stack-base-class

feature/update cicd pipeline stack base class

pyproject.toml

@@ -33,7 +33,10 @@ dev = [
 version = {attr = "aibs_informatics_cdk_lib._version.__version__"}
 
 [tool.setuptools.package-data]
-"*" = ['py.typed']
+"*" = [
+    'py.typed',
+    '*.sh',
+]
 
 [tool.setuptools.packages.find]
 where = ["src"]

src/aibs_informatics_cdk_lib/cicd/pipeline/scripts/cicd-release.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+#################################################################
+#                   CI/CD Release Script
+#  Description:
+#   Purpose of this script is to facilitate submit Pull Requests 
+#   from a source branch/commit to a destination branch. 
+#    
+# Input Environment Variables:
+#   
+#   CICD_RELEASE_SOURCE_ENV_TYPE:
+#       Environment Type of source branch
+#   CICD_RELEASE_TARGET_ENV_TYPE: 
+#       Environment Type of source branch
+#   CICD_RELEASE_TARGET_BRANCH:
+#       Target branch to submit pull request into
+#   CICD_RELEASE_REVIEWER:
+#       Reviewers for the PR
+
+###################################
+
+
+export CICD_RELEASE_SOURCE_COMMIT=$CODEBUILD_RESOLVED_SOURCE_VERSION
+export CICD_RELEASE_CANDIDATE_BRANCH="candidate/$CICD_RELEASE_TARGET_BRANCH"
+
+echo "==> CI/CD Release Inputs:"
+echo "==> CICD_RELEASE_SOURCE_ENV_TYPE = $CICD_RELEASE_SOURCE_ENV_TYPE"
+echo "==> CICD_RELEASE_TARGET_ENV_TYPE = $CICD_RELEASE_TARGET_ENV_TYPE"
+echo "==> CICD_RELEASE_SOURCE_COMMIT = $CICD_RELEASE_SOURCE_COMMIT"
+echo "==> CICD_RELEASE_CANDIDATE_BRANCH = $CICD_RELEASE_CANDIDATE_BRANCH"
+echo "==> CICD_RELEASE_TARGET_BRANCH = $CICD_RELEASE_TARGET_BRANCH"
+echo "==> CICD_RELEASE_REVIEWER = $CICD_RELEASE_REVIEWER"
+
+export CICD_RELEASE_GIT_MESSAGE="$(git log -1 --pretty=%B)"
+export CICD_RELEASE_GIT_AUTHOR="$(git log -1 --pretty=%an)"
+export CICD_RELEASE_GIT_AUTHOR_EMAIL="$(git log -1 --pretty=%ae)"
+export CICD_RELEASE_GIT_COMMIT="$(git log -1 --pretty=%H)"
+export CICD_RELEASE_GIT_SHORT_COMMIT="$(git log -1 --pretty=%h)"
+
+echo "==> CICD_RELEASE_GIT_MESSAGE = $CICD_RELEASE_GIT_MESSAGE"
+echo "==> CICD_RELEASE_GIT_AUTHOR = $CICD_RELEASE_GIT_AUTHOR"
+echo "==> CICD_RELEASE_GIT_AUTHOR_EMAIL = $CICD_RELEASE_GIT_AUTHOR_EMAIL"
+echo "==> CICD_RELEASE_GIT_COMMIT = $CICD_RELEASE_GIT_COMMIT"
+echo "==> CICD_RELEASE_GIT_SHORT_COMMIT = $CICD_RELEASE_GIT_SHORT_COMMIT"
+echo 
+
+
+echo "Verify gh command is on PATH"
+
+if ! command -v gh &> /dev/null; then
+    echo "==! Could not find gh command on PATH. EXITING"
+    exit 1
+fi
+
+echo 
+echo "==> Promoting commits up to $CICD_RELEASE_GIT_SHORT_COMMIT to release candidate branch."
+echo "==> Release candidate branch: $CICD_RELEASE_CANDIDATE_BRANCH"
+
+echo "[command] git checkout -B $CICD_RELEASE_CANDIDATE_BRANCH $CICD_RELEASE_SOURCE_COMMIT"
+git checkout -B $CICD_RELEASE_CANDIDATE_BRANCH $CICD_RELEASE_SOURCE_COMMIT
+echo "[command] git push --set-upstream --force"
+git push --set-upstream --force origin $CICD_RELEASE_CANDIDATE_BRANCH
+
+CICD_RELEASE_DATE=$(date '+%Y-%m-%d')
+CICD_RELEASE_PR_TITLE="Release $CICD_RELEASE_SOURCE_ENV_TYPE -> $CICD_RELEASE_TARGET_ENV_TYPE ($CICD_RELEASE_DATE)"
+
+CICD_RELEASE_PR_MESSAGE_FILE=$(mktemp)
+
+
+cat <<EOF > $CICD_RELEASE_PR_MESSAGE_FILE
+# Release
+## Release Summary
+| Release Attribute | Value |
+| --- | --- |
+| Target Branch | $CICD_RELEASE_TARGET_BRANCH |
+| Source Branch | $CICD_RELEASE_CANDIDATE_BRANCH ($CICD_RELEASE_GIT_SHORT_COMMIT) |
+| Date          | $(date '+%Y-%m-%d %H:%M:%S') |
+
+## Release Notes
+
+This release includes changes up to $CICD_RELEASE_GIT_SHORT_COMMIT. This includes the following:
+- (fill me please)
+- (fill me please)
+- (fill me please)
+
+## Checklist
+- [ ] All of GCS works impeccably
+
+EOF
+
+
+echo "==> Checking for open Pull Requests..."
+EXISTING_PR_NUMBER=$(gh pr list -B $CICD_RELEASE_TARGET_BRANCH -L 1 | cut -f1)
+
+if [[ ! -z $EXISTING_PR_NUMBER ]]; then
+    echo "==> Pull Request already exists ($EXISTING_PR_NUMBER). Updating..."
+
+    # Update the PR message
+    echo "" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE 
+    echo "---" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE 
+    echo "# Previous Revisions" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE
+    echo "---" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE
+    echo "" | cat >> $CICD_RELEASE_PR_MESSAGE_FILE
+    gh pr view --json body | jq -r '.body' >> $CICD_RELEASE_PR_MESSAGE_FILE
+
+    gh pr edit $EXISTING_PR_NUMBER \
+        --title "$CICD_RELEASE_PR_TITLE" \
+        --body-file $CICD_RELEASE_PR_MESSAGE_FILE
+
+else
+
+    echo "==> Creating new Pull Request"
+
+    gh pr create \
+        --base $CICD_RELEASE_TARGET_BRANCH \
+        --title "$CICD_RELEASE_PR_TITLE" \
+        --body-file "$CICD_RELEASE_PR_MESSAGE_FILE" \
+        --reviewer "$CICD_RELEASE_REVIEWER"
+fi

src/aibs_informatics_cdk_lib/cicd/target.py

@@ -1,6 +1,79 @@
 from enum import Enum
+from typing import Optional, Type, TypeVar, Union
 
+import constructs
+from aibs_informatics_core.utils.os_operations import get_env_var
 
-class CDKStackTarget(str, Enum):
-    PIPELINE = "pipeline"
-    INFRA = "infra"
+from aibs_informatics_cdk_lib.project.utils import _get_from_context
+
+CDK_STACK_TARGET_ENV_VAR = "CDK_STACK_TARGET"
+
+T = TypeVar("T", bound="CDKStackTargetBaseEnum")
+
+
+class CDKStackTargetBaseEnum(Enum):
+    """Base class for CDK stack target types
+
+    Usage:
+        class MyCDKStackTarget(str, CDKStackTargetBaseEnum):
+            INFRA = "pipeline"
+
+    """
+
+    @classmethod
+    def from_env(cls: Type[T], default: Union[str, T]) -> T:
+        target = get_env_var(CDK_STACK_TARGET_ENV_VAR)
+        target = target or default
+        return cls(target)
+
+    @classmethod
+    def from_context(
+        cls: Type[T],
+        node: constructs.Node,
+        default: Union[str, T],
+        context_keys: Optional[list[str]] = None,
+    ) -> T:
+        """Resolves the CDK stack target type from context
+
+        Args:
+            cls (Type[T]): subclassed CDKStackTargetBase
+            node (constructs.Node): cdk construct node
+            default (str): default to use.
+            context_keys (Optional[list[str]], optional): overrides for context names.
+                Defaults to None.
+
+        Returns:
+            T: CDKStackTargetBase instance
+        """
+        context_keys = context_keys or ["target", "stack_target"]
+
+        target = _get_from_context(node, context_keys) or default
+
+        return cls(target)
+
+    @classmethod
+    def from_context_or_env(
+        cls: Type[T],
+        node: constructs.Node,
+        default: Union[str, T],
+        context_keys: Optional[list[str]] = None,
+    ) -> T:
+        """Resolves the CDK stack target type from context or environment
+
+        Order of resolution:
+            1. CDK context value (specifying -c K=V)
+            2. env variable
+            3. default value ("dev")
+
+        Args:
+            cls (Type[T]): subclassed CDKStackTargetBase
+            node (constructs.Node): cdk construct node
+            default (str): default to use.
+            context_keys (Optional[list[str]], optional): overrides for context names.
+                Defaults to None.
+
+        """
+
+        return cls.from_context(
+            node=node, default=cls.from_env(default), context_keys=context_keys
+        )

src/aibs_informatics_cdk_lib/constructs_/batch/infrastructure.py

@@ -65,6 +65,23 @@ def __init__(
         instance_role_name: Optional[str] = None,
         instance_role_policy_statements: Optional[List[iam.PolicyStatement]] = None,
     ) -> None:
+        """Batch Infrastructure Construct
+
+        Creates the shared infrastructure for Batch Environments.
+        Has the ability to create multiple Batch Environments with different configurations.
+
+
+        Args:
+            scope (constructs.Construct): scope
+            id (str): id
+            env_base (EnvBase): env base to use
+            vpc (ec2.IVpc): vpc to use
+            instance_role_name (Optional[str]): Optionally can specify the name of the instance
+                role created. Defaults to None (will be auto-generated).
+            instance_role_policy_statements (Optional[List[iam.PolicyStatement]]): Optionally can
+                specify additional policy statements to add to the instance role
+                Defaults to None.
+        """
         super().__init__(scope, id, env_base)
         self.vpc = vpc
 

src/aibs_informatics_cdk_lib/constructs_/service/compute.py

@@ -59,6 +59,7 @@ def __init__(
         buckets: Optional[Iterable[s3.Bucket]] = None,
         file_systems: Optional[Iterable[Union[efs.FileSystem, efs.IFileSystem]]] = None,
         mount_point_configs: Optional[Iterable[MountPointConfiguration]] = None,
+        instance_role_name: Optional[str] = None,
         instance_role_policy_statements: Optional[List[iam.PolicyStatement]] = None,
         **kwargs,
     ) -> None:
@@ -69,6 +70,7 @@ def __init__(
             batch_name,
             self.env_base,
             vpc=vpc,
+            instance_role_name=instance_role_name,
             instance_role_policy_statements=instance_role_policy_statements,
         )
 

src/aibs_informatics_cdk_lib/project/config.py

@@ -85,8 +85,8 @@ class CodePipelineBuildConfig(BaseModel):
 class CodePipelineSourceConfig(BaseModel):
     repository: str
     branch: Annotated[str, PlainValidator(EnvVarStr.validate)]
-    codestar_connection: Optional[UniqueIDType]
-    oauth_secret_name: Optional[str]
+    codestar_connection: Optional[UniqueIDType] = None
+    oauth_secret_name: Optional[str] = None
 
     @model_validator(mode="after")
     @classmethod

src/aibs_informatics_cdk_lib/project/utils.py

@@ -6,8 +6,9 @@
 ]
 
 import logging
+import os
 import pathlib
-from typing import List, Optional, Type, Union
+from typing import List, Optional, Tuple, Type, Union
 
 import constructs
 from aibs_informatics_core.env import (
@@ -21,7 +22,7 @@
 )
 from aibs_informatics_core.utils.os_operations import get_env_var, set_env_var
 
-from aibs_informatics_cdk_lib.project.config import BaseProjectConfig, G, ProjectConfig, S
+from aibs_informatics_cdk_lib.project.config import BaseProjectConfig, G, P, ProjectConfig, S
 
 logger = logging.getLogger(__name__)
 
@@ -120,19 +121,53 @@ def get_env_base(node: constructs.Node) -> EnvBase:
         return EnvBase.from_type_and_label(env_type=env_type, env_label=env_label)
 
 
-def get_config(
-    node: constructs.Node, project_config_cls: Type[BaseProjectConfig[G, S]] = ProjectConfig
-) -> S:
-    env_base = get_env_base(node)
+def set_env_base(env_base: EnvBase) -> None:
+    """Set the environment base
 
+    Args:
+        env_base (EnvBase): environment base
+    """
     set_env_var(EnvBase.ENV_BASE_KEY, env_base)
     set_env_var(EnvBase.ENV_TYPE_KEY, env_base.env_type)
     if env_base.env_label:
         set_env_var(EnvBase.ENV_LABEL_KEY, env_base.env_label)
+    else:
+        os.environ.pop(EnvBase.ENV_LABEL_KEY, None)
+
+
+def get_project_config_and_env_base(
+    node: constructs.Node, project_config_cls: Type[P] = ProjectConfig
+) -> Tuple[P, EnvBase]:
+    env_base = get_env_base(node)
+
+    config = project_config_cls.load_config()
+    return config, env_base
+
+
+def get_config(
+    node: constructs.Node, project_config_cls: Type[BaseProjectConfig[G, S]] = ProjectConfig
+) -> S:
+    """
+    Retrieves the stage configuration for a given node.
+
+    Args:
+        node (constructs.Node): The node for which to retrieve the configuration.
+        project_config_cls (Type[BaseProjectConfig[G, S]], optional): The project configuration class to use.
+            Defaults to ProjectConfig.
+
+    Returns:
+        S: The stage configuration object.
+
+    """
+    project_config, env_base = get_project_config_and_env_base(  # type: ignore
+        node, project_config_cls=project_config_cls
+    )
+    set_env_base(env_base)
+
+    stage_config: S = project_config.get_stage_config(env_type=env_base.env_type)
+    stage_config.env.label = env_base.env_label
 
-    config: S = project_config_cls.load_stage_config(env_type=env_base.env_type)
-    config.env.label = env_base.env_label
-    return config
+    return stage_config
 
 
 def _get_from_context(

test/aibs_informatics_cdk_lib/project/test_utils.py

@@ -1,3 +1,6 @@
+import os
+from unittest import mock
+
 import aws_cdk as cdk
 import constructs
 import pytest
@@ -11,14 +14,14 @@
     LABEL_KEY,
     LABEL_KEY_ALIAS,
     EnvBase,
-    EnvType,
 )
 
 from aibs_informatics_cdk_lib.project.utils import (
     ENV_BASE_KEYS,
     ENV_LABEL_KEYS,
     ENV_TYPE_KEYS,
     get_env_base,
+    set_env_base,
 )
 
 USER = "marmotdev"
@@ -104,3 +107,18 @@ def test__get_env_base__context_and_env_vars(env_vars, dummy_node):
     # Base from context supercedes type/label
     dummy_node.set_context(ENV_BASE_KEY, "dev")
     assert get_env_base(dummy_node) == EnvBase("dev")
+
+
+def test__set_env_base__env_vars_only(env_vars):
+    env_base = EnvBase("dev")
+    with mock.patch.dict(os.environ, clear=True):
+        set_env_base(env_base)
+        assert os.environ.get(ENV_BASE_KEY) == "dev"
+        assert os.environ.get(ENV_TYPE_KEY) == "dev"
+        assert os.environ.get(ENV_LABEL_KEY) is None
+
+        env_base = EnvBase("prod-marmot")
+        set_env_base(env_base)
+        assert os.environ.get(ENV_BASE_KEY) == "prod-marmot"
+        assert os.environ.get(ENV_TYPE_KEY) == "prod"
+        assert os.environ.get(ENV_LABEL_KEY) == "marmot"