Test/end to end infra (#45)

* cleaning up code for iam permissions * add: release command * rm: ref to * rm: option to be able to release to prod * test: build * add: docker_echo * edit: pull main * edit: pull main * add: image-tag * tweak * tweak --------- Co-authored-by: mohit <[email protected]>
i-dot-ai · Mar 26, 2024 · 2975a2c · 2975a2c
1 parent 5586478
commit 2975a2c
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 45 deletions.
diff --git a/.github/workflows/build-gh.yml b/.github/workflows/build-gh.yml
@@ -30,7 +30,7 @@ jobs:
           echo "RUNNER_LABEL=self-hosted" >> $GITHUB_OUTPUT
 
   start-runner:
-        uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/start-runner.yml@feature/apply-terraform
+        uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/start-runner.yml@main
         needs: set-vars
         with:
             EC2_INSTANCE_TYPE: ${{ needs.set-vars.outputs.ec2-instance-type }}
@@ -45,7 +45,7 @@ jobs:
     needs:
       - set-vars
       - start-runner
-    uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/build-docker.yml@feature/apply-terraform
+    uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/build-docker.yml@main
     with:
       APP_NAME: ${{ needs.set-vars.outputs.app-name }}
       RUNNER_LABEL: ${{ needs.set-vars.outputs.runner_label }}
@@ -58,7 +58,7 @@ jobs:
 
 
   stop-runner:
-        uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/stop-runner.yml@feature/apply-terraform
+        uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/stop-runner.yml@main
         if: needs.start-runner.outputs.use-persisted == 0 && always()
         needs: 
             - set-vars

diff --git a/.github/workflows/build-infra.yml → .github/workflows/release-infra.yml b/.github/workflows/build-infra.yml → .github/workflows/release-infra.yml
@@ -2,6 +2,10 @@ name: build-infrastructure
 
 on:
   push:
+    tags:
+      - release-dev-**
+      - release-preprod-**
+
     branches:
       - main
 
@@ -33,8 +37,20 @@ jobs:
           echo "environment=${ENVIRONMENT}" >> $GITHUB_OUTPUT
           echo "::add-mask::${AWS_REGION}"
 
+      - id: env-var
+        run: |
+          # prod env from main branch
+          if [ ${{ github.ref == 'refs/heads/main' }} ]; then
+            export ENVIRONMENT=prod
+          # Otherwise get the environment from the tag
+          else
+            export ENVIRONMENT=$(echo ${{ github.ref_name }} | awk -F- '{ print $2 }')
+          fi
+          echo "environment=${ENVIRONMENT}" >> $GITHUB_OUTPUT
+
+
   start-runner:
-        uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/start-runner.yml@feature/apply-terraform
+        uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/start-runner.yml@main
         needs: set-vars
         with:
             EC2_INSTANCE_TYPE: ${{ needs.set-vars.outputs.ec2-instance-type }}
@@ -50,7 +66,7 @@ jobs:
     needs:
       - set-vars
       - start-runner
-    uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/apply-terraform.yml@feature/apply-terraform
+    uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/apply-terraform.yml@main
     with:
       APP_NAME: ${{ needs.set-vars.outputs.app-name }}
       RUNNER_LABEL: ${{ needs.start-runner.outputs.label }}
@@ -67,7 +83,7 @@ jobs:
       - set-vars
       - start-runner
       - build-infra
-    uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/stop-runner.yml@feature/apply-terraform
+    uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/stop-runner.yml@main
     if: needs.start-runner.outputs.use-persisted == 0 && always() 
     with:
       RUNNER_LABEL: ${{ needs.start-runner.outputs.label }}

diff --git a/Makefile b/Makefile
@@ -58,24 +58,29 @@ ECR_REPO_NAME=$(APP_NAME)
 IMAGE_TAG=$$(git rev-parse HEAD)
 tf_build_args=-var "image_tag=$(IMAGE_TAG)"
 
-.PHONY: docker/login
-docker/login:
+.PHONY: docker_login
+docker_login:
 	aws ecr get-login-password --region $(AWS_REGION) | docker login --username AWS --password-stdin $(ECR_URL)
 
 .PHONY: docker_build
-docker/build:
+docker_build:
 	cd frontend && \
 	docker build -t $(ECR_REPO_URL):$(IMAGE_TAG) .
 
-.PHONY: docker/push
-docker/push:
+.PHONY: docker_push
+docker_push:
 	docker push $(IMAGE)
 
 .PHONY: docker_update_tag
 docker_update_tag:
 	MANIFEST=$$(aws ecr batch-get-image --repository-name $(ECR_REPO_NAME) --image-ids imageTag=$(IMAGE_TAG) --query 'images[].imageManifest' --output text) && \
 	aws ecr put-image --repository-name $(ECR_REPO_NAME) --image-tag $(tag) --image-manifest "$$MANIFEST"
 
+# Ouputs the value that you're after - useful to get a value i.e. IMAGE_TAG out of the Makefile
+.PHONY: docker_echo
+docker_echo:
+	echo $($(value))
+
 
 CONFIG_DIR=../../consultation-analyser-infra-config
 TF_BACKEND_CONFIG=$(CONFIG_DIR)/backend.hcl
@@ -107,4 +112,10 @@ tf_apply: ## Apply terraform
 .PHONY: tf_destroy
 tf_destroy: ## Destroy terraform
 	make tf_set_workspace && \
-	terraform -chdir=./infrastructure destroy -var-file=$(CONFIG_DIR)/${env}-input-params.tfvars ${tf_build_args}
+	terraform -chdir=./infrastructure destroy -var-file=$(CONFIG_DIR)/${env}-input-params.tfvars ${tf_build_args}
+
+
+# Release commands to deploy your app to AWS
+.PHONY: release
+release: ## Deploy app
+	chmod +x ./infrastructure/scripts/release.sh && ./infrastructure/scripts/release.sh $(env)
diff --git a/infrastructure/batch.tf b/infrastructure/batch.tf
@@ -21,25 +21,6 @@ module "batch_job_defintiion" {
   region                  = var.region
   compute_environment_arn = [module.batch_compute.ec2_compute_environment_arn]
   state_bucket            = var.state_bucket
-  image                   = module.ecr_pre_processing.ecr_repository_url
+  image                   = module.ecr.ecr_repository_url
   fargate_flag            = false
 }
-
-# The ECS task will need to invoke SageMaker endpoint
-resource "aws_iam_policy" "sagemaker_invoke" {
-  name        = "SageMakerInvokePolicy"
-  description = "Policy that allows ECS tasks to invoke SageMaker endpoints"
-
-  policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Action": "sagemaker:InvokeEndpoint",
-      "Effect": "Allow",
-      "Resource": "*"
-    }
-  ]
-}
-EOF
-}
diff --git a/infrastructure/ecr.tf b/infrastructure/ecr.tf
@@ -1,13 +1,6 @@
 
-module "ecr_front_end" {
-  source = "../../i-ai-core-infrastructure//modules/ecr"
-
-  project_name_prefix = var.project_name
-  container_name      = "front-end"
-}
-
-module "ecr_pre_processing" {
-  source              = "../../i-ai-core-infrastructure//modules/ecr"
-  project_name_prefix = var.project_name
-  container_name      = "pre-processing"
+module "ecr" {
+  source                = "../../i-ai-core-infrastructure//modules/ecr"
+  project_name          = var.project_name
+  container_name_suffix = null
 }
diff --git a/infrastructure/ecs.tf b/infrastructure/ecs.tf
@@ -1,9 +1,9 @@
 module "ecs" {
   source             = "../../i-ai-core-infrastructure//modules/ecs"
   project_name       = var.project_name
-  image_tag          = "dfcde6a235a58d4e708e53c9c2252477516c414b"
+  image_tag          = var.image_tag
   prefix             = "i-dot-ai"
-  ecr_repository_uri = module.ecr_front_end.ecr_repository_url
+  ecr_repository_uri = module.ecr.ecr_repository_url
   ecs_cluster_id     = data.terraform_remote_state.platform.outputs.ecs_cluster_id
   health_check = {
     healthy_threshold   = 3

diff --git a/infrastructure/provider.tf b/infrastructure/provider.tf
@@ -10,8 +10,18 @@ terraform {
   backend "s3" {
     key = "consultation-analyser/terraform.tfstate"
   }
+
 }
 
 provider "random" {
 
 }
+
+provider "aws" {
+  default_tags {
+    tags = {
+      Environment = terraform.workspace
+      Deployed    = "github"
+    }
+  }
+}
diff --git a/infrastructure/scripts/release.sh b/infrastructure/scripts/release.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Usage:
+## Just pass the name of the env we want to tag and deploy.
+## This will create a tag locally with a format of $$ENV-$BRANCH-$CURRENT_USER-$TIMESTAMP 
+## Then push it to the remote git.
+ENV=$1
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
+CURRENT_USER=$(whoami)
+TIMESTAMP=$(date +%d-%m-%y--%H%M%S)
+TAG_NAME="release-$ENV-$BRANCH-$CURRENT_USER-$TIMESTAMP"
+
+echo "Current branch name is" "$BRANCH"
+echo "Current environment name is" "$ENV"
+echo "Timestamp assigned will be $TIMESTAMP"
+echo "New tag name will be " "$TAG_NAME"
+
+if [ $ENV == 'prod' ]; then
+    echo ''
+    if [ $BRANCH != 'main' ]; then
+        echo -e "\033[0;31mYou can only deploy to prod through a PR into main\033[0m"
+        exit 0
+    fi
+fi
+
+##
+echo "Removing Local tags"
+git tag -d $(git tag -l)
+
+# Command to run
+echo "Applying local tag" && \
+git tag "$TAG_NAME" && \
+echo "Pushing tag" && \
+git push origin $TAG_NAME