Merge branch 'feature/integrate_actions' into add/ecs-terraform

i-dot-ai · Mar 25, 2024 · 4d15f7c · 4d15f7c
2 parents 11f259a + 0160f38
commit 4d15f7c
Show file tree

Hide file tree

Showing 9 changed files with 212 additions and 12 deletions.
diff --git a/.github/.github.env b/.github/.github.env
@@ -0,0 +1,10 @@
+# github specific
+EC2_INSTANCE_TYPE=t3.large
+ENVIRONMENT=dev
+RUNNER_LABEL=TEST1
+# App related
+APP_NAME=consultations
+
+REPO=consultation-analyser
+INFRA_CONFIG_REPO=consultation-analyser-infra-config
+STATIC_PATH=./static
diff --git a/.github/workflows/build-gh.yml b/.github/workflows/build-gh.yml
@@ -0,0 +1,74 @@
+
+name: build
+on:
+  push:
+    branches:
+      - "**"
+
+
+jobs:
+  set-vars:
+    runs-on: ubuntu-latest
+    outputs:
+      app-name: ${{ steps.export.outputs.app-name }}
+      ec2-instance-type: ${{ steps.export.outputs.ec2-instance-type }}
+      version: ${{ steps.export.outputs.version }}
+      runner_label: ${{ steps.export.outputs.runner_label }}
+      environment: ${{ steps.export.outputs.environment }}
+
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - id: export
+        run: |
+          . ./.github/.github.env
+          echo "ec2-instance-type=${EC2_INSTANCE_TYPE}" >> $GITHUB_OUTPUT
+          echo "app-name=${APP_NAME}" >> $GITHUB_OUTPUT
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          echo "environment=${ENVIRONMENT}" >> $GITHUB_OUTPUT
+          echo "RUNNER_LABEL=self-hosted" >> $GITHUB_OUTPUT
+
+  start-runner:
+        uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/start-runner.yml@feature/apply-terraform
+        needs: set-vars
+        with:
+            EC2_INSTANCE_TYPE: ${{ needs.set-vars.outputs.ec2-instance-type }}
+            ENVIRONMENT: ${{ needs.set-vars.outputs.environment }}
+        secrets:
+            AWS_GITHUBRUNNER_USER_ACCESS_KEY: ${{ secrets.AWS_GITHUBRUNNER_USER_ACCESS_KEY }}
+            AWS_GITHUBRUNNER_USER_SECRET_ID: ${{ secrets.AWS_GITHUBRUNNER_USER_SECRET_ID }}
+            AWS_GITHUBRUNNER_PAT: ${{ secrets.AWS_GITHUBRUNNER_PAT}}
+            AWS_REGION: ${{ secrets.AWS_REGION }}
+
+  build-image:
+    needs:
+      - set-vars
+      - start-runner
+    uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/build-docker.yml@feature/apply-terraform
+    with:
+      APP_NAME: ${{ needs.set-vars.outputs.app-name }}
+      RUNNER_LABEL: ${{ needs.set-vars.outputs.runner_label }}
+      INFRASTRUCTURE_FOLDER: "infrastructure"
+    secrets:
+      AWS_GITHUBRUNNER_PAT: ${{ secrets.AWS_GITHUBRUNNER_PAT}}
+      AWS_GITHUBRUNNER_PAT_USER: ${{ secrets.AWS_GITHUBRUNNER_PAT_USER }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+
+
+  stop-runner:
+        uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/stop-runner.yml@feature/apply-terraform
+        if: needs.start-runner.outputs.use-persisted == 0 && always()
+        needs: 
+            - set-vars
+            - start-runner
+            - build-image
+        with:
+            RUNNER_LABEL: ${{ needs.start-runner.outputs.label }}
+            EC2_INSTANCE_ID: ${{ needs.start-runner.outputs.ec2-instance-id }}
+        secrets:
+            AWS_GITHUBRUNNER_USER_ACCESS_KEY: ${{ secrets.AWS_GITHUBRUNNER_USER_ACCESS_KEY }}
+            AWS_GITHUBRUNNER_USER_SECRET_ID: ${{ secrets.AWS_GITHUBRUNNER_USER_SECRET_ID }}
+            AWS_GITHUBRUNNER_PAT: ${{ secrets.AWS_GITHUBRUNNER_PAT}}
+            AWS_REGION: ${{ secrets.AWS_REGION }}
diff --git a/.github/workflows/build-infra.yml b/.github/workflows/build-infra.yml
@@ -0,0 +1,79 @@
+name: build-infrastructure
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  set-vars:
+    runs-on: ubuntu-latest
+    outputs:
+      app-name: ${{ steps.export.outputs.app-name }}
+      ec2-instance-type: ${{ steps.export.outputs.ec2-instance-type }}
+      version: ${{ steps.export.outputs.version }}
+      repo: ${{ steps.export.outputs.repo }}
+      infra_config_repo: ${{ steps.export.outputs.infra-config-repo }}
+      environment: ${{ steps.export.outputs.environment }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - id: export
+        run: |
+          . ./.github/.github.env
+          echo "ec2-instance-type=${EC2_INSTANCE_TYPE}" >> $GITHUB_OUTPUT
+          echo "app-name=${APP_NAME}" >> $GITHUB_OUTPUT
+          echo "aws-region=${AWS_REGION}" >> $GITHUB_OUTPUT
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          echo "data-path=${DATA_PATH}" >> $GITHUB_OUTPUT
+          echo "repo=${REPO}" >> $GITHUB_OUTPUT
+          echo "infra-config-repo=${INFRA_CONFIG_REPO}" >> $GITHUB_OUTPUT
+          echo "environment=${ENVIRONMENT}" >> $GITHUB_OUTPUT
+          echo "::add-mask::${AWS_REGION}"
+
+  start-runner:
+        uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/start-runner.yml@feature/apply-terraform
+        needs: set-vars
+        with:
+            EC2_INSTANCE_TYPE: ${{ needs.set-vars.outputs.ec2-instance-type }}
+            ENVIRONMENT: ${{ needs.set-vars.outputs.environment }}
+        secrets:
+            AWS_GITHUBRUNNER_USER_ACCESS_KEY: ${{ secrets.AWS_GITHUBRUNNER_USER_ACCESS_KEY }}
+            AWS_GITHUBRUNNER_USER_SECRET_ID: ${{ secrets.AWS_GITHUBRUNNER_USER_SECRET_ID }}
+            AWS_GITHUBRUNNER_PAT: ${{ secrets.AWS_GITHUBRUNNER_PAT}}
+            AWS_REGION: ${{ secrets.AWS_REGION }}
+
+
+  build-infra:
+    needs:
+      - set-vars
+      - start-runner
+    uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/apply-terraform.yml@feature/apply-terraform
+    with:
+      APP_NAME: ${{ needs.set-vars.outputs.app-name }}
+      RUNNER_LABEL: ${{ needs.start-runner.outputs.label }}
+      ENVIRONMENT: ${{ needs.set-vars.outputs.environment }}
+      PUBLIC_INFRA_DEPLOYMENT: true
+      INFRA_CONFIG_REPO: ${{ needs.set-vars.outputs.infra_config_repo }}
+    secrets:
+      AWS_GITHUBRUNNER_PAT: ${{ secrets.AWS_GITHUBRUNNER_PAT }}
+      AWS_GITHUBRUNNER_PAT_USER: ${{ secrets.AWS_GITHUBRUNNER_PAT_USER }} 
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+
+  stop-runner:
+    needs:
+      - set-vars
+      - start-runner
+      - build-infra
+    uses: i-dot-ai/i-dot-ai-core-github-actions/.github/workflows/stop-runner.yml@feature/apply-terraform
+    if: needs.start-runner.outputs.use-persisted == 0 && always() 
+    with:
+      RUNNER_LABEL: ${{ needs.start-runner.outputs.label }}
+      EC2_INSTANCE_ID: ${{ needs.start-runner.outputs.ec2-instance-id }}
+    secrets:
+      AWS_GITHUBRUNNER_USER_ACCESS_KEY: ${{ secrets.AWS_GITHUBRUNNER_USER_ACCESS_KEY }}
+      AWS_GITHUBRUNNER_USER_SECRET_ID: ${{ secrets.AWS_GITHUBRUNNER_USER_SECRET_ID }}
+      AWS_GITHUBRUNNER_PAT: ${{ secrets.AWS_GITHUBRUNNER_PAT }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
diff --git a/Makefile b/Makefile
@@ -58,17 +58,17 @@ ECR_REPO_NAME=$(APP_NAME)
 IMAGE_TAG=$$(git rev-parse HEAD)
 tf_build_args=-var "image_tag=$(IMAGE_TAG)"
 
-.PHONY: docker_login
-docker_login:
+.PHONY: docker/login
+docker/login:
 	aws ecr get-login-password --region $(AWS_REGION) | docker login --username AWS --password-stdin $(ECR_URL)
 
 .PHONY: docker_build
-docker_build:
+docker/build:
 	cd frontend && \
 	docker build -t $(ECR_REPO_URL):$(IMAGE_TAG) .
 
-.PHONY: docker_push
-docker_push:
+.PHONY: docker/push
+docker/push:
 	docker push $(IMAGE)
 
 .PHONY: docker_update_tag
@@ -103,6 +103,7 @@ tf_apply: ## Apply terraform
 	make tf_set_workspace && \
 	terraform -chdir=./infrastructure apply -var-file=$(CONFIG_DIR)/${env}-input-params.tfvars ${tf_build_args}
 
+
 .PHONY: tf_destroy
 tf_destroy: ## Destroy terraform
 	make tf_set_workspace && \

diff --git a/infrastructure/batch.tf b/infrastructure/batch.tf
@@ -0,0 +1,26 @@
+
+
+module "batch_compute" {
+  source          = "../../i-ai-core-infrastructure/modules/batch/batch_compute_environment/"
+  project         = "i-dot-ai"
+  name            = "consultations"
+  region          = var.region
+  vpc_id          = data.terraform_remote_state.vpc.outputs.vpc_id
+  desired_vcpus   = 0
+  min_vcpus       = 0
+  max_vcpus       = 20
+  private_subnets = data.terraform_remote_state.vpc.outputs.private_subnets
+  state_bucket    = var.state_bucket
+
+}
+
+module "batch_job_defintiion" {
+  source                  = "../../i-ai-core-infrastructure/modules/batch/batch_job_definitons/"
+  project                 = "i-dot-ai"
+  name                    = "consultations"
+  region                  = var.region
+  compute_environment_arn = module.batch_compute.compute_environment_arn
+  state_bucket            = var.state_bucket
+  image                   = module.ecr_back_end.repository_url
+  fargate_flag            = false
+}
diff --git a/infrastructure/data.tf b/infrastructure/data.tf
@@ -1,6 +1,10 @@
 locals {
-  record_prefix = var.env == "prod" ? var.project_name : "${var.project_name}-${var.env}"
-  host          = terraform.workspace == "prod" ? "${var.project_name}.ai.cabinetoffice.gov.uk" : "${var.project_name}-${terraform.workspace}.ai.cabinetoffice.gov.uk"
+  record_prefix           = var.env == "prod" ? var.project_name : "${var.project_name}-${var.env}"
+  host                    = terraform.workspace == "prod" ? "${var.project_name}.ai.cabinetoffice.gov.uk" : "${var.project_name}-${terraform.workspace}.ai.cabinetoffice.gov.uk"
+  image_tag_frontend_test = "a07a0d03cfd1aeebf6baed86b7dccf4d79504405"
+
+  image_frontend = 
+
 }
 
 data "terraform_remote_state" "vpc" {
@@ -44,3 +48,5 @@ data "terraform_remote_state" "account" {
     region = var.region
   }
 }
+
+
diff --git a/infrastructure/ecr.tf b/infrastructure/ecr.tf
@@ -7,8 +7,7 @@ module "ecr_front_end" {
 }
 
 module "ecr_back_end" {
-  source = "../../i-ai-core-infrastructure//modules/ecr"
-
+  source              = "../../i-ai-core-infrastructure//modules/ecr"
   project_name_prefix = var.project_name
-  container_name      = "back-end"
+  container_name      = "pre-processing"
 }
diff --git a/infrastructure/ecs.tf b/infrastructure/ecs.tf
@@ -3,8 +3,8 @@ module "ecs" {
   project_name       = var.project_name
   image_tag          = "dfcde6a235a58d4e708e53c9c2252477516c414b"
   prefix             = "i-dot-ai"
-  ecr_repository_uri = var.ecr_repository_uri
-  ecs_cluster_id     = data.terraform_remote_state.platform.outputs.ecs_cluster_id
+  ecr_repository_uri = module.ecr_front_end.repository_url
+  cluster_name       = data.terraform_remote_state.platform.outputs.ecs_cluster_id
   health_check = {
     healthy_threshold   = 3
     unhealthy_threshold = 3
@@ -28,6 +28,7 @@ module "ecs" {
     user_pool_client_id : module.cognito.user_pool_client_id,
     user_pool_domain : module.cognito.user_pool_domain
   }
+
 }
 
 

diff --git a/infrastructure/main.tf b/infrastructure/main.tf
@@ -3,3 +3,7 @@ terraform {
     key = "consultation-analyser/terraform.tfstate"
   }
 }
+
+provider "random" {
+
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,3 +3,7 @@ terraform { @@
         key = "consultation-analyser/terraform.tfstate"
       }
     }
+    provider "random" {
+    }