Infrastructure

.env.example

@@ -8,3 +8,5 @@ GOVUK_NOTIFY_API_KEY=insert-test-key-here
 GOVUK_NOTIFY_PLAIN_EMAIL_TEMPLATE_ID=insert-generic-template-id-here
 AWS_REGION="eu-west-2"
 LLM_BACKEND="fake"
+REDIS_HOST="localhost"
+REDIS_PORT=6379

Makefile

@@ -106,6 +106,9 @@ IMAGE_TAG=$$(git rev-parse HEAD)
 AUTO_APPLY_RESOURCES = module.ecs.aws_ecs_task_definition.aws-ecs-task \
                        module.ecs.aws_ecs_service.aws-ecs-service \
                        module.ecs.data.aws_ecs_task_definition.main \
+                       module.worker.aws_ecs_task_definition.aws-ecs-task \
+                       module.worker.aws_ecs_service.aws-ecs-service \
+                       module.worker.data.aws_ecs_task_definition.main \
                        module.batch_job_definition.aws_batch_job_definition.job_definition \
                        module.waf.aws_wafv2_ip_set.london \
                        aws_secretsmanager_secret.django_secret \

consultation_analyser/settings/base.py

@@ -219,13 +219,17 @@
 GIT_SHA = env("GIT_SHA", default=None)
 
 # redis
+redis_host = env.str("REDIS_HOST", "localhost")
+redis_port = env.str("REDIS_PORT", "6379")
+redis_url = f"redis://{redis_host}:{redis_port}"
+
 CACHES = {
     "default": {
         "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
     },
     "redis": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": env.str("REDIS_URL", "redis://localhost:6379/"),
+        "LOCATION": redis_url,
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "MAX_ENTRIES": 5000,

consultation_analyser/settings/production.py

@@ -11,7 +11,7 @@
 
 STORAGES["default"] = {  # noqa: F405
     "BACKEND": "storages.backends.s3.S3Storage",
-    "OPTIONS": {"bucket_name": env("AWS_STORAGE_BUCKET_NAME"), "location": "app_data/"},  # noqa: F405
+    "OPTIONS": {"bucket_name": env("APP_BUCKET"), "location": "app_data/"},  # noqa: F405
 }
 
 

infrastructure/ecs.tf

@@ -13,6 +13,7 @@ locals {
     "DATABASE_URL"                         = local.rds_fqdn,
     "DOMAIN_NAME"                          = "${local.host}"
     "GIT_SHA"                              = var.image_tag
+    "APP_BUCKET"                           = local.secret_env_vars.APP_BUCKET,
   }
 
   batch_env_vars = merge(local.base_env_vars, {
@@ -23,6 +24,8 @@ locals {
     "BATCH_JOB_QUEUE"      = module.batch_job_definition.job_queue_name,
     "BATCH_JOB_DEFINITION" = module.batch_job_definition.job_definition_name,
     "EXECUTION_CONTEXT"    = "ecs"
+    "REDIS_HOST"           = module.elasticache.redis_address,
+    "REDIS_PORT"           = module.elasticache.redis_port,
   })
 
   additional_policy_arns = {for idx, arn in [aws_iam_policy.ecs.arn] : idx => arn}
@@ -61,7 +64,44 @@ module "ecs" {
   additional_execution_role_tags = {
     "RolePassableByRunner" = "True"
   }
+  entrypoint = ["./start.sh"]
+}
 
+module "worker" {
+  source             = "../../i-ai-core-infrastructure//modules/ecs"
+  name               = "${local.name}-worker"
+  image_tag          = var.image_tag
+  ecr_repository_uri = var.ecr_repository_uri
+  ecs_cluster_id     = data.terraform_remote_state.platform.outputs.ecs_cluster_id
+  ecs_cluster_name   = data.terraform_remote_state.platform.outputs.ecs_cluster_name
+  memory             = local.ecs_memory
+  cpu                = local.ecs_cpus
+  health_check = {
+    healthy_threshold   = 3
+    unhealthy_threshold = 3
+    accepted_response   = "200"
+    path                = "/"
+    timeout             = 6
+    port                = 8000
+  }
+  environment_variables = local.ecs_env_vars
+
+  state_bucket                 = var.state_bucket
+  vpc_id                       = data.terraform_remote_state.vpc.outputs.vpc_id
+  private_subnets              = data.terraform_remote_state.vpc.outputs.private_subnets
+  container_port               = "8000"
+  load_balancer_security_group = module.load_balancer.load_balancer_security_group_id
+  aws_lb_arn                   = module.load_balancer.alb_arn
+  host                         = local.host
+  route53_record_name          = aws_route53_record.type_a_record.name
+  ip_whitelist                 = var.external_ips
+  create_listener              = false
+  create_networking            = false
+  task_additional_iam_policies = local.additional_policy_arns
+  additional_execution_role_tags = {
+    "RolePassableByRunner" = "True"
+  }
+  entrypoint = ["./start_worker.sh"]
 }
 
 resource "aws_route53_record" "type_a_record" {

infrastructure/elasticache.tf

@@ -0,0 +1,11 @@
+module "elasticache" {
+  source          = "../../../i-ai-core-infrastructure//modules/elasticache"
+  name            = local.name
+  vpc_id          = data.terraform_remote_state.vpc.outputs.vpc_id
+  private_subnets = data.terraform_remote_state.vpc.outputs.private_subnets
+  security_group_ids = tomap(
+    {
+      "worker"   = module.worker.ecs_sg_id
+    }
+  )
+}

start-worker.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec venv/bin/python3.12 manage.py rqworker default