Make workers CPU and memory configurable

infrastructure/ecs_services/airflow_scheduler.tf

@@ -35,10 +35,11 @@ resource "aws_ssm_parameter" "airflow_ecs_cloudwatch_agent_config" {
   )
 }
 
+
 resource "aws_ecs_task_definition" "airflow_scheduler" {
   family             = "${var.prefix}-scheduler"
-  cpu                = 1024
-  memory             = 2048
+  cpu                = var.scheduler_cpu
+  memory             = var.scheduler_memory
   execution_role_arn = aws_iam_role.ecs_task_execution_role.arn
   task_role_arn      = aws_iam_role.airflow_task.arn
   network_mode       = "awsvpc"
@@ -51,8 +52,8 @@ resource "aws_ecs_task_definition" "airflow_scheduler" {
     {
       name   = "scheduler"
       image  = join(":", [aws_ecr_repository.airflow.repository_url, "latest"])
-      cpu    = 1024
-      memory = 2048
+      cpu    = var.scheduler_cpu
+      memory = var.scheduler_memory
       healthcheck = {
         command = [
           "CMD-SHELL",
@@ -114,6 +115,7 @@ resource "aws_security_group" "airflow_scheduler_service" {
   }
 }
 
+
 resource "aws_ecs_service" "airflow_scheduler" {
   name = "${var.prefix}-scheduler"
   depends_on = [ null_resource.build_ecr_image ,  aws_ecr_repository.airflow ]
@@ -126,7 +128,7 @@ resource "aws_ecs_service" "airflow_scheduler" {
   }
   deployment_maximum_percent         = 200
   deployment_minimum_healthy_percent = 100
-  desired_count                      = 1
+  desired_count                      = var.number_of_schedulers
   lifecycle {
     ignore_changes = [desired_count]
   }

infrastructure/ecs_services/ecr.tf

@@ -73,8 +73,6 @@ resource "null_resource" "build_worker_ecr_image" {
   triggers = {
     build_path_worker = sha1(join("", [for f in fileset(local.worker_build_path, "**") : filesha1("${local.worker_build_path}/${f}")]))
     dag_folder_path       = sha1(join("", [for f in fileset(local.dag_folder_path, "**") : filesha1("${local.dag_folder_path}/${f}")]))
-
-
   }
 
   provisioner "local-exec" {

infrastructure/ecs_services/varaibles.tf

@@ -64,3 +64,16 @@ variable "custom_worker_policy_statement" {
 
 }
 
+variable "number_of_schedulers" {
+   type = number
+}
+
+variable "scheduler_cpu" {
+
+  type = number
+}
+variable "scheduler_memory" {
+  type = number
+
+}
+

infrastructure/main.tf

@@ -74,6 +74,9 @@ module "ecs_services" {
   worker_memory                    = var.worker_memory #4096 * 2
 
   custom_worker_policy_statement = var.custom_worker_policy_statement
+  number_of_schedulers           = var.number_of_schedulers
+  scheduler_memory               = var.scheduler_memory
+  scheduler_cpu                  = var.scheduler_cpu
 }
 
 resource "null_resource" "airflow_create_airflow_user" {

infrastructure/varaibles.tf

@@ -130,4 +130,18 @@ variable "custom_worker_policy_statement" {
   }))
   default = []
 
+}
+
+variable "number_of_schedulers" {
+  type = number
+  default = 1
+}
+
+variable "scheduler_cpu" {
+  type = number
+  default = 1024
+}
+variable "scheduler_memory" {
+  type = number
+  default = 2048
 }