Merge pull request #11 from NASA-IMPACT/use-arm64-architecture

Use arm64 architecture
NASA-IMPACT · Jul 10, 2024 · 2279c4f · 2279c4f
2 parents c83c6c3 + 32dc48a
commit 2279c4f
Show file tree

Hide file tree

Showing 14 changed files with 97 additions and 170 deletions.
diff --git a/infrastructure/ecs_services/airflow_metrics.tf b/infrastructure/ecs_services/airflow_metrics.tf
@@ -29,22 +29,9 @@ resource "aws_ecs_task_definition" "airflow_metrics" {
   network_mode       = "awsvpc"
   runtime_platform {
     operating_system_family = "LINUX"
-    cpu_architecture        = "X86_64"
+    cpu_architecture        = "ARM64"
   }
   requires_compatibilities = ["FARGATE"]
-  volume {
-    name = "efs-${var.prefix}"
-    efs_volume_configuration {
-      file_system_id          = aws_efs_file_system.efs.id
-      root_directory          = "/mnt/data"
-      transit_encryption      = "ENABLED"
-      transit_encryption_port = 2999
-      authorization_config {
-        access_point_id = aws_efs_access_point.access.id
-        iam             = "ENABLED"
-      }
-    }
-  }
   container_definitions = jsonencode([
     {
       name      = "metrics"
@@ -55,13 +42,6 @@ resource "aws_ecs_task_definition" "airflow_metrics" {
       entryPoint = [
         "python"
       ]
-      mountPoints : [
-        {
-          "containerPath" : "/opt/airflow/dags_efs",
-          "sourceVolume" : "efs-${var.prefix}"
-
-        }
-      ]
       command = [
         "scripts/put_airflow_worker_autoscaling_metric_data.py",
         "--cluster-name",
@@ -75,7 +55,14 @@ resource "aws_ecs_task_definition" "airflow_metrics" {
         "--period",
         "30"
       ]
-      environment = var.airflow_task_common_environment
+      environment = concat(var.airflow_task_common_environment,
+        [
+      {
+            name  = "SERVICES_HASH"
+            value = join(",", local.services_hashes)
+      }
+
+      ])
       user        = "50000:0"
       logConfiguration = {
         logDriver = "awslogs"
@@ -109,5 +96,6 @@ resource "aws_ecs_service" "airflow_metrics" {
   }
   platform_version     = "1.4.0"
   scheduling_strategy  = "REPLICA"
-  # force_new_deployment = var.force_new_ecs_service_deployment
+  # Update from scripts folder
+  force_new_deployment = var.force_new_ecs_service_deployment
 }
diff --git a/infrastructure/ecs_services/airflow_scheduler.tf b/infrastructure/ecs_services/airflow_scheduler.tf
@@ -45,36 +45,17 @@ resource "aws_ecs_task_definition" "airflow_scheduler" {
   network_mode       = "awsvpc"
   runtime_platform {
     operating_system_family = "LINUX"
-    cpu_architecture        = "X86_64"
+    cpu_architecture        = "ARM64"
   }
   requires_compatibilities = ["FARGATE"]
-  volume {
-    name = "efs-${var.prefix}"
-    efs_volume_configuration {
-      file_system_id          = aws_efs_file_system.efs.id
-      root_directory          = "/mnt/data"
-      transit_encryption      = "ENABLED"
-      transit_encryption_port = 2999
-      authorization_config {
-        access_point_id = aws_efs_access_point.access.id
-        iam             = "ENABLED"
-      }
-    }
-  }
 
   container_definitions = jsonencode([
     {
       name   = "scheduler"
       image  = join(":", [aws_ecr_repository.airflow.repository_url, "latest"])
       cpu    = var.scheduler_cpu
       memory = var.scheduler_memory
-      mountPoints : [
-        {
-          "containerPath" : "/opt/airflow/dags_efs",
-          "sourceVolume" : "efs-${var.prefix}"
 
-        }
-      ]
       healthcheck = {
         command = [
           "CMD-SHELL",
@@ -92,7 +73,14 @@ resource "aws_ecs_task_definition" "airflow_scheduler" {
       linuxParameters = {
         initProcessEnabled = true
       }
-      environment = var.airflow_task_common_environment
+      environment = concat(var.airflow_task_common_environment,
+        [
+      {
+            name  = "SERVICES_HASH"
+            value = join(",", local.services_hashes)
+      }
+
+      ])
       user        = "50000:0"
       logConfiguration = {
         logDriver = "awslogs"
@@ -143,7 +131,6 @@ resource "aws_ecs_service" "airflow_scheduler" {
   # Note: If a revision is not specified, the latest ACTIVE revision is used.
   task_definition = aws_ecs_task_definition.airflow_scheduler.family
   cluster         = aws_ecs_cluster.airflow.arn
-
   deployment_controller {
     type = "ECS"
   }
@@ -159,7 +146,8 @@ resource "aws_ecs_service" "airflow_scheduler" {
   }
   platform_version     = "1.4.0"
   scheduling_strategy  = "REPLICA"
-  # force_new_deployment = var.force_new_ecs_service_deployment
+  # Update from requirements
+  #force_new_deployment = var.force_new_ecs_service_deployment
 }
 
 
diff --git a/infrastructure/ecs_services/airflow_server.tf b/infrastructure/ecs_services/airflow_server.tf
@@ -15,9 +15,7 @@ resource "aws_ecs_task_definition" "airflow_webserver" {
   network_mode       = "awsvpc"
   runtime_platform {
     operating_system_family = "LINUX"
-    # ARM64 currently does not work because of upstream dependencies
-    # https://github.com/apache/airflow/issues/15635
-    cpu_architecture = "X86_64"
+    cpu_architecture = "ARM64"
   }
   requires_compatibilities = ["FARGATE"]
   container_definitions = jsonencode([
@@ -48,7 +46,14 @@ resource "aws_ecs_task_definition" "airflow_webserver" {
       }
       essential   = true
       command     = ["webserver"]
-      environment = var.airflow_task_common_environment
+      environment = concat(var.airflow_task_common_environment,
+        [
+      {
+            name  = "SERVICES_HASH"
+            value = "${local.config_folder_hash},${local.services_build_path_hash}"
+      }
+
+      ])
       user        = "50000:0"
       logConfiguration = {
         logDriver = "awslogs"
@@ -110,7 +115,8 @@ resource "aws_ecs_service" "airflow_webserver" {
     container_name   = "webserver"
     container_port   = 8080
   }
-  # force_new_deployment = var.force_new_ecs_service_deployment
+  # Update from services folder
+  #force_new_deployment = var.force_new_ecs_service_deployment
   # This can be used to update tasks to use a newer container image with same
   # image/tag combination (e.g., myimage:latest)
 }
diff --git a/infrastructure/ecs_services/airflow_standalone_task.tf b/infrastructure/ecs_services/airflow_standalone_task.tf
@@ -31,35 +31,16 @@ resource "aws_ecs_task_definition" "airflow_standalone_task" {
   network_mode       = "awsvpc"
   runtime_platform {
     operating_system_family = "LINUX"
-    cpu_architecture        = "X86_64"
+    cpu_architecture        = "ARM64"
   }
   requires_compatibilities = ["FARGATE"]
-  volume {
-    name = "efs-${var.prefix}"
-    efs_volume_configuration {
-      file_system_id          = aws_efs_file_system.efs.id
-      root_directory          = "/mnt/data"
-      transit_encryption      = "ENABLED"
-      transit_encryption_port = 2999
-      authorization_config {
-        access_point_id = aws_efs_access_point.access.id
-        iam             = "ENABLED"
-      }
-    }
-  }
+
   container_definitions = jsonencode([
     {
       name        = "airflow"
       image       = join(":", [aws_ecr_repository.airflow.repository_url, "latest"])
       cpu         = 256
       memory      = 512
-      mountPoints : [
-        {
-          "containerPath" : "/opt/airflow/dags_efs",
-          "sourceVolume" : "efs-${var.prefix}"
-
-        }
-      ]
       essential   = true
       command     = ["version"]
       environment = var.airflow_task_common_environment

diff --git a/infrastructure/ecs_services/airflow_worker.tf b/infrastructure/ecs_services/airflow_worker.tf
@@ -16,23 +16,9 @@ resource "aws_ecs_task_definition" "airflow_worker" {
   network_mode       = "awsvpc"
   runtime_platform {
     operating_system_family = "LINUX"
-    cpu_architecture        = "X86_64"
+    cpu_architecture        = "ARM64"
   }
   requires_compatibilities = ["FARGATE"]
-  volume {
-    name = "efs-${var.prefix}"
-    efs_volume_configuration {
-      file_system_id          = aws_efs_file_system.efs.id
-      root_directory          = "/mnt/data"
-      transit_encryption      = "ENABLED"
-      transit_encryption_port = 2999
-      authorization_config {
-        access_point_id = aws_efs_access_point.access.id
-        iam             = "ENABLED"
-      }
-    }
-  }
-
   container_definitions = jsonencode([
     {
       name      = "worker"
@@ -41,13 +27,6 @@ resource "aws_ecs_task_definition" "airflow_worker" {
       memory    = var.worker_memory
       essential = true
       command   = var.worker_cmd != [] ? var.worker_cmd : ["celery", "worker"]
-      mountPoints : [
-        {
-          "containerPath" : "/opt/airflow/dags_efs",
-          "sourceVolume" : "efs-${var.prefix}"
-
-        }
-      ]
       linuxParameters = {
         initProcessEnabled = true
       }
@@ -59,6 +38,10 @@ resource "aws_ecs_task_definition" "airflow_worker" {
           {
             name  = "DUMB_INIT_SETSID"
             value = "0"
+          },
+          {
+            name  = "WORKER_HASHES"
+            value = join(",", local.workers_hashes)
           }
         ]
       )
@@ -115,6 +98,7 @@ resource "aws_ecs_service" "airflow_worker" {
     capacity_provider = "FARGATE"
     weight            = 1
   }
+  # Update from workers folder
   # force_new_deployment = var.force_new_ecs_service_deployment
 
 }

diff --git a/infrastructure/ecs_services/certificate.tf b/infrastructure/ecs_services/certificate.tf
@@ -8,6 +8,7 @@ resource "aws_acm_certificate" "ecs-domain-certificate" {
   }
 }
 
+
 data "aws_route53_zone" "ecs_domain" {
   name         = var.domain_name
   private_zone = false

diff --git a/infrastructure/ecs_services/ecr.tf b/infrastructure/ecs_services/ecr.tf
@@ -37,31 +37,19 @@ resource "aws_ecr_lifecycle_policy" "ecr_policy" {
   })
 
 }
-locals {
-
-  build_path        = "../${path.root}/airflow_services"
-  dag_folder_path   = "../${path.root}/dags"
-  scripts_path      = "../${path.root}/scripts"
-  config_path       = "../${path.root}/configuration"
-  worker_build_path = "../${path.root}/airflow_worker"
-}
-
-
 resource "null_resource" "build_ecr_image" {
   triggers = {
-    build_path         = sha1(join("", [for f in fileset(local.build_path, "**") : filesha1("${local.build_path}/${f}")]))
-    scripts_path       = sha1(join("", [for f in fileset(local.scripts_path, "**") : filesha1("${local.scripts_path}/${f}")]))
-    dag_folder_path    = sha1(join("", [for f in fileset(local.dag_folder_path, "**") : filesha1("${local.dag_folder_path}/${f}")]))
-    config_folder_path = sha1(join("", [for f in fileset(local.config_path, "**") : filesha1("${local.config_path}/${f}")]))
-
-
+    services_build_path_hash         = local.services_build_path_hash
+    scripts_folder_hash       = local.scripts_folder_hash
+    dag_folder_hash    = local.dag_folder_hash
+    config_folder_hash = local.config_folder_hash
   }
 
   provisioner "local-exec" {
     command = <<EOF
           cd ../${path.root}
           aws ecr get-login-password --region ${var.aws_region} | docker login --username AWS --password-stdin ${var.account_id}.dkr.ecr.${var.aws_region}.amazonaws.com
-          docker buildx build -t ${aws_ecr_repository.airflow.repository_url}:latest -f airflow_services/Dockerfile --platform linux/amd64 .
+          docker buildx build -t ${aws_ecr_repository.airflow.repository_url}:latest -f airflow_services/Dockerfile .
           docker push ${aws_ecr_repository.airflow.repository_url}:latest
           cd -
        EOF
@@ -71,19 +59,20 @@ resource "null_resource" "build_ecr_image" {
 
 resource "null_resource" "build_worker_ecr_image" {
   triggers = {
-    build_path_worker = sha1(join("", [for f in fileset(local.worker_build_path, "**") : filesha1("${local.worker_build_path}/${f}")]))
-    dag_folder_path   = sha1(join("", [for f in fileset(local.dag_folder_path, "**") : filesha1("${local.dag_folder_path}/${f}")]))
+    worker_folder_hash = local.worker_folder_hash
+    dag_folder_hash    = local.dag_folder_hash
   }
 
   provisioner "local-exec" {
     command = <<EOF
           cd ../${path.root}
           aws ecr get-login-password --region ${var.aws_region} | docker login --username AWS --password-stdin ${var.account_id}.dkr.ecr.${var.aws_region}.amazonaws.com
-          docker buildx build -t ${aws_ecr_repository.worker_airflow.repository_url}:latest -f airflow_worker/Dockerfile --platform linux/amd64 .
+          docker buildx build -t ${aws_ecr_repository.worker_airflow.repository_url}:latest -f airflow_worker/Dockerfile .
           docker push ${aws_ecr_repository.worker_airflow.repository_url}:latest
           cd -
        EOF
   }
 }
 
 
+
diff --git a/infrastructure/ecs_services/efs.tf b/infrastructure/ecs_services/efs.tf