[DOP-2325][DEV-12391][DEV-12396][DOP-2328] EKS cluster module update, IAM module introduction

aws_specific_modules.tf

@@ -111,7 +111,6 @@ resource "aws_eks_addon" "guardduty" {
   ]
   count = var.eks_addon_version_guardduty != null ? 1 : 0
 
-
   cluster_name      = var.label
   addon_name        = "aws-guardduty-agent"
   addon_version     = "v1.7.1-eksbuild.1"

ipa.tf

@@ -795,7 +795,7 @@ crunchy-postgres:
         repo1-retention-full: '5'
         repo1-s3-key-type: auto
         repo1-s3-kms-key-id: "${module.kms_key.key_arn}"
-        repo1-s3-role: ${module.cluster.s3_role_id}
+        repo1-s3-role: ${module.iam.node_role_name}
       repos:
       - name: repo1
         s3:
@@ -869,7 +869,7 @@ crunchy-postgres:
         repo1-retention-full: '5'
         repo1-s3-key-type: auto
         repo1-s3-kms-key-id: "${module.kms_key.key_arn}"
-        repo1-s3-role: ${module.cluster.s3_role_id}
+        repo1-s3-role: ${module.iam.node_role_name}
       repos:
       - name: repo1
         s3:

main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "5.68.0"
+      version = "5.74.0"
     }
     time = {
       source  = "hashicorp/time"
@@ -100,24 +100,13 @@ data "aws_caller_identity" "current" {}
 # define the networking module we're using locally
 locals {
   network = var.network_module == "public_networking" ? module.public_networking : module.networking
-  aws_usernames = [
-    "svc_jenkins",
-    "terraform-sa"
-  ]
-  eks_users = {
-    for user in local.aws_usernames : user => {
-      userarn  = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:user/${user}"
-      username = user
-      groups   = ["system:masters"]
-    }
-  }
 
   argo_app_name           = lower("${var.aws_account}.${var.region}.${var.label}-ipa")
   argo_smoketest_app_name = lower("${var.aws_account}.${var.region}.${var.label}-smoketest")
   argo_cluster_name       = "${var.aws_account}.${var.region}.${var.label}"
-  
+
   chart_version_parts = split("-", var.ipa_version)
-  chart_suffix = trimprefix(var.ipa_version, local.chart_version_parts[0])
+  chart_suffix        = trimprefix(var.ipa_version, local.chart_version_parts[0])
 }
 
 resource "tls_private_key" "pk" {
@@ -167,12 +156,16 @@ module "networking" {
 }
 
 module "sqs_sns" {
-  count             = var.sqs_sns == true ? 1 : 0
-  source            = "app.terraform.io/indico/indico-aws-sqs-sns/mod"
-  version           = "1.2.0"
-  region            = var.region
-  label             = var.label
-  kms_master_key_id = module.kms_key.key.id
+  count                      = var.sqs_sns == true ? 1 : 0
+  source                     = "app.terraform.io/indico/indico-aws-sqs-sns/mod"
+  version                    = "2.0.0"
+  region                     = var.region
+  label                      = var.label
+  kms_master_key_id          = module.kms_key.key.id
+  sqs_sns_type               = var.sqs_sns_type
+  ipa_sns_topic_name         = var.ipa_sns_topic_name
+  ipa_sqs_queue_name         = var.ipa_sqs_queue_name
+  indico_sqs_sns_policy_name = var.indico_sqs_sns_policy_name
 }
 
 module "lambda-sns-forwarder" {
@@ -211,18 +204,21 @@ module "security-group" {
   network_module = var.network_module
 }
 
-
 module "s3-storage" {
-  source                = "app.terraform.io/indico/indico-aws-buckets/mod"
-  version               = "3.3.1"
-  force_destroy         = true # allows terraform to destroy non-empty buckets.
-  label                 = var.label
-  kms_key_arn           = module.kms_key.key.arn
-  submission_expiry     = var.submission_expiry
-  uploads_expiry        = var.uploads_expiry
-  include_rox           = var.include_rox
-  enable_backup         = var.enable_s3_backup
-  enable_access_logging = var.enable_s3_access_logging
+  source                             = "app.terraform.io/indico/indico-aws-buckets/mod"
+  version                            = "4.1.1"
+  force_destroy                      = true # allows terraform to destroy non-empty buckets.
+  label                              = var.label
+  kms_key_arn                        = module.kms_key.key.arn
+  submission_expiry                  = var.submission_expiry
+  uploads_expiry                     = var.uploads_expiry
+  include_rox                        = var.include_rox
+  enable_backup                      = var.enable_s3_backup
+  enable_access_logging              = var.enable_s3_access_logging
+  bucket_type                        = var.bucket_type
+  data_s3_bucket_name_override       = var.data_s3_bucket_name_override
+  api_models_s3_bucket_name_override = var.api_models_s3_bucket_name_override
+  pgbackup_s3_bucket_name_override   = var.pgbackup_s3_bucket_name_override
 }
 
 
@@ -310,39 +306,57 @@ module "fsx-storage" {
   fsx_rox_arn                 = var.fsx_rox_arn
 }
 
+module "iam" {
+  source  = "app.terraform.io/indico/indico-aws-iam/mod"
+  version = "0.0.9"
+
+  # EKS node role
+  create_node_role           = var.create_node_role
+  eks_cluster_nodes_iam_role = var.node_role_name_override == null ? "${var.label}-${var.region}-node-role" : var.node_role_name_override
+  label                      = var.label
+  region                     = var.region
+  cluster_node_policies      = var.cluster_node_policies
+  aws_primary_dns_role_arn   = var.aws_primary_dns_role_arn
+  efs_filesystem_id          = [var.include_efs == true ? module.efs-storage[0].efs_filesystem_id : ""]
+  fsx_arns                   = [var.include_rox ? module.fsx-storage[0].fsx-rox.arn : "", var.include_fsx == true ? module.fsx-storage[0].fsx-rwx.arn : ""]
+  s3_buckets                 = compact([module.s3-storage.data_s3_bucket_name, var.include_pgbackup ? module.s3-storage.pgbackup_s3_bucket_name : "", var.include_rox ? module.s3-storage.api_models_s3_bucket_name : "", lower("${var.aws_account}-aws-cod-snapshots"), var.performance_bucket ? "indico-locust-benchmark-test-results" : ""])
+  kms_key_arn                = module.kms_key.key_arn
+  # s3 replication
+  enable_s3_replication                            = var.enable_s3_replication
+  create_s3_replication_role                       = var.create_s3_replication_role
+  s3_replication_role_name                         = var.s3_replication_role_name_override == null ? "s3-bucket-replication-${var.label}" : var.s3_replication_role_name_override
+  s3_replication_destination_kms_key_arn           = var.destination_kms_key_arn
+  s3_replication_data_destination_bucket_name      = var.data_destination_bucket
+  s3_replication_api_model_destination_bucket_name = var.api_model_destination_bucket
+}
+
 module "cluster" {
-  cod_snapshots_enabled                 = true
-  allow_dns_management                  = true
-  aws_account_name                      = var.aws_account
-  oidc_enabled                          = false
-  source                                = "app.terraform.io/indico/indico-aws-eks-cluster/mod"
-  version                               = "8.2.3"
-  label                                 = var.label
-  additional_tags                       = var.additional_tags
-  region                                = var.region
-  map_users                             = values(local.eks_users)
-  vpc_id                                = local.network[0].indico_vpc_id
-  security_group_id                     = var.network_module == "networking" ? local.network[0].all_subnets_sg_id : module.security-group.all_subnets_sg_id
+  source          = "app.terraform.io/indico/indico-aws-eks-cluster/mod"
+  version         = "9.0.31"
+  label           = var.label
+  region          = var.region
+  cluster_version = var.k8s_version
+  default_tags    = merge(coalesce(var.default_tags, {}), coalesce(var.additional_tags, {}))
+
+  kms_key_arn = module.kms_key.key_arn
+
+  vpc_id     = local.network[0].indico_vpc_id
+  az_count   = var.az_count
+  subnet_ids = flatten([local.network[0].private_subnet_ids])
+
+  node_groups          = var.node_groups
+  node_role_name       = module.iam.node_role_name
+  node_role_arn        = module.iam.node_role_arn
+  instance_volume_size = var.instance_volume_size
+  instance_volume_type = var.instance_volume_type
+
+  additional_users = var.additional_users
+
+  public_endpoint_enabled  = var.cluster_api_endpoint_public == true ? true : false
+  private_endpoint_enabled = var.network_allow_public == true ? false : true
+
+  cluster_security_group_id             = var.network_module == "networking" ? local.network[0].all_subnets_sg_id : module.security-group.all_subnets_sg_id
   cluster_additional_security_group_ids = var.network_module == "networking" ? [local.network[0].all_subnets_sg_id] : []
-  subnet_ids                            = flatten([local.network[0].private_subnet_ids])
-  node_groups                           = var.node_groups
-  cluster_node_policies                 = var.cluster_node_policies
-  eks_cluster_iam_role                  = var.eks_cluster_iam_role
-  eks_cluster_nodes_iam_role            = "${var.label}-${var.region}-node-role"
-  fsx_arns                              = [var.include_rox ? module.fsx-storage[0].fsx_rox_arn : "", var.include_fsx == true ? module.fsx-storage[0].fsx_rwx_arn : ""]
-  kms_key_arn                           = module.kms_key.key_arn
-  az_count                              = var.az_count
-  key_pair                              = aws_key_pair.kp.key_name
-  snapshot_id                           = var.snapshot_id
-  default_tags                          = var.default_tags
-  s3_buckets                            = [module.s3-storage.data_s3_bucket_name, var.include_pgbackup ? module.s3-storage.pgbackup_s3_bucket_name : "", var.include_rox ? module.s3-storage.api_models_s3_bucket_name : "", lower("${var.aws_account}-aws-cod-snapshots"), var.performance_bucket ? "indico-locust-benchmark-test-results" : ""]
-  cluster_version                       = var.k8s_version
-  efs_filesystem_id                     = [var.include_efs == true ? module.efs-storage[0].efs_filesystem_id : ""]
-  aws_primary_dns_role_arn              = var.aws_primary_dns_role_arn
-  private_endpoint_enabled              = var.network_allow_public == true ? false : true
-  public_endpoint_enabled               = var.cluster_api_endpoint_public == true ? true : false
-  instance_volume_size                  = var.instance_volume_size
-  instance_volume_type                  = var.instance_volume_type
 }
 
 locals {
@@ -379,13 +393,13 @@ provider "argocd" {
 }
 
 data "aws_eks_cluster" "local" {
-  depends_on = [ module.cluster.kubernetes_host ]
-  name     = var.label
+  depends_on = [module.cluster.kubernetes_host]
+  name       = var.label
 }
 
 data "aws_eks_cluster_auth" "local" {
-  depends_on = [ module.cluster.kubernetes_host ]
-  name     = var.label
+  depends_on = [module.cluster.kubernetes_host]
+  name       = var.label
 }
 
 provider "kubernetes" {
@@ -462,7 +476,7 @@ module "argo-registration" {
   }
 
   source                       = "app.terraform.io/indico/indico-argo-registration/mod"
-  version                      = "1.2.2"
+  version                      = "1.3.0"
   cluster_name                 = var.label
   region                       = var.region
   argo_password                = var.argo_password

moved.tf

@@ -0,0 +1,14 @@
+moved {
+  from = module.cluster.aws_iam_policy.cluster_node_ebs_iam_policy
+  to   = module.iam.module.create_eks_node_role[0].aws_iam_policy.policies[0]
+}
+
+moved {
+  from = module.cluster.aws_iam_policy.cluster_node_iam_policy
+  to   = module.iam.module.create_eks_node_role[0].aws_iam_policy.policies[1]
+}
+
+moved {
+  from = module.cluster.aws_iam_role.node_role
+  to   = module.iam.module.create_eks_node_role[0].aws_iam_role.role
+}

on_prem_test.tf

@@ -95,7 +95,7 @@ YAML
 data "local_file" "nfs_ip" {
   count      = var.on_prem_test == true ? 1 : 0
   filename   = "${path.module}/nfs_server_ip.txt"
-  depends_on = ["null_resource.get_nfs_server_ip"]
+  depends_on = [null_resource.get_nfs_server_ip]
 }
 
 
@@ -183,4 +183,4 @@ resource "null_resource" "update_storage_class" {
   provisioner "local-exec" {
     command = "./kubectl patch storageclass nfs-client -p '{\"metadata\": {\"annotations\":{\"storageclass.kubernetes.io/is-default-class\":\"true\"}}}'"
   }
-}
+}

outputs.tf

@@ -9,12 +9,11 @@ output "data_s3_bucket_name" {
   value       = module.s3-storage.data_s3_bucket_name
 }
 
-output "s3_role_id" {
-  description = "ID of the S3 role"
-  value       = module.cluster.s3_role_id
+output "pgbackup_s3_bucket_name" {
+  description = "Name of the pgbackup s3 bucket"
+  value       = module.s3-storage.pgbackup_s3_bucket_name
 }
 
-
 output "efs_filesystem_id" {
   description = "ID of the EFS filesystem"
   value       = var.include_efs == true ? module.efs-storage[0].efs_filesystem_id : ""

tf-smoketest-variables.tf

@@ -173,6 +173,32 @@ resource "kubernetes_config_map" "terraform-variables" {
     vpc_flow_logs_iam_role_arn = "${jsonencode(var.vpc_flow_logs_iam_role_arn)}"
     instance_volume_size = "${jsonencode(var.instance_volume_size)}"
     instance_volume_type = "${jsonencode(var.instance_volume_type)}"
+    additional_users = "${jsonencode(var.additional_users)}"
+    aws_account_name = "${jsonencode(var.aws_account_name)}"
+    access_key = "${jsonencode(var.access_key)}"
+    secret_key = "${jsonencode(var.secret_key)}"
+    cluster_type = "${jsonencode(var.cluster_type)}"
+    argo_bcrypt_password = "${jsonencode(var.argo_bcrypt_password)}"
+    harbor_admin_password = "${jsonencode(var.harbor_admin_password)}"
+    azure_indico_io_client_secret_id = "${jsonencode(var.azure_indico_io_client_secret_id)}"
+    az_readapi_subscription_id = "${jsonencode(var.az_readapi_subscription_id)}"
+    az_readapi_client_id = "${jsonencode(var.az_readapi_client_id)}"
+    az_readapi_client_secret_id = "${jsonencode(var.az_readapi_client_secret_id)}"
+    aws_account_ids = "${jsonencode(var.aws_account_ids)}"
+    ipa_smoketest_cronjob_enabled = "${jsonencode(var.ipa_smoketest_cronjob_enabled)}"
+    local_registry_harbor_robot_account_name = "${jsonencode(var.local_registry_harbor_robot_account_name)}"
+    bucket_type = "${jsonencode(var.bucket_type)}"
+    data_s3_bucket_name_override = "${jsonencode(var.data_s3_bucket_name_override)}"
+    api_models_s3_bucket_name_override = "${jsonencode(var.api_models_s3_bucket_name_override)}"
+    pgbackup_s3_bucket_name_override = "${jsonencode(var.pgbackup_s3_bucket_name_override)}"
+    enable_s3_replication = "${jsonencode(var.enable_s3_replication)}"
+    create_s3_replication_role = "${jsonencode(var.create_s3_replication_role)}"
+    s3_replication_role_name_override = "${jsonencode(var.s3_replication_role_name_override)}"
+    destination_kms_key_arn = "${jsonencode(var.destination_kms_key_arn)}"
+    data_destination_bucket = "${jsonencode(var.data_destination_bucket)}"
+    api_model_destination_bucket = "${jsonencode(var.api_model_destination_bucket)}"
+    create_node_role = "${jsonencode(var.create_node_role)}"
+    node_role_name_override = "${jsonencode(var.node_role_name_override)}"
     fsx_deployment_type = "${jsonencode(var.fsx_deployment_type)}"
     fsx_type = "${jsonencode(var.fsx_type)}"
     fsx_rwx_id = "${jsonencode(var.fsx_rwx_id)}"