Dtspo 22631 - custom settings per aks cluster on SDS

components/aks/aks.tf

@@ -5,7 +5,7 @@ resource "azurerm_resource_group" "kubernetes_resource_group" {
   name = format("%s-%s-%s-rg",
     var.project,
     var.env,
-    each.value
+    each.key
   )
   tags = module.ctags.common_tags
 }
@@ -15,46 +15,6 @@ module "loganalytics" {
   environment = var.env
 }
 
-locals {
-  linux_node_pool = {
-    name                = "linux"
-    vm_size             = lookup(var.linux_node_pool, "vm_size", "Standard_D4ds_v5")
-    min_count           = lookup(var.linux_node_pool, "min_nodes", 2)
-    max_count           = lookup(var.linux_node_pool, "max_nodes", 10)
-    max_pods            = lookup(var.linux_node_pool, "max_pods", 30)
-    os_type             = "Linux"
-    node_taints         = []
-    enable_auto_scaling = true
-    mode                = "User"
-    availability_zones  = var.availability_zones
-  }
-
-  system_node_pool = {
-    name                = "msnode"
-    vm_size             = lookup(var.windows_node_pool, "vm_size", "Standard_D4ds_v5")
-    min_count           = lookup(var.windows_node_pool, "min_nodes", 2)
-    max_count           = lookup(var.windows_node_pool, "max_nodes", 4)
-    max_pods            = lookup(var.windows_node_pool, "max_pods", 30)
-    os_type             = "Windows"
-    node_taints         = ["kubernetes.io/os=windows:NoSchedule"]
-    enable_auto_scaling = true
-    mode                = "User"
-    availability_zones  = var.availability_zones
-  }
-  cron_job_node_pool = {
-    name                = "cronjob"
-    vm_size             = "Standard_D4ds_v5"
-    min_count           = 0
-    max_count           = 10
-    max_pods            = 30
-    os_type             = "Linux"
-    node_taints         = ["dedicated=jobs:NoSchedule"]
-    enable_auto_scaling = true
-    mode                = "User"
-    availability_zones  = var.availability_zones
-  }
-}
-
 data "azuread_service_principal" "version_checker" {
   display_name = "DTS SDS AKS version checker"
 }
@@ -64,14 +24,14 @@ data "azuread_service_principal" "aks_auto_shutdown" {
 }
 
 module "kubernetes" {
-  for_each    = toset((var.env == "sbox" && var.cluster_automatic) ? [for k, v in var.clusters : k if k == "00"] : [for k, v in var.clusters : k])
+  for_each    = var.env == "sbox" && var.cluster_automatic ? { for k, v in var.clusters : k => v if k == "00" } : var.clusters
   source      = "git::https://github.com/hmcts/aks-module-kubernetes.git?ref=4.x"
   environment = var.env
   location    = var.location
 
   kubelet_uami_enabled = true
   oms_agent_enabled    = var.oms_agent_enabled
-  csi_driver_enabled   = tobool(lookup(var.clusters[each.value], "csi_driver_enabled", true))
+  csi_driver_enabled   = var.csi_driver_enabled
 
   providers = {
     azurerm               = azurerm
@@ -80,13 +40,13 @@ module "kubernetes" {
     azurerm.global_acr    = azurerm.global_acr
   }
 
-  resource_group_name = azurerm_resource_group.kubernetes_resource_group[each.value].name
+  resource_group_name = azurerm_resource_group.kubernetes_resource_group[each.key].name
 
   network_name                = local.network_name
   network_shortname           = local.network_shortname
   network_resource_group_name = local.network_resource_group_name
 
-  cluster_number    = each.value
+  cluster_number    = each.key
   service_shortname = var.service_shortname
   project           = var.project
 
@@ -99,35 +59,92 @@ module "kubernetes" {
 
   control_vault = var.control_vault
 
-  kubernetes_cluster_ssh_key = var.kubernetes_cluster_ssh_key
+  kubernetes_cluster_ssh_key = each.value.kubernetes_cluster_ssh_key
 
-  kubernetes_cluster_agent_min_count = lookup(var.system_node_pool, "min_nodes", 2)
-  kubernetes_cluster_agent_max_count = lookup(var.system_node_pool, "max_nodes", 4)
-  kubernetes_cluster_agent_vm_size   = lookup(var.system_node_pool, "vm_size", "Standard_D4ds_v5")
+  kubernetes_cluster_agent_min_count = lookup(each.value.system_node_pool, "min_nodes", 2)
+  kubernetes_cluster_agent_max_count = lookup(each.value.system_node_pool, "max_nodes", 4)
+  kubernetes_cluster_agent_vm_size   = lookup(each.value.system_node_pool, "vm_size", "Standard_D4ds_v5")
 
-  kubernetes_cluster_version            = var.clusters[each.value]["kubernetes_version"]
+  kubernetes_cluster_version            = each.value.kubernetes_cluster_version
   kubernetes_cluster_agent_os_disk_size = "128"
 
   tags     = module.ctags.common_tags
   sku_tier = var.sku_tier
 
   enable_user_system_nodepool_split = true
 
-  additional_node_pools = contains(["ptlsbox", "ptl"], var.env) ? tolist([local.linux_node_pool, local.cron_job_node_pool]) : tolist([local.linux_node_pool, local.system_node_pool, local.cron_job_node_pool])
-
-  availability_zones = var.availability_zones
+  availability_zones = each.value.availability_zones
 
   aks_version_checker_principal_id = data.azuread_service_principal.version_checker.object_id
 
   aks_role_definition = "Contributor"
 
   aks_auto_shutdown_principal_id = data.azuread_service_principal.aks_auto_shutdown.object_id
 
-  enable_automatic_channel_upgrade_patch = var.enable_automatic_channel_upgrade_patch
+  enable_automatic_channel_upgrade_patch = each.value.enable_automatic_channel_upgrade_patch
 
   enable_node_os_channel_upgrade_nodeimage = true
 
-  node_os_maintenance_window_config = var.node_os_maintenance_window_config
+  node_os_maintenance_window_config = each.value.node_os_maintenance_window_config
+
+  additional_node_pools = contains(["ptlsbox", "ptl"], var.env) ? tolist([
+    {
+      name                = "linux"
+      vm_size             = lookup(each.value.linux_node_pool, "vm_size", "Standard_D4ds_v5")
+      min_count           = lookup(each.value.linux_node_pool, "min_nodes", 2)
+      max_count           = lookup(each.value.linux_node_pool, "max_nodes", 10)
+      max_pods            = lookup(each.value.linux_node_pool, "max_pods", 30)
+      os_type             = "Linux"
+      node_taints         = []
+      enable_auto_scaling = true
+      mode                = "User"
+    },
+    {
+      name                = "cronjob"
+      vm_size             = "Standard_D4ds_v5"
+      min_count           = 0
+      max_count           = 10
+      max_pods            = 30
+      os_type             = "Linux"
+      node_taints         = ["dedicated=jobs:NoSchedule"]
+      enable_auto_scaling = true
+      mode                = "User"
+    }
+    ]) : tolist([
+    {
+      name                = "linux"
+      vm_size             = lookup(each.value.linux_node_pool, "vm_size", "Standard_D4ds_v5")
+      min_count           = lookup(each.value.linux_node_pool, "min_nodes", 2)
+      max_count           = lookup(each.value.linux_node_pool, "max_nodes", 10)
+      max_pods            = lookup(each.value.linux_node_pool, "max_pods", 30)
+      os_type             = "Linux"
+      node_taints         = []
+      enable_auto_scaling = true
+      mode                = "User"
+    },
+    {
+      name                = "msnode"
+      vm_size             = lookup(var.windows_node_pool, "vm_size", "Standard_D4ds_v5")
+      min_count           = lookup(var.windows_node_pool, "min_nodes", 2)
+      max_count           = lookup(var.windows_node_pool, "max_nodes", 4)
+      max_pods            = lookup(var.windows_node_pool, "max_pods", 30)
+      os_type             = "Windows"
+      node_taints         = ["kubernetes.io/os=windows:NoSchedule"]
+      enable_auto_scaling = true
+      mode                = "User"
+    },
+    {
+      name                = "cronjob"
+      vm_size             = "Standard_D4ds_v5"
+      min_count           = 0
+      max_count           = 10
+      max_pods            = 30
+      os_type             = "Linux"
+      node_taints         = ["dedicated=jobs:NoSchedule"]
+      enable_auto_scaling = true
+      mode                = "User"
+    }
+  ])
 }
 
 module "ctags" {
@@ -165,8 +182,7 @@ resource "null_resource" "register_automatic_sku_preview" {
 }
 
 resource "azapi_resource" "managedCluster" {
-
-  count     = var.cluster_automatic ? 1 : 0
+  count     = var.cluster_automatic && var.env == "sbox" ? 1 : 0
   type      = "Microsoft.ContainerService/managedClusters@2024-03-02-preview"
   parent_id = azurerm_resource_group.kubernetes_resource_group["01"].id
   name      = "ss-sbox-01-aks"

components/aks/init.tf

@@ -73,4 +73,4 @@ provider "azurerm" {
   skip_provider_registration = "true"
   features {}
   alias = "dts-ss-stg"
-}
+}

components/aks/inputs-default.tf

@@ -18,16 +18,6 @@ variable "sku_tier" {
   default = "Standard"
 }
 
-variable "system_node_pool" {
-  description = "Map to override the system node pool config"
-  default     = {}
-}
-
-variable "linux_node_pool" {
-  description = "Map to override the linux node pool config"
-  default     = {}
-}
-
 variable "windows_node_pool" {
   description = "Map to override the windows node pool config"
   default     = {}

components/aks/inputs-required.tf

@@ -7,37 +7,97 @@ variable "builtFrom" {}
 
 variable "product" {}
 
-# Remote State
+# # Remote State
 variable "control_vault" {}
 
 # Kubernetes
-variable "kubernetes_cluster_ssh_key" {}
+variable "clusters" {
+  description = <<-EOF
+    Map of clusters to manage. Example:
+    clusters = {
+      "00" = {
+        kubernetes_cluster_version = "1.22.6"
+        kubernetes_cluster_ssh_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCe..."
+        enable_automatic_channel_upgrade_patch = true
 
-variable "kubernetes_cluster_agent_min_count" {
-  default = 1
-}
-variable "kubernetes_cluster_agent_max_count" {
-  default = 3
-}
-variable "kubernetes_cluster_agent_vm_size" {
-  default = "Standard_D4ds_v5"
-}
+        system_node_pool = {
+          vm_size   = "Standard_D4ds_v5"
+          min_nodes = 2
+          max_nodes = 4
+        }
 
-variable "availability_zones" {
-  type = list(any)
-}
+        linux_node_pool = {
+          vm_size   = "Standard_D4ds_v5"
+          min_nodes = 4
+          max_nodes = 10
+        }
 
-variable "clusters" {
-  type        = map(map(string))
-  description = <<-EOF
-Map of clusters to manage. Example:
-clusters = {
-  "00" = {
-    kubernetes_version = "1.22.6"
-  },
-  "01" = {
-    kubernetes_version = "1.22.6"
-  }
+        windows_node_pool = {
+          vm_size   = "Standard_D4ds_v5"
+          min_nodes = 2
+          max_nodes = 4
+        }
+
+        availability_zones = ["1"]
+        autoShutdown       = true
+
+        node_os_maintenance_window_config = {
+          frequency  = "Daily"
+          start_time = "16:00"
+          is_prod    = false
+        }
+      },
+      "01" = {
+        kubernetes_cluster_version = "1.30"
+        kubernetes_cluster_ssh_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCe..."
+        enable_automatic_channel_upgrade_patch = true
+
+        system_node_pool = {
+          min_nodes = 2
+          max_nodes = 4
+        }
+
+        linux_node_pool = {
+          vm_size   = "Standard_D4ds_v5"
+          max_nodes = 10
+          max_nodes = 10
+        }
+
+        availability_zones = ["1"]
+        autoShutdown       = true
+
+        node_os_maintenance_window_config = {
+          frequency  = "Daily"
+          start_time = "16:00"
+          is_prod    = false
+        }
+      }
+    }
+  EOF
+  type = map(object({
+    kubernetes_cluster_version             = string
+    kubernetes_cluster_ssh_key             = string
+    enable_automatic_channel_upgrade_patch = optional(bool, false)
+    system_node_pool = object({
+      min_nodes = number
+      max_nodes = number
+    })
+
+    linux_node_pool = object({
+      max_nodes = number
+      max_pods  = number
+    })
+
+    node_os_maintenance_window_config = object({
+      frequency  = string
+      start_time = string
+      is_prod    = bool
+    })
+
+    availability_zones = list(string)
+  }))
 }
-EOF
+
+variable "csi_driver_enabled" {
+  default = true
 }