[DEV-12746] Change method for acquiring token for kubernetes connecti…

…on (#369) * [DEV-12746] Bump kubernetes provider to latest version. * [DEV-12746] Change the way the token is procured for the kubernetes provider. * [DEV-12746] Add dependency on cluster to prevent gp2 storage class from failing to apply. * [DEV-12746] Add a short sleep after the cluster creates to prevent race conditions. Dependency added to all objects which depend on the module.cluster
IndicoDataSolutions · Dec 20, 2024 · 759d27c · 759d27c
1 parent e5d63ea
commit 759d27c
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 26 deletions.
diff --git a/aws_specific_modules.tf b/aws_specific_modules.tf
@@ -107,7 +107,8 @@ data "aws_iam_policy_document" "eks_vpc_guardduty" {
 
 resource "aws_eks_addon" "guardduty" {
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
   count = var.eks_addon_version_guardduty != null ? 1 : 0
 

diff --git a/harness.tf b/harness.tf
@@ -2,7 +2,8 @@ module "harness_delegate" {
   count = var.harness_delegate && strcontains(lower(var.aws_account), "indico") ? 1 : 0
 
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   source = "./modules/harness"

diff --git a/ipa.tf b/ipa.tf
@@ -246,7 +246,8 @@ EOT
 }
 resource "kubernetes_secret" "issuer-secret" {
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   metadata {
@@ -269,7 +270,8 @@ resource "kubernetes_secret" "issuer-secret" {
 #TODO: move to prereqs
 resource "kubernetes_secret" "harbor-pull-secret" {
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   metadata {
@@ -349,7 +351,8 @@ data "github_repository_file" "data-pre-reqs-values" {
 
 module "secrets-operator-setup" {
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
   count           = var.secrets_operator_enabled == true ? 1 : 0
   source          = "./modules/common/vault-secrets-operator-setup"
@@ -366,7 +369,8 @@ resource "helm_release" "ipa-vso" {
   depends_on = [
     module.cluster,
     data.github_repository_file.data-crds-values,
-    module.secrets-operator-setup
+    module.secrets-operator-setup,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   verify           = false
@@ -432,7 +436,8 @@ resource "helm_release" "external-secrets" {
   depends_on = [
     module.cluster,
     data.github_repository_file.data-crds-values,
-    module.secrets-operator-setup
+    module.secrets-operator-setup,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
 
@@ -466,7 +471,8 @@ resource "helm_release" "ipa-crds" {
   depends_on = [
     module.cluster,
     data.github_repository_file.data-crds-values,
-    module.secrets-operator-setup
+    module.secrets-operator-setup,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   verify           = false
@@ -608,6 +614,10 @@ EOT
 }
 
 resource "kubectl_manifest" "gp2-storageclass" {
+  depends_on = [
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
+  ]
   yaml_body = <<YAML
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
@@ -665,7 +675,8 @@ resource "helm_release" "ipa-pre-requisites" {
     helm_release.ipa-crds,
     data.vault_kv_secret_v2.zerossl_data,
     data.github_repository_file.data-pre-reqs-values,
-    null_resource.update_storage_class
+    null_resource.update_storage_class,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   verify           = false
@@ -1192,7 +1203,8 @@ resource "helm_release" "local-registry" {
     kubernetes_namespace.local-registry,
     time_sleep.wait_1_minutes_after_pre_reqs,
     module.cluster,
-    kubernetes_persistent_volume_claim.local-registry
+    kubernetes_persistent_volume_claim.local-registry,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   count = var.local_registry_enabled == true ? 1 : 0
@@ -1406,7 +1418,8 @@ resource "github_repository_file" "alb-values-yaml" {
   }
   depends_on = [
     module.cluster,
-    aws_acm_certificate_validation.alb[0]
+    aws_acm_certificate_validation.alb[0],
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   content = local.alb_ipa_values
@@ -1427,7 +1440,8 @@ resource "github_repository_file" "argocd-application-yaml" {
   }
   depends_on = [
     module.cluster,
-    aws_wafv2_web_acl.wafv2-acl[0]
+    aws_wafv2_web_acl.wafv2-acl[0],
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   content = <<EOT

diff --git a/main.tf b/main.tf
@@ -18,7 +18,7 @@ terraform {
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
-      version = ">= 2.33.0"
+      version = ">= 2.35.0"
     }
     kubectl = {
       source  = "gavinbunney/kubectl"
@@ -375,6 +375,12 @@ module "cluster" {
   cluster_additional_security_group_ids = var.network_module == "networking" ? [local.network[0].all_subnets_sg_id] : []
 }
 
+resource "time_sleep" "wait_1_minutes_after_cluster" {
+  depends_on = [module.cluster]
+
+  create_duration = "1m"
+}
+
 locals {
   readapi_secret_path = var.environment == "production" ? "prod-readapi" : "dev-readapi"
 }
@@ -386,7 +392,10 @@ data "vault_kv_secret_v2" "readapi_secret" {
 
 resource "kubernetes_secret" "readapi" {
   count      = var.enable_readapi ? 1 : 0
-  depends_on = [module.cluster]
+  depends_on = [
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
+  ]
   metadata {
     name = "readapi-secret"
   }
@@ -421,12 +430,13 @@ data "aws_eks_cluster_auth" "local" {
 provider "kubernetes" {
   host                   = module.cluster.kubernetes_host
   cluster_ca_certificate = module.cluster.kubernetes_cluster_ca_certificate
+  token                  = data.aws_eks_cluster_auth.local.token
   #token                  = module.cluster.kubernetes_token
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    args        = ["eks", "get-token", "--cluster-name", var.label]
-    command     = "aws"
-  }
+  # exec {
+  #   api_version = "client.authentication.k8s.io/v1beta1"
+  #   args        = ["eks", "get-token", "--cluster-name", var.label]
+  #   command     = "aws"
+  # }
 }
 
 provider "kubectl" {
@@ -483,7 +493,8 @@ module "argo-registration" {
   count = var.argo_enabled == true ? 1 : 0
 
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   providers = {

diff --git a/oidc.tf b/oidc.tf
@@ -12,7 +12,8 @@ resource "null_resource" "enable-oidc" {
 
 resource "kubernetes_cluster_role_binding" "cod-role-bindings" {
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   count = var.oidc_enabled == true && strcontains(lower(var.aws_account), "indico-") ? 1 : 0
@@ -48,7 +49,8 @@ resource "kubernetes_cluster_role_binding" "cod-role-bindings" {
 
 resource "kubernetes_cluster_role_binding" "eng-qa-rbac-bindings" {
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   count = var.oidc_enabled == true && strcontains(lower(var.aws_account), "indico-") ? 1 : 0
@@ -78,7 +80,8 @@ resource "kubernetes_cluster_role_binding" "eng-qa-rbac-bindings" {
 
 resource "kubernetes_cluster_role_binding" "devops-rbac-bindings" {
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   count = var.oidc_enabled == true ? 1 : 0

diff --git a/on_prem_test.tf b/on_prem_test.tf
@@ -7,7 +7,8 @@
 
 resource "kubectl_manifest" "nfs_volume" {
   depends_on = [
-    module.cluster
+    module.cluster,
+    time_sleep.wait_1_minutes_after_cluster
   ]
   count     = var.on_prem_test == true ? 1 : 0
   yaml_body = <<YAML
@@ -103,7 +104,8 @@ resource "null_resource" "get_nfs_server_ip" {
   count = var.on_prem_test == true ? 1 : 0
   depends_on = [
     module.cluster,
-    kubectl_manifest.nfs_server_service
+    kubectl_manifest.nfs_server_service,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   triggers = {
@@ -140,7 +142,8 @@ resource "helm_release" "nfs-provider" {
   namespace  = "default"
   depends_on = [
     module.cluster,
-    kubectl_manifest.nfs_server_service
+    kubectl_manifest.nfs_server_service,
+    time_sleep.wait_1_minutes_after_cluster
   ]
 
   # // prometheus URL

diff --git a/tf-smoketest-variables.tf b/tf-smoketest-variables.tf
@@ -173,6 +173,10 @@ resource "kubernetes_config_map" "terraform-variables" {
     vpc_flow_logs_iam_role_arn = "${jsonencode(var.vpc_flow_logs_iam_role_arn)}"
     instance_volume_size = "${jsonencode(var.instance_volume_size)}"
     instance_volume_type = "${jsonencode(var.instance_volume_type)}"
+    sqs_sns_type = "${jsonencode(var.sqs_sns_type)}"
+    ipa_sns_topic_name = "${jsonencode(var.ipa_sns_topic_name)}"
+    ipa_sqs_queue_name = "${jsonencode(var.ipa_sqs_queue_name)}"
+    indico_sqs_sns_policy_name = "${jsonencode(var.indico_sqs_sns_policy_name)}"
     additional_users = "${jsonencode(var.additional_users)}"
     aws_account_name = "${jsonencode(var.aws_account_name)}"
     access_key = "${jsonencode(var.access_key)}"