feat: Adding Apache pinot support (awslabs#353)

distributed-databases/pinot/addons.tf

@@ -0,0 +1,203 @@
+#---------------------------------------------------------------
+# IRSA for EBS CSI Driver
+#---------------------------------------------------------------
+module "ebs_csi_driver_irsa" {
+  source                = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  version               = "~> 5.14"
+  role_name             = format("%s-%s", local.name, "ebs-csi-driver")
+  attach_ebs_csi_policy = true
+  oidc_providers = {
+    main = {
+      provider_arn               = module.eks.oidc_provider_arn
+      namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
+    }
+  }
+  tags = local.tags
+}
+
+#---------------------------------------------------------------
+# GP3 Encrypted Storage Class
+#---------------------------------------------------------------
+resource "kubernetes_annotations" "disable_gp2" {
+  annotations = {
+    "storageclass.kubernetes.io/is-default-class" : "false"
+  }
+  api_version = "storage.k8s.io/v1"
+  kind        = "StorageClass"
+  metadata {
+    name = "gp2"
+  }
+  force = true
+
+  depends_on = [module.eks]
+}
+
+resource "kubernetes_storage_class" "ebs_csi_encrypted_gp3_storage_class" {
+  metadata {
+    name = "gp3"
+    annotations = {
+      "storageclass.kubernetes.io/is-default-class" : "true"
+    }
+  }
+
+  storage_provisioner    = "ebs.csi.aws.com"
+  reclaim_policy         = "Delete"
+  allow_volume_expansion = true
+  volume_binding_mode    = "WaitForFirstConsumer"
+  parameters = {
+    fsType    = "xfs"
+    encrypted = true
+    type      = "gp3"
+  }
+
+  depends_on = [kubernetes_annotations.disable_gp2]
+}
+
+#---------------------------------------------------------------
+# EKS Blueprints Kubernetes Addons
+#---------------------------------------------------------------
+module "eks_blueprints_kubernetes_addons" {
+  source  = "aws-ia/eks-blueprints-addons/aws"
+  version = "~> 1.0"
+
+  cluster_name      = module.eks.cluster_name
+  cluster_endpoint  = module.eks.cluster_endpoint
+  cluster_version   = module.eks.cluster_version
+  oidc_provider_arn = module.eks.oidc_provider_arn
+
+  #---------------------------------------------------------------
+  # Amazon EKS Managed Add-ons
+  #---------------------------------------------------------------
+  eks_addons = {
+    aws-ebs-csi-driver = {
+      service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn
+    }
+    coredns = {
+      preserve = true
+    }
+    vpc-cni = {
+      preserve = true
+    }
+    kube-proxy = {
+      preserve = true
+    }
+  }
+  #---------------------------------------
+  # Kubernetes Add-ons
+  #---------------------------------------
+
+  enable_kube_prometheus_stack = true
+  kube_prometheus_stack = {
+    namespace     = "monitoring"
+    name          = "prometheus"
+    chart_version = "48.1.1"
+    set_sensitive = [
+      {
+        name  = "grafana.adminPassword"
+        value = data.aws_secretsmanager_secret_version.admin_password_version.secret_string
+    }]
+
+    values = [
+      templatefile("${path.module}/helm/kube-prometheus-stack-values.yaml", {
+        storage_class_type = kubernetes_storage_class.ebs_csi_encrypted_gp3_storage_class.id,
+      })
+    ]
+  }
+
+  #---------------------------------------
+  # AWS for FluentBit - DaemonSet
+  #---------------------------------------
+  enable_aws_for_fluentbit = true
+  aws_for_fluentbit_cw_log_group = {
+    use_name_prefix   = false
+    name              = "/${local.name}/aws-fluentbit-logs" # Add-on creates this log group
+    retention_in_days = 30
+  }
+  aws_for_fluentbit = {
+    values = [templatefile("${path.module}/helm/aws-for-fluentbit-values.yaml", {
+      region               = local.region,
+      cloudwatch_log_group = "/${local.name}/aws-fluentbit-logs"
+      cluster_name         = module.eks.cluster_name
+    })]
+  }
+  #---------------------------------------
+  # AWS Load Balancer Controller
+  #---------------------------------------
+  enable_aws_load_balancer_controller = false
+
+
+  tags = local.tags
+}
+
+#---------------------------------------------------------------
+# Apache Pinot
+#---------------------------------------------------------------
+
+resource "random_string" "random_suffix" {
+  length  = 10
+  special = false
+  upper   = false
+}
+
+resource "random_password" "sensitive_key" {
+  length  = 16
+  special = false
+}
+
+module "eks_data_addons" {
+  source  = "aws-ia/eks-data-addons/aws"
+  version = "~> 1.2.6"
+
+  oidc_provider_arn = module.eks.oidc_provider_arn
+
+  #---------------------------------------------------------------
+  # Apache Pinot Add-on
+  #---------------------------------------------------------------
+  enable_pinot = true
+  pinot_helm_config = {
+    namespace = "pinot"
+    values    = [templatefile("${path.module}/helm/pinot-values.yaml", {})]
+    set = [
+      {
+        name  = "cluster.name"
+        value = local.cluster_name
+      },
+      {
+        name  = "controller.replicaCount"
+        value = 3
+      },
+      {
+        name  = "controller.persistence.storageClass"
+        value = "gp3"
+      },
+      {
+        name  = "broker.replicaCount"
+        value = 3
+      },
+      {
+        name  = "server.replicaCount"
+        value = 3
+      },
+      {
+        name  = "server.persistence.storageClass"
+        value = "gp3"
+      },
+      {
+        name  = "minionStateless.replicaCount"
+        value = 3
+      },
+      {
+        name  = "minionStateless.persistence.storageClass"
+        value = "gp3"
+      },
+      {
+        name  = "zookeeper.replicaCount"
+        value = 3
+      },
+      {
+        name  = "zookeeper.persistence.storageClass"
+        value = "gp3"
+      }
+    ]
+  }
+}

distributed-databases/pinot/cleanup.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -o errexit
+set -o pipefail
+
+# Make sure you have `terraform.tfvars` file with the desired region.
+# Otherwise script will ask you to input your region each time it runs `terraform destroy`
+
+targets=(
+  "module.eks_data_addons"
+  "module.eks_blueprints_kubernetes_addons"
+  "module.vpc_cni_irsa"
+  "module.ebs_csi_driver_irsa"
+  "module.eks"
+  "module.vpc"
+)
+
+#-------------------------------------------
+# Terraform destroy per module target
+#-------------------------------------------
+for target in "${targets[@]}"
+do
+  terraform destroy -target="$target" -auto-approve
+done
+
+#-------------------------------------------
+# Terraform destroy full
+#-------------------------------------------
+terraform destroy -auto-approve