Option to create Elasticsearch delivery

cxcloud · Jan 26, 2021 · 190dcc1 · 190dcc1
1 parent 2a479bb
commit 190dcc1
Show file tree

Hide file tree

Showing 8 changed files with 116 additions and 71 deletions.
diff --git a/README.md b/README.md
@@ -13,6 +13,7 @@ The following table list the configurable parameters and their default values.
 | Name | Description | Default |
 | --- | --- | --- |
 | region | AWS Region | eu-west-1 |
+| create_es | Should Elasticsearch cluster be created | true |
 | es_name | Domain name for Elasticsearch | cxcloud |
 | es_ver | Elasticsearch Version | 6.5 |
 | es_instance_type | Instance type for Elasticsearch | t2.small.elasticsearch |
@@ -28,7 +29,6 @@ The following table list the configurable parameters and their default values.
 | s3_buffer_interval | S3 buffer interval | 60 |
 | s3_compression_format | S3 log compression format | GZIP |
 | es_index_name | Elasticsearch index name | cxcloud |
-| es_type_name | Elasticsearch index type | logs |
 | es_buffering_size | Elasticsearch buffering size | 10 |
 | es_buffering_interval | Elasticsearch buffering interval | 60 |
 | s3_backup_mode | S3 backup mode | AllDocuments |
@@ -48,30 +48,30 @@ The following table list the output variables
 
 ### Example
 
-The following example show how the module can be used. In this example release 1.1.0 is used.
+The following example show how the module can be used. In this example release 1.2.0 is used.
 
-Note that release 1.1.0 requires terraform client ver 0.12 or above. Use release 1.0.0 for terraform versions below 0.12.
+Note that release 1.2.0 requires terraform client ver 0.12 or above. Use release 1.0.0 for terraform versions below 0.12.
 
 ```console
 module "kinesis-firehose-elasticsearch" {
-  source                       = "github.com/cxcloud/terraform-kinesis-firehose-elasticsearch?ref=v1.1.0"
+  source                       = "github.com/cxcloud/terraform-kinesis-firehose-elasticsearch?ref=v1.2.0"
   region                       = "eu-west-1"
+  create_es                    = true
   es_name                      = "cxcloud"
-  es_ver                       = 6.5
-  es_instance_type             = "t2.small.elasticsearch"
+  es_ver                       = 7.9
+  es_instance_type             = "t3.small.elasticsearch"
   es_instance_count            = 1
   es_dedicated_master_enabled  = false
   es_ebs_size                  = 35
   es_snapshot_start_hour       = 23
   es_name_tag                  = "CX Cloud"
   es_whitelisted_ips           = ["34.240.135.89/32"]
   stream_name                  = "cxcloud"
-  s3_bucket                    = "sandbox-logging"
+  s3_bucket                    = "cxcloud-logging"
   s3_buffer_size               = 10
   s3_buffer_interval           = 60
   s3_compression_format        = "GZIP"
   es_index_name                = "cxcloud"
-  es_type_name                 = "logs"
   es_buffering_size            = 10
   es_buffering_interval        = 60
   s3_backup_mode               = "AllDocuments"

diff --git a/main.tf b/main.tf
@@ -3,6 +3,7 @@ data "aws_caller_identity" "current" {}
 # AWS Elasticsearch
 module "es" {
   source                   = "./modules/es"
+  create_es                = var.create_es
   name                     = var.es_name
   region                   = var.region
   account_id               = data.aws_caller_identity.current.account_id
@@ -28,8 +29,8 @@ module "kinesis-firehose" {
   s3_buffer_size               = var.s3_buffer_size
   s3_buffer_interval           = var.s3_buffer_interval
   s3_compression_format        = var.s3_compression_format
+  create_es_destination        = var.create_es
   es_index_name                = var.es_index_name
-  es_type_name                 = var.es_type_name
   es_buffering_size            = var.es_buffering_size
   es_buffering_interval        = var.es_buffering_interval
   s3_backup_mode               = var.s3_backup_mode

diff --git a/modules/es/main.tf b/modules/es/main.tf
@@ -39,6 +39,7 @@ data "aws_iam_policy_document" "es_policy" {
 }
 
 resource "aws_elasticsearch_domain" "cxcloud" {
+  count                 = var.create_es ? 1 : 0
   domain_name           = var.name
   elasticsearch_version = var.es_ver
 

diff --git a/modules/es/outputs.tf b/modules/es/outputs.tf
@@ -1,19 +1,19 @@
 output "arn" {
-  description = "ES ARN"
-  value       = aws_elasticsearch_domain.cxcloud.arn
+  description = "Amazon Resource Name (ARN) of the domain"
+  value       = var.create_es ? aws_elasticsearch_domain.cxcloud[0].arn : ""
 }
 
 output "endpoint" {
-  description = "ES Endpoint"
-  value       = aws_elasticsearch_domain.cxcloud.endpoint
+  description = "Domain-specific endpoint used to submit index, search, and data upload requests"
+  value       = var.create_es ? aws_elasticsearch_domain.cxcloud[0].endpoint : ""
 }
 
 output "domain_id" {
-  description = "ES ARN"
-  value       = aws_elasticsearch_domain.cxcloud.domain_id
+  description = "Unique identifier for the domain"
+  value       = var.create_es ? aws_elasticsearch_domain.cxcloud[0].domain_id : ""
 }
 
 output "domain_name" {
-  description = "ES ARN"
-  value       = aws_elasticsearch_domain.cxcloud.domain_name
+  description = "The name of the Elasticsearch domain"
+  value       = var.create_es ? aws_elasticsearch_domain.cxcloud[0].domain_name : ""
 }
diff --git a/modules/es/variables.tf b/modules/es/variables.tf
@@ -1,3 +1,9 @@
+variable "create_es" {
+  type        = bool
+  description = "Should ES cluster be created"
+  default     = "true"
+}
+
 variable "name" {
   type        = string
   description = "Domain name for Elasticsearch"
@@ -17,7 +23,7 @@ variable "account_id" {
 variable "es_ver" {
   type        = string
   description = "Elasticsearch Version"
-  default     = 6.5
+  default     = 7.9
 }
 
 variable "instance_type" {

diff --git a/modules/kinesis-firehose/main.tf b/modules/kinesis-firehose/main.tf
@@ -24,8 +24,8 @@ POLICY
 }
 
 resource "aws_iam_policy" "firehose_delivery_policy" {
-  name = "firehose-delivery-policy"
-  path = "/"
+  name        = "firehose-delivery-policy"
+  path        = "/"
   description = "Kinesis Firehose delivery policy"
 
   policy = <<POLICY
@@ -43,6 +43,41 @@ resource "aws_iam_policy" "firehose_delivery_policy" {
                 "${aws_s3_bucket.logs.arn}/*"
             ]
         },
+        {
+            "Sid": "",
+            "Effect": "Allow",
+            "Action": [
+                "logs:PutLogEvents"
+            ],
+            "Resource": [
+                "arn:aws:logs:${var.region}:${var.account_id}:log-group:/aws/kinesisfirehose/%FIREHOSE_STREAM_NAME%:log-stream:*"
+            ]
+        },
+        {
+            "Sid": "",
+            "Effect": "Allow",
+            "Action": [
+                "kinesis:DescribeStream",
+                "kinesis:GetShardIterator",
+                "kinesis:GetRecords"
+            ],
+            "Resource": "arn:aws:kinesis:${var.region}:${var.account_id}:stream/%FIREHOSE_STREAM_NAME%"
+        }
+    ]
+}
+POLICY
+}
+
+resource "aws_iam_policy" "firehose_es_delivery_policy" {
+  count       = var.create_es_destination ? 1 : 0
+  name        = "firehose-es-delivery-policy"
+  path        = "/"
+  description = "Kinesis Firehose ES delivery policy"
+
+  policy = <<POLICY
+{
+    "Version": "2012-10-17",
+    "Statement": [
         {
             "Sid": "",
             "Effect": "Allow",
@@ -74,26 +109,6 @@ resource "aws_iam_policy" "firehose_delivery_policy" {
                 "${var.es_arn}/_stats",
                 "${var.es_arn}/cxcloud*/_stats"
             ]
-        },
-        {
-            "Sid": "",
-            "Effect": "Allow",
-            "Action": [
-                "logs:PutLogEvents"
-            ],
-            "Resource": [
-                "arn:aws:logs:${var.region}:${var.account_id}:log-group:/aws/kinesisfirehose/%FIREHOSE_STREAM_NAME%:log-stream:*"
-            ]
-        },
-        {
-            "Sid": "",
-            "Effect": "Allow",
-            "Action": [
-                "kinesis:DescribeStream",
-                "kinesis:GetShardIterator",
-                "kinesis:GetRecords"
-            ],
-            "Resource": "arn:aws:kinesis:${var.region}:${var.account_id}:stream/%FIREHOSE_STREAM_NAME%"
         }
     ]
 }
@@ -105,6 +120,12 @@ resource "aws_iam_role_policy_attachment" "attach_delivery_policy" {
   policy_arn = aws_iam_policy.firehose_delivery_policy.arn
 }
 
+resource "aws_iam_role_policy_attachment" "attach_es_delivery_policy" {
+  count      = var.create_es_destination ? 1 : 0
+  role       = aws_iam_role.firehose_delivery_role.name
+  policy_arn = aws_iam_policy.firehose_es_delivery_policy[0].arn
+}
+
 data "aws_iam_policy_document" "assume_kinesis_firehose" {
   statement {
     actions = ["sts:AssumeRole"]
@@ -128,23 +149,39 @@ resource "aws_iam_role_policy_attachment" "attach_kinesis_firehose" {
 
 resource "aws_kinesis_firehose_delivery_stream" "cxcloud" {
   name        = var.stream_name
-  destination = "elasticsearch"
-
-  s3_configuration {
-    role_arn           = aws_iam_role.firehose_delivery_role.arn
-    bucket_arn         = aws_s3_bucket.logs.arn
-    buffer_size        = var.s3_buffer_size
-    buffer_interval    = var.s3_buffer_interval
-    compression_format = var.s3_compression_format
+  destination = var.create_es_destination ? "elasticsearch" : "extended_s3"
+
+  dynamic "extended_s3_configuration" {
+    for_each = var.create_es_destination ? [] : [1]
+    content {
+      role_arn           = aws_iam_role.firehose_delivery_role.arn
+      bucket_arn         = aws_s3_bucket.logs.arn
+      buffer_size        = var.s3_buffer_size
+      buffer_interval    = var.s3_buffer_interval
+      compression_format = var.s3_compression_format
+    }
   }
 
-  elasticsearch_configuration {
-    domain_arn         = var.es_arn
-    role_arn           = aws_iam_role.firehose_delivery_role.arn
-    index_name         = var.es_index_name
-    type_name          = var.es_type_name
-    buffering_size     = var.es_buffering_size
-    buffering_interval = var.es_buffering_interval
-    s3_backup_mode     = var.s3_backup_mode
+  dynamic "s3_configuration" {
+    for_each = var.create_es_destination ? [1] : []
+    content {
+      role_arn           = aws_iam_role.firehose_delivery_role.arn
+      bucket_arn         = aws_s3_bucket.logs.arn
+      buffer_size        = var.s3_buffer_size
+      buffer_interval    = var.s3_buffer_interval
+      compression_format = var.s3_compression_format
+    }
+  }
+
+  dynamic "elasticsearch_configuration" {
+    for_each = var.create_es_destination ? [1] : []
+    content {
+      domain_arn         = var.es_arn
+      role_arn           = aws_iam_role.firehose_delivery_role.arn
+      index_name         = var.es_index_name
+      buffering_size     = var.es_buffering_size
+      buffering_interval = var.es_buffering_interval
+      s3_backup_mode     = var.s3_backup_mode
+    }
   }
 }
diff --git a/modules/kinesis-firehose/variables.tf b/modules/kinesis-firehose/variables.tf
@@ -43,18 +43,18 @@ variable "s3_compression_format" {
   default     = "GZIP"
 }
 
+variable "create_es_destination" {
+  type        = bool
+  description = "Should ES cluster be created"
+  default     = "true"
+}
+
 variable "es_index_name" {
   type        = string
   description = "Elasticsearch index name"
   default     = "cxcloud"
 }
 
-variable "es_type_name" {
-  type        = string
-  description = "Elasticsearch index type"
-  default     = "logs"
-}
-
 variable "es_buffering_size" {
   type        = string
   description = "Elasticsearch buffering size"

diff --git a/variables.tf b/variables.tf
@@ -8,6 +8,12 @@ variable "region" {
 
 # Elasticsearch variables
 
+variable "create_es" {
+  type        = bool
+  description = "Should ES cluster be created"
+  default     = "true"
+}
+
 variable "es_name" {
   type        = string
   description = "Cluster name for Elasticsearch"
@@ -17,13 +23,13 @@ variable "es_name" {
 variable "es_ver" {
   type        = string
   description = "Elasticsearch version"
-  default     = 6.5
+  default     = 7.9
 }
 
 variable "es_instance_type" {
   type        = string
   description = "Instance type for Elasticsearch"
-  default     = "t2.small.elasticsearch"
+  default     = "t3.small.elasticsearch"
 }
 
 variable "es_instance_count" {
@@ -62,7 +68,7 @@ variable "es_whitelisted_ips" {
   default     = []
 }
 
-# Kinesis Firehose variables 
+# Kinesis Firehose variables
 variable "stream_name" {
   type        = string
   description = "Kinesis Firehose Stream Name"
@@ -98,12 +104,6 @@ variable "es_index_name" {
   default     = "cxcloud"
 }
 
-variable "es_type_name" {
-  type        = string
-  description = "Elasticsearch index type"
-  default     = "logs"
-}
-
 variable "es_buffering_size" {
   type        = string
   description = "Elasticsearch buffering size"