Merge pull request #232 from silinternational/feature/remove-id-sync-api

Remove id-sync API and run the sync as a scheduled task

terraform/070-id-sync/README.md

@@ -4,9 +4,7 @@ store.
 
 ## What this does
 
- - Create ALB target group for SSP with hostname based routing
  - Create task definition and ECS service
- - Create Cloudflare DNS record
 
 ## Required Inputs
 
@@ -15,9 +13,6 @@ store.
  - `aws_region` - AWS region
  - `cloudwatch_log_group_name` - CloudWatch log group name
  - `vpc_id` - ID for VPC
- - `alb_https_listener_arn` - ARN for ALB HTTPS listener
- - `subdomain` - Subdomain for SSP IdP
- - `cloudflare_domain` - Top level domain name for use with Cloudflare
  - `docker_image` - URL to Docker image
  - `email_service_accessToken` - Access token for Email Service API
  - `email_service_baseUrl` - Base URL (e.g. 'https://email.example.com') to Email Service API
@@ -32,7 +27,6 @@ store.
  - `idp_display_name` - Friendly name for IdP
  - `ecs_cluster_id` - ID for ECS Cluster
  - `ecsServiceRole_arn` - ARN for ECS Service Role
- - `alb_dns_name` - DNS name for application load balancer
  - `memory` - Amount of memory to allocate to container
  - `cpu` - Amount of CPU to allocate to container
 
@@ -44,15 +38,9 @@ store.
 - `notifier_email_to` - Who to send notifications to about sync problems (e.g. users lacking email addresses)
 - `sync_safety_cutoff` - The percentage of records allowed to be changed during a sync, provided as a float, ex: `0.2` for `20%`
 - `allow_empty_email` - Whether or not to allow the primary email property to be empty. Default: `false`
-- `create_dns_record` - Controls creation of a DNS CNAME record for the ECS service. Default: `true`
 - `enable_new_user_notification` - Enable email notification to HR Contact upon creation of a new user, if set to 'true'. Default: `false`
-- `enable_sync` - Set to false to disable the sync process.
 - `sentry_dsn` - Sentry DSN for error logging and alerting. Obtain from Sentry dashboard: Settings - Projects - (project) - Client Keys
-
-## Outputs
-
- - `idsync_url` - URL for ID Sync webhook endpoint
- - `access_token_external` - Access token for external systems to use to make webhook calls to Sync
+- `event_schedule` - AWS Cloudwatch schedule for the sync task. Use cron format "cron(Minutes Hours Day-of-month Month Day-of-week Year)" where either `day-of-month` or `day-of-week` must be a question mark, or rate format "rate(15 minutes)". Default = "cron(*/15 * * * ? *)"
 
 ## Usage Example
 
@@ -65,9 +53,7 @@ module "idsync" {
   app_env                     = var.app_env
   vpc_id                      = data.terraform_remote_state.cluster.vpc_id
   alb_https_listener_arn      = data.terraform_remote_state.cluster.alb_https_listener_arn
-  subdomain                   = var.sync_subdomain
   aws_region                  = var.aws_region
-  cloudflare_domain           = var.cloudflare_domain
   cloudwatch_log_group_name   = var.cloudwatch_log_group_name
   docker_image                = data.terraform_remote_state.ecr.ecr_repo_idsync
   email_service_accessToken   = data.terraform_remote_state.email.access_token_idsync
@@ -84,8 +70,6 @@ module "idsync" {
   idp_name                    = var.idp_name
   idp_display_name            = var.idp_display_name
   ecs_cluster_id              = data.terraform_remote_state.core.ecs_cluster_id
-  ecsServiceRole_arn          = data.terraform_remote_state.core.ecsServiceRole_arn
-  alb_dns_name                = data.terraform_remote_state.cluster.alb_dns_name
   alerts_email                = var.alerts_email
   notifier_email_to           = var.notifier_email_to
   allow_empty_email           = var.allow_empty_email

terraform/070-id-sync/main.tf

@@ -1,54 +1,7 @@
-/*
- * Create target group for ALB
- */
-resource "aws_alb_target_group" "idsync" {
-  name                 = substr("tg-${var.idp_name}-${var.app_name}-${var.app_env}", 0, 32)
-  port                 = "80"
-  protocol             = "HTTP"
-  vpc_id               = var.vpc_id
-  deregistration_delay = "30"
-
-  health_check {
-    path    = "/site/system-status"
-    matcher = "200"
-  }
-}
-
-/*
- * Create listener rule for hostname routing to new target group
- */
-resource "aws_alb_listener_rule" "idsync" {
-  listener_arn = var.alb_https_listener_arn
-  priority     = "70"
-
-  action {
-    type             = "forward"
-    target_group_arn = aws_alb_target_group.idsync.arn
-  }
-
-  condition {
-    host_header {
-      values = [
-        "${var.subdomain}.${var.cloudflare_domain}",
-        "${local.subdomain_with_region}.${var.cloudflare_domain}"
-      ]
-    }
-  }
-}
-
-/*
- * Generate access token for external system to use when calling ID Sync
- */
-resource "random_id" "access_token_external" {
-  byte_length = 16
-}
-
 /*
  * Create ECS service
  */
 locals {
-  subdomain_with_region = "${var.subdomain}-${var.aws_region}"
-
   id_store_config = join(",",
     [for k, v in var.id_store_config : jsonencode({
       name  = "ID_STORE_CONFIG_${k}"
@@ -73,7 +26,6 @@ locals {
     id_broker_trustedIpRanges    = join(",", var.id_broker_trustedIpRanges)
     id_store_adapter             = var.id_store_adapter
     id_store_config              = local.id_store_config
-    id_sync_access_tokens        = random_id.access_token_external.hex
     idp_name                     = var.idp_name
     idp_display_name             = var.idp_display_name
     alerts_email                 = var.alerts_email
@@ -87,41 +39,86 @@ locals {
   })
 }
 
-module "ecsservice" {
-  source             = "github.com/silinternational/terraform-modules//aws/ecs/service-only?ref=8.6.0"
-  cluster_id         = var.ecs_cluster_id
-  service_name       = "${var.idp_name}-${var.app_name}"
-  service_env        = var.app_env
-  container_def_json = local.task_def
-  desired_count      = var.enable_sync ? 1 : 0
-  tg_arn             = aws_alb_target_group.idsync.arn
-  lb_container_name  = "web"
-  lb_container_port  = "80"
-  ecsServiceRole_arn = var.ecsServiceRole_arn
-}
-
 /*
- * Create Cloudflare DNS record(s)
+ * Create role for scheduled running of cron task definitions.
  */
-resource "cloudflare_record" "idsyncdns" {
-  count = var.create_dns_record ? 1 : 0
+resource "aws_iam_role" "ecs_events" {
+  name = "ecs_events-${var.idp_name}-${var.app_name}-${var.app_env}-${var.aws_region}"
 
-  zone_id = data.cloudflare_zone.domain.id
-  name    = var.subdomain
-  value   = cloudflare_record.idsyncdns_intermediate.hostname
-  type    = "CNAME"
-  proxied = true
+  assume_role_policy = jsonencode(
+    {
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Sid    = ""
+          Effect = "Allow"
+          Principal = {
+            Service = "events.amazonaws.com"
+          }
+          Action = "sts:AssumeRole"
+        },
+      ]
+    }
+  )
 }
 
-resource "cloudflare_record" "idsyncdns_intermediate" {
-  zone_id = data.cloudflare_zone.domain.id
-  name    = local.subdomain_with_region
-  value   = var.alb_dns_name
-  type    = "CNAME"
-  proxied = true
+resource "aws_iam_role_policy" "ecs_events_run_task_with_any_role" {
+  name = "ecs_events_run_task_with_any_role"
+  role = aws_iam_role.ecs_events.id
+
+  policy = jsonencode(
+    {
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Effect   = "Allow"
+          Action   = "iam:PassRole"
+          Resource = "*"
+        },
+        {
+          Effect   = "Allow"
+          Action   = "ecs:RunTask"
+          Resource = "${aws_ecs_task_definition.cron_td.arn_without_revision}:*"
+        },
+      ]
+    }
+  )
+
 }
 
-data "cloudflare_zone" "domain" {
-  name = var.cloudflare_domain
+/*
+ * Create cron task definition
+ */
+resource "aws_ecs_task_definition" "cron_td" {
+  family                = "${var.idp_name}-${var.app_name}-cron-${var.app_env}"
+  container_definitions = local.task_def
+  network_mode          = "bridge"
 }
 
+/*
+ * CloudWatch configuration to start scheduled tasks.
+ */
+resource "aws_cloudwatch_event_rule" "event_rule" {
+  name        = "${var.idp_name}-${var.app_name}-${var.app_env}"
+  description = "Start ID Sync scheduled tasks"
+
+  schedule_expression = var.event_schedule
+
+  tags = {
+    app_name = var.app_name
+    app_env  = var.app_env
+  }
+}
+
+resource "aws_cloudwatch_event_target" "id_sync_event_target" {
+  target_id = "${var.idp_name}-${var.app_name}-${var.app_env}"
+  rule      = aws_cloudwatch_event_rule.event_rule.name
+  arn       = var.ecs_cluster_id
+  role_arn  = aws_iam_role.ecs_events.arn
+
+  ecs_target {
+    task_count          = 1
+    launch_type         = "EC2"
+    task_definition_arn = aws_ecs_task_definition.cron_td.arn
+  }
+}

terraform/070-id-sync/task-definition.json

@@ -6,13 +6,6 @@
     "dnsServers": null,
     "disableNetworking": null,
     "dnsSearchDomains": null,
-    "portMappings": [
-      {
-        "hostPort": 0,
-        "containerPort": 80,
-        "protocol": "tcp"
-      }
-    ],
     "hostname": null,
     "essential": true,
     "entryPoint": null,
@@ -65,10 +58,6 @@
         "name": "ID_STORE_ADAPTER",
         "value": "${id_store_adapter}"
       },
-      {
-        "name": "ID_SYNC_ACCESS_TOKENS",
-        "value": "${id_sync_access_tokens}"
-      },
       {
         "name": "IDP_NAME",
         "value": "${idp_name}"

terraform/070-id-sync/vars.tf

@@ -19,22 +19,10 @@ variable "vpc_id" {
   type = string
 }
 
-variable "alb_https_listener_arn" {
-  type = string
-}
-
 variable "aws_region" {
   type = string
 }
 
-variable "subdomain" {
-  type = string
-}
-
-variable "cloudflare_domain" {
-  type = string
-}
-
 variable "cloudwatch_log_group_name" {
   type = string
 }
@@ -106,14 +94,6 @@ variable "ecs_cluster_id" {
   type = string
 }
 
-variable "ecsServiceRole_arn" {
-  type = string
-}
-
-variable "alb_dns_name" {
-  type = string
-}
-
 variable "notifier_email_to" {
   default     = ""
   type        = string
@@ -138,14 +118,10 @@ variable "enable_new_user_notification" {
   default = "false"
 }
 
-variable "enable_sync" {
-  default = true
-}
-
-variable "create_dns_record" {
-  description = "Controls creation of a DNS CNAME record for the ECS service."
-  type        = bool
-  default     = true
+variable "event_schedule" {
+  description = "AWS Cloudwatch schedule for the sync task"
+  type        = string
+  default     = "cron(*/15 * * * ? *)"
 }
 
 variable "sentry_dsn" {

test/070-id-sync.tf

@@ -1,26 +1,21 @@
 module "sync" {
   source = "../terraform/070-id-sync"
 
-  alb_dns_name                 = ""
-  alb_https_listener_arn       = ""
   alerts_email                 = ""
   allow_empty_email            = ""
   app_env                      = ""
   app_name                     = ""
   aws_region                   = ""
-  cloudflare_domain            = ""
   cloudwatch_log_group_name    = ""
   cpu                          = ""
-  create_dns_record            = true
   docker_image                 = ""
-  ecsServiceRole_arn           = ""
   ecs_cluster_id               = ""
   email_service_accessToken    = ""
   email_service_assertValidIp  = ""
   email_service_baseUrl        = ""
   email_service_validIpRanges  = [""]
   enable_new_user_notification = ""
-  enable_sync                  = ""
+  event_schedule               = ""
   id_broker_access_token       = ""
   id_broker_adapter            = ""
   id_broker_assertValidIp      = ""
@@ -32,7 +27,6 @@ module "sync" {
   idp_name                     = ""
   memory                       = ""
   notifier_email_to            = ""
-  subdomain                    = ""
   sync_safety_cutoff           = ""
   vpc_id                       = ""
 }