diff --git a/catalogs/data/dagster/terraform/aws/iam.tf b/catalogs/data/dagster/terraform/aws/iam.tf index cd56c6bc..c35ab5ab 100644 --- a/catalogs/data/dagster/terraform/aws/iam.tf +++ b/catalogs/data/dagster/terraform/aws/iam.tf @@ -1,15 +1,27 @@ -module "assumable_role_airflow" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "3.14.0" - create_role = true - role_name = "${data.plural_cluster.cluster.name}-${var.role_name}" - provider_url = replace(data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer, "https://", "") - role_policy_arns = [module.s3_buckets.policy_arn] - oidc_fully_qualified_subjects = ["system:serviceaccount:${var.namespace}:${var.dagster_serviceaccount}"] +data "aws_iam_policy_document" "dagster" { + statement { + sid = "admin" + effect = "Allow" + actions = ["s3:*"] + + resources = [ + "arn:aws:s3:::${var.dagster_bucket}", + "arn:aws:s3:::${var.dagster_bucket}/*", + ] + } +} + +resource "aws_iam_policy" "dagster" { + name_prefix = "dagster" + description = "policy for the plural admin dagster" + policy = data.aws_iam_policy_document.dagster.json } resource "aws_iam_user" "dagster" { name = "${data.plural_cluster.cluster.name}-dagster" + + depends_on = [ data.plural_cluster.cluster ] + } resource "aws_iam_access_key" "dagster" { @@ -19,7 +31,17 @@ resource "aws_iam_access_key" "dagster" { resource "aws_iam_policy_attachment" "dagster-user" { name = "${data.plural_cluster.cluster.name}-dagster-policy" users = [aws_iam_user.dagster.name] - policy_arn = module.s3_buckets.policy_arn + policy_arn = aws_iam_policy.dagster.arn +} + +resource "kubernetes_namespace" "dagster" { + metadata { + name = var.namespace + labels = { + "app.kubernetes.io/managed-by" = "plural" + "app.plural.sh/name" = "dagster" + } + } } resource "kubernetes_secret" "dagster_s3_secret" { diff --git a/catalogs/data/dagster/terraform/aws/main.tf b/catalogs/data/dagster/terraform/aws/main.tf deleted file mode 100644 index a4df2656..00000000 --- a/catalogs/data/dagster/terraform/aws/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -resource "kubernetes_namespace" "dagster" { - metadata { - name = var.namespace - labels = { - "app.kubernetes.io/managed-by" = "plural" - "app.plural.sh/name" = "dagster" - "platform.plural.sh/sync-target" = "pg" - } - } -} - -module "s3_buckets" { - source = "github.com/pluralsh/module-library//terraform/s3-buckets?ref=bucket-protection" - bucket_names = [var.dagster_bucket] - policy_prefix = "dagster" - force_destroy = var.force_destroy_bucket -} - -data "aws_eks_cluster" "cluster" { - name = data.plural_cluster.cluster.name -} diff --git a/catalogs/data/dagster/terraform/aws/oidc.tf b/catalogs/data/dagster/terraform/aws/oidc.tf new file mode 100644 index 00000000..5425c2bd --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/oidc.tf @@ -0,0 +1,15 @@ +resource "random_password" "oidc_cookie" { + length = 24 + min_lower = 1 + min_numeric = 1 + min_upper = 1 + special = false +} + +resource "plural_oidc_provider" "dagster" { + name = "dagster-{{ context.cluster }}" + auth_method = "BASIC" + type = "PLURAL" + description = "OIDC provider for Dagster deployed to the {{ context.cluster }} cluster" + redirect_uris = ["https://{{ context.hostname }}/oauth2/callback"] +} diff --git a/catalogs/data/dagster/terraform/aws/outputs.tf b/catalogs/data/dagster/terraform/aws/outputs.tf index 2a306c11..076908be 100644 --- a/catalogs/data/dagster/terraform/aws/outputs.tf +++ b/catalogs/data/dagster/terraform/aws/outputs.tf @@ -1,3 +1,36 @@ output "iam_user" { value = aws_iam_user.dagster -} \ No newline at end of file +} + +output "access_key_id" { + value = aws_iam_access_key.dagster.id +} + +output "secret_access_key" { + value = aws_iam_access_key.dagster.secret + sensitive = true +} + +output "postgres_host" { + value = try(module.db.db_instance_address, "") +} + +output "postgres_password" { + value = random_password.password.result + sensitive = true +} + +output "oidc_cookie_secret" { + value = random_password.oidc_cookie.result + sensitive = true +} + +output "oidc_client_id" { + value = plural_oidc_provider.dagster.client_id + sensitive = true +} + +output "oidc_client_secret" { + value = plural_oidc_provider.dagster.client_secret + sensitive = true +} diff --git a/catalogs/data/dagster/terraform/aws/postgres.tf b/catalogs/data/dagster/terraform/aws/postgres.tf index 6fc1583a..f04449de 100644 --- a/catalogs/data/dagster/terraform/aws/postgres.tf +++ b/catalogs/data/dagster/terraform/aws/postgres.tf @@ -1,18 +1,86 @@ -data "aws_iam_role" "postgres" { - name = "${data.plural_cluster.cluster.name}-postgres" +resource "random_password" "password" { + length = 20 + min_lower = 1 + min_numeric = 1 + min_upper = 1 + special = false } -resource "kubernetes_service_account" "postgres" { - metadata { - name = "postgres-pod" - namespace = var.namespace +data "aws_eks_cluster" "mgmt" { + name = data.plural_cluster.cluster.name - annotations = { - "eks.amazonaws.com/role-arn" = data.aws_iam_role.postgres.arn + depends_on = [ data.plural_cluster.cluster ] +} + +data "aws_vpc" "mgmt" { + id = one(data.aws_eks_cluster.mgmt.vpc_config).vpc_id +} + +module "db" { + source = "terraform-aws-modules/rds/aws" + version = "~> 6.3" + + identifier = var.db_name + + engine = "postgres" + engine_version = var.postgres_vsn + family = "postgres14" + major_engine_version = var.postgres_vsn + instance_class = var.db_instance_class + allocated_storage = var.db_storage + + db_name = "dagster" + username = "dagster" + password = random_password.password.result + manage_master_user_password = false + + maintenance_window = "Mon:00:00-Mon:03:00" + backup_window = "03:00-06:00" + backup_retention_period = var.backup_retention_period + + monitoring_interval = "30" + monitoring_role_name = "${var.db_name}-PluralRDSMonitoringRole" + create_monitoring_role = true + apply_immediately = true + + multi_az = true + + create_db_subnet_group = true + subnet_ids = one(data.aws_eks_cluster.mgmt.vpc_config).subnet_ids + vpc_security_group_ids = [module.security_group.security_group_id] + + create_cloudwatch_log_group = true + enabled_cloudwatch_logs_exports = ["postgresql"] + + parameters = [ + { + name = "autovacuum" + value = 1 + }, + { + name = "client_encoding" + value = "utf8" } - } + ] - depends_on = [ - kubernetes_namespace.dagster + deletion_protection = var.deletion_protection +} + +module "security_group" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 5.0" + + name = "${var.db_name}-db-security-group" + description = "security group for your plural console db" + vpc_id = data.aws_vpc.mgmt.id + + ingress_with_cidr_blocks = [ + { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + description = "PostgreSQL access from within VPC" + cidr_blocks = data.aws_vpc.mgmt.cidr_block + }, ] -} \ No newline at end of file +} diff --git a/catalogs/data/dagster/terraform/aws/s3.tf b/catalogs/data/dagster/terraform/aws/s3.tf new file mode 100644 index 00000000..2f2fb32e --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/s3.tf @@ -0,0 +1,14 @@ +resource "aws_s3_bucket" "dagster" { + bucket = var.dagster_bucket + force_destroy = var.force_destroy_bucket +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "dagster" { + bucket = aws_s3_bucket.dagster.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} diff --git a/catalogs/data/dagster/terraform/aws/variables.tf b/catalogs/data/dagster/terraform/aws/variables.tf index 81f2e3b5..38c1e755 100644 --- a/catalogs/data/dagster/terraform/aws/variables.tf +++ b/catalogs/data/dagster/terraform/aws/variables.tf @@ -13,18 +13,34 @@ variable "dagster_bucket" { default = "{{ context.bucket }}" } -variable "dagster_serviceaccount" { - type = string - default = "dagster" -} - -variable "role_name" { - type = string - default = "dagster" -} - variable "force_destroy_bucket" { type = bool default = true description = "If true, the bucket will be deleted even if it contains objects." +} + +variable "db_name" { + default = "plrl-{{ context.cluster }}-dagster" +} + +variable "postgres_vsn" { + default = "14" +} + +variable "db_storage" { + default = 20 +} + +variable "deletion_protection" { + type = bool + default = true +} + +variable "backup_retention_period" { + type = number + default = 7 +} + +variable "db_instance_class" { + default = "db.t4g.large" } \ No newline at end of file diff --git a/catalogs/data/dagster/terraform/aws/versions.tf b/catalogs/data/dagster/terraform/aws/versions.tf index 54f47de0..a265c4d5 100644 --- a/catalogs/data/dagster/terraform/aws/versions.tf +++ b/catalogs/data/dagster/terraform/aws/versions.tf @@ -1,4 +1,3 @@ - terraform { required_providers { aws = {