Skip to content

Commit

Permalink
updatew dagster tf
Browse files Browse the repository at this point in the history
  • Loading branch information
maciaszczykm committed Dec 13, 2024
1 parent cc59681 commit 7d561e8
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 14 deletions.
35 changes: 34 additions & 1 deletion catalogs/data/dagster/terraform/aws/outputs.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,36 @@
output "iam_user" {
value = aws_iam_user.dagster
}
}

output "access_key_id" {
value = aws_iam_access_key.airbyte.id
}

output "secret_access_key" {
value = aws_iam_access_key.airbyte.secret
sensitive = true
}

output "postgres_host" {
value = try(module.db.db_instance_address, "")
}

output "postgres_password" {
value = random_password.password.result
sensitive = true
}

output "oidc_cookie_secret" {
value = random_password.oidc_cookie.result
sensitive = true
}

output "oidc_client_id" {
value = plural_oidc_provider.airbyte.client_id
sensitive = true
}

output "oidc_client_secret" {
value = plural_oidc_provider.airbyte.client_secret
sensitive = true
}
92 changes: 80 additions & 12 deletions catalogs/data/dagster/terraform/aws/postgres.tf
Original file line number Diff line number Diff line change
@@ -1,18 +1,86 @@
data "aws_iam_role" "postgres" {
name = "${data.plural_cluster.cluster.name}-postgres"
resource "random_password" "password" {
length = 20
min_lower = 1
min_numeric = 1
min_upper = 1
special = false
}

resource "kubernetes_service_account" "postgres" {
metadata {
name = "postgres-pod"
namespace = var.namespace
data "aws_eks_cluster" "mgmt" {
name = data.plural_cluster.cluster.name

annotations = {
"eks.amazonaws.com/role-arn" = data.aws_iam_role.postgres.arn
depends_on = [ data.plural_cluster.cluster ]
}

data "aws_vpc" "mgmt" {
id = one(data.aws_eks_cluster.mgmt.vpc_config).vpc_id
}

module "db" {
source = "terraform-aws-modules/rds/aws"
version = "~> 6.3"

identifier = var.db_name

engine = "postgres"
engine_version = var.postgres_vsn
family = "postgres14"
major_engine_version = var.postgres_vsn
instance_class = var.db_instance_class
allocated_storage = var.db_storage

db_name = "dagster"
username = "dagster"
password = random_password.password.result
manage_master_user_password = false

maintenance_window = "Mon:00:00-Mon:03:00"
backup_window = "03:00-06:00"
backup_retention_period = var.backup_retention_period

monitoring_interval = "30"
monitoring_role_name = "${var.db_name}-PluralRDSMonitoringRole"
create_monitoring_role = true
apply_immediately = true

multi_az = true

create_db_subnet_group = true
subnet_ids = one(data.aws_eks_cluster.mgmt.vpc_config).subnet_ids
vpc_security_group_ids = [module.security_group.security_group_id]

create_cloudwatch_log_group = true
enabled_cloudwatch_logs_exports = ["postgresql"]

parameters = [
{
name = "autovacuum"
value = 1
},
{
name = "client_encoding"
value = "utf8"
}
}
]

depends_on = [
kubernetes_namespace.dagster
deletion_protection = var.deletion_protection
}

module "security_group" {
source = "terraform-aws-modules/security-group/aws"
version = "~> 5.0"

name = "${var.db_name}-db-security-group"
description = "security group for your plural console db"
vpc_id = data.aws_vpc.mgmt.id

ingress_with_cidr_blocks = [
{
from_port = 5432
to_port = 5432
protocol = "tcp"
description = "PostgreSQL access from within VPC"
cidr_blocks = data.aws_vpc.mgmt.cidr_block
},
]
}
}
26 changes: 26 additions & 0 deletions catalogs/data/dagster/terraform/aws/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,30 @@ variable "force_destroy_bucket" {
type = bool
default = true
description = "If true, the bucket will be deleted even if it contains objects."
}

variable "db_name" {
default = "plrl-{{ context.cluster }}-airbyte"
}

variable "postgres_vsn" {
default = "14"
}

variable "db_storage" {
default = 20
}

variable "deletion_protection" {
type = bool
default = true
}

variable "backup_retention_period" {
type = number
default = 7
}

variable "db_instance_class" {
default = "db.t4g.large"
}
1 change: 0 additions & 1 deletion catalogs/data/dagster/terraform/aws/versions.tf
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

terraform {
required_providers {
aws = {
Expand Down

0 comments on commit 7d561e8

Please sign in to comment.