diff --git a/catalogs/data/dagster/dagster.yaml b/catalogs/data/dagster/dagster.yaml new file mode 100644 index 00000000..ae5cf5a6 --- /dev/null +++ b/catalogs/data/dagster/dagster.yaml @@ -0,0 +1,55 @@ +apiVersion: deployments.plural.sh/v1alpha1 +kind: InfrastructureStack +metadata: + name: dagster-{{ context.cluster }} + namespace: apps +spec: + detach: false + type: TERRAFORM + approval: true + manageState: true + actor: console@plural.sh + git: + ref: main + folder: terraform/apps/dagster/{{ context.cluster }} + repositoryRef: + name: infra + namespace: infra + configuration: + version: '1.8' + clusterRef: + name: {{ context.cluster }} + namespace: infra +--- +apiVersion: deployments.plural.sh/v1alpha1 +kind: ServiceDeployment +metadata: + name: dagster-{{ context.cluster }} + namespace: apps +spec: + namespace: dagster + git: + folder: helm/dagster/{{ context.cluster }} + ref: main + repositoryRef: + kind: GitRepository + name: infra + namespace: infra + helm: + url: https://dagster-io.github.io/helm + version: "1.x.x" + chart: dagster + valuesFiles: + - dagster.yaml.liquid + imports: + - stackRef: + name: dagster-{{ context.cluster }} + namespace: infra + configuration: + cluster: {{ context.cluster }} + hostname: {{ context.hostname }} + bucket: {{ context.bucket }} + clusterRef: + kind: Cluster + name: {{ context.cluster }} + namespace: infra diff --git a/catalogs/data/dagster/helmrepository.yaml b/catalogs/data/dagster/helmrepository.yaml deleted file mode 100644 index 5160cfad..00000000 --- a/catalogs/data/dagster/helmrepository.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1beta2 -kind: HelmRepository -metadata: - name: dagster - namespace: apps -spec: - interval: 5m0s - url: https://dagster-io.github.io/helm diff --git a/catalogs/data/dagster/terraform/aws/iam.tf b/catalogs/data/dagster/terraform/aws/iam.tf new file mode 100644 index 00000000..cd56c6bc --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/iam.tf @@ -0,0 +1,35 @@ +module "assumable_role_airflow" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "3.14.0" + create_role = true + role_name = "${data.plural_cluster.cluster.name}-${var.role_name}" + provider_url = replace(data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer, "https://", "") + role_policy_arns = [module.s3_buckets.policy_arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:${var.namespace}:${var.dagster_serviceaccount}"] +} + +resource "aws_iam_user" "dagster" { + name = "${data.plural_cluster.cluster.name}-dagster" +} + +resource "aws_iam_access_key" "dagster" { + user = aws_iam_user.dagster.name +} + +resource "aws_iam_policy_attachment" "dagster-user" { + name = "${data.plural_cluster.cluster.name}-dagster-policy" + users = [aws_iam_user.dagster.name] + policy_arn = module.s3_buckets.policy_arn +} + +resource "kubernetes_secret" "dagster_s3_secret" { + metadata { + name = "dagster-aws-env" + namespace = kubernetes_namespace.dagster.id + } + + data = { + "AWS_ACCESS_KEY_ID" = aws_iam_access_key.dagster.id + "AWS_SECRET_ACCESS_KEY" = aws_iam_access_key.dagster.secret + } +} \ No newline at end of file diff --git a/catalogs/data/dagster/terraform/aws/main.tf b/catalogs/data/dagster/terraform/aws/main.tf new file mode 100644 index 00000000..7891c1e1 --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/main.tf @@ -0,0 +1,21 @@ +resource "kubernetes_namespace" "dagster" { + metadata { + name = var.namespace + labels = { + "app.kubernetes.io/managed-by" = "plural" + "app.plural.sh/name" = "dagster" + "platform.plural.sh/sync-target" = "pg" + } + } +} + +module "s3_buckets" { + source = "github.com/pluralsh/module-library//terraform/s3-buckets?ref=bucket-protection" + bucket_names = [var.dagster_bucket] + policy_prefix = "dagster" + force_destroy = var.force_destroy_bucket +} + +data "aws_eks_cluster" "cluster" { + name = var.cluster_name +} diff --git a/catalogs/data/dagster/terraform/aws/outputs.tf b/catalogs/data/dagster/terraform/aws/outputs.tf new file mode 100644 index 00000000..2a306c11 --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/outputs.tf @@ -0,0 +1,3 @@ +output "iam_user" { + value = aws_iam_user.dagster +} \ No newline at end of file diff --git a/catalogs/data/dagster/terraform/aws/plural.tf b/catalogs/data/dagster/terraform/aws/plural.tf new file mode 100644 index 00000000..c30d03d4 --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/plural.tf @@ -0,0 +1,3 @@ +data "plural_cluster" "cluster" { + handle = var.cluster_name +} diff --git a/catalogs/data/dagster/terraform/aws/postgres.tf b/catalogs/data/dagster/terraform/aws/postgres.tf new file mode 100644 index 00000000..6fc1583a --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/postgres.tf @@ -0,0 +1,18 @@ +data "aws_iam_role" "postgres" { + name = "${data.plural_cluster.cluster.name}-postgres" +} + +resource "kubernetes_service_account" "postgres" { + metadata { + name = "postgres-pod" + namespace = var.namespace + + annotations = { + "eks.amazonaws.com/role-arn" = data.aws_iam_role.postgres.arn + } + } + + depends_on = [ + kubernetes_namespace.dagster + ] +} \ No newline at end of file diff --git a/catalogs/data/dagster/terraform/aws/variables.tf b/catalogs/data/dagster/terraform/aws/variables.tf new file mode 100644 index 00000000..81f2e3b5 --- /dev/null +++ b/catalogs/data/dagster/terraform/aws/variables.tf @@ -0,0 +1,30 @@ +variable "namespace" { + type = string + default = "dagster" +} + +variable "cluster_name" { + type = string + default = "{{ context.cluster }}" +} + +variable "dagster_bucket" { + type = string + default = "{{ context.bucket }}" +} + +variable "dagster_serviceaccount" { + type = string + default = "dagster" +} + +variable "role_name" { + type = string + default = "dagster" +} + +variable "force_destroy_bucket" { + type = bool + default = true + description = "If true, the bucket will be deleted even if it contains objects." +} \ No newline at end of file diff --git a/setup/catalogs/data/dagster.yaml b/setup/catalogs/data/dagster.yaml index 5906a50a..d10c2871 100644 --- a/setup/catalogs/data/dagster.yaml +++ b/setup/catalogs/data/dagster.yaml @@ -14,8 +14,14 @@ spec: - source: README.md destination: documentation/dagster/README.md external: true - - source: helmrepository.yaml - destination: "bootstrap/apps/dagster/{{ context.cluster }}/helmrepository.yaml" + - source: dagster.yaml + destination: "bootstrap/apps/dagster/{{ context.cluster }}/dagster.yaml" + external: true + - source: helm + destination: helm/dagster/{{ context.cluster }} + external: true + - source: "terraform/{{ context.cloud }}" + destination: "terraform/apps/dagster/{{ context.cluster }}" external: true repositoryRef: name: scaffolds @@ -34,6 +40,9 @@ spec: documentation: The cloud you want to deploy to. values: - aws + - name: bucket + type: STRING + documentation: The name of the bucket you want to use. This must be globally unique. - name: hostname type: STRING documentation: The DNS name you want to host Dagster under.