Merge branch 'sebastian/prod-2981-set-up-catalog-pipeline' of github.…

…com:pluralsh/scaffolds into sebastian/prod-2981-set-up-catalog-pipeline

catalogs/data/dagster/dagster.yaml

@@ -0,0 +1,55 @@
+apiVersion: deployments.plural.sh/v1alpha1
+kind: InfrastructureStack
+metadata:
+  name: dagster-{{ context.cluster }}
+  namespace: apps
+spec:
+  detach: false
+  type: TERRAFORM
+  approval: true
+  manageState: true
+  actor: [email protected]
+  git:
+    ref: main
+    folder: terraform/apps/dagster/{{ context.cluster }}
+  repositoryRef:
+    name: infra
+    namespace: infra
+  configuration:
+    version: '1.8'
+  clusterRef:
+    name: {{ context.cluster }}
+    namespace: infra
+---
+apiVersion: deployments.plural.sh/v1alpha1
+kind: ServiceDeployment
+metadata:
+  name: dagster-{{ context.cluster }}
+  namespace: apps
+spec:
+  namespace: dagster
+  git:
+    folder: helm/dagster/{{ context.cluster }}
+    ref: main
+  repositoryRef:
+    kind: GitRepository
+    name: infra
+    namespace: infra
+  helm:
+    url: https://dagster-io.github.io/helm
+    version: "1.x.x"
+    chart: dagster
+    valuesFiles:
+    - dagster.yaml.liquid
+  imports:
+  - stackRef:
+      name: dagster-{{ context.cluster }}
+      namespace: infra
+  configuration:
+    cluster: {{ context.cluster }}
+    hostname: {{ context.hostname }}
+    bucket: {{ context.bucket }}
+  clusterRef:
+    kind: Cluster
+    name: {{ context.cluster }}
+    namespace: infra

catalogs/data/dagster/terraform/aws/iam.tf

@@ -0,0 +1,35 @@
+module "assumable_role_airflow" {
+  source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  version                       = "3.14.0"
+  create_role                   = true
+  role_name                     = "${data.plural_cluster.cluster.name}-${var.role_name}"
+  provider_url                  = replace(data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer, "https://", "")
+  role_policy_arns              = [module.s3_buckets.policy_arn]
+  oidc_fully_qualified_subjects = ["system:serviceaccount:${var.namespace}:${var.dagster_serviceaccount}"]
+}
+
+resource "aws_iam_user" "dagster" {
+  name = "${data.plural_cluster.cluster.name}-dagster"
+}
+
+resource "aws_iam_access_key" "dagster" {
+  user = aws_iam_user.dagster.name
+}
+
+resource "aws_iam_policy_attachment" "dagster-user" {
+  name = "${data.plural_cluster.cluster.name}-dagster-policy"
+  users = [aws_iam_user.dagster.name]
+  policy_arn = module.s3_buckets.policy_arn
+}
+
+resource "kubernetes_secret" "dagster_s3_secret" {
+  metadata {
+    name = "dagster-aws-env"
+    namespace = kubernetes_namespace.dagster.id
+  }
+
+  data = {
+    "AWS_ACCESS_KEY_ID" = aws_iam_access_key.dagster.id
+    "AWS_SECRET_ACCESS_KEY" = aws_iam_access_key.dagster.secret
+  }
+}

catalogs/data/dagster/terraform/aws/main.tf

@@ -0,0 +1,21 @@
+resource "kubernetes_namespace" "dagster" {
+  metadata {
+    name = var.namespace
+    labels = {
+      "app.kubernetes.io/managed-by" = "plural"
+      "app.plural.sh/name" = "dagster"
+      "platform.plural.sh/sync-target" = "pg"
+    }
+  }
+}
+
+module "s3_buckets" {
+  source        = "github.com/pluralsh/module-library//terraform/s3-buckets?ref=bucket-protection"
+  bucket_names  = [var.dagster_bucket]
+  policy_prefix = "dagster"
+  force_destroy = var.force_destroy_bucket
+}
+
+data "aws_eks_cluster" "cluster" {
+  name = var.cluster_name
+}

catalogs/data/dagster/terraform/aws/outputs.tf

@@ -0,0 +1,3 @@
+output "iam_user" {
+  value = aws_iam_user.dagster
+}

catalogs/data/dagster/terraform/aws/plural.tf

@@ -0,0 +1,3 @@
+data "plural_cluster" "cluster" {
+  handle = var.cluster_name
+}

catalogs/data/dagster/terraform/aws/postgres.tf

@@ -0,0 +1,18 @@
+data "aws_iam_role" "postgres" {
+  name = "${data.plural_cluster.cluster.name}-postgres"
+}
+
+resource "kubernetes_service_account" "postgres" {
+  metadata {
+    name      = "postgres-pod"
+    namespace = var.namespace
+
+    annotations = {
+      "eks.amazonaws.com/role-arn" = data.aws_iam_role.postgres.arn
+    }
+  }
+
+  depends_on = [
+    kubernetes_namespace.dagster
+  ]
+}

catalogs/data/dagster/terraform/aws/variables.tf

@@ -0,0 +1,30 @@
+variable "namespace" {
+  type = string
+  default = "dagster"
+}
+
+variable "cluster_name" {
+  type = string
+  default = "{{ context.cluster }}"
+}
+
+variable "dagster_bucket" {
+  type = string
+  default = "{{ context.bucket }}"
+}
+
+variable "dagster_serviceaccount" {
+  type = string
+  default = "dagster"
+}
+
+variable "role_name" {
+  type = string
+  default = "dagster"
+}
+
+variable "force_destroy_bucket" {
+  type        = bool
+  default     = true
+  description = "If true, the bucket will be deleted even if it contains objects."
+}

setup/catalogs/data/dagster.yaml

@@ -14,8 +14,14 @@ spec:
       - source: README.md
         destination: documentation/dagster/README.md
         external: true
-      - source: helmrepository.yaml
-        destination: "bootstrap/apps/dagster/{{ context.cluster }}/helmrepository.yaml"
+      - source: dagster.yaml
+        destination: "bootstrap/apps/dagster/{{ context.cluster }}/dagster.yaml"
+        external: true
+      - source: helm
+        destination: helm/dagster/{{ context.cluster }}
+        external: true
+      - source: "terraform/{{ context.cloud }}"
+        destination: "terraform/apps/dagster/{{ context.cluster }}"
         external: true
   repositoryRef:
     name: scaffolds
@@ -34,6 +40,9 @@ spec:
       documentation: The cloud you want to deploy to.
       values:
         - aws
+    - name: bucket
+      type: STRING
+      documentation: The name of the bucket you want to use. This must be globally unique.
     - name: hostname
       type: STRING
       documentation: The DNS name you want to host Dagster under.