Merge branch 'sebastian/prod-2981-set-up-catalog-pipeline' of github.…

…com:pluralsh/scaffolds into sebastian/prod-2981-set-up-catalog-pipeline
pluralsh · Dec 17, 2024 · c3aecd1 · c3aecd1
2 parents 826ed9b + e558ea2
commit c3aecd1
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 45 deletions.
diff --git a/catalogs/data/dagster/dagster-aws-env.yaml b/catalogs/data/dagster/dagster-aws-env.yaml
@@ -0,0 +1,25 @@
+apiVersion: deployments.plural.sh/v1alpha1
+kind: ServiceDeployment
+metadata:
+  name: dagster-aws-env-{{ context.cluster }}
+  namespace: apps
+spec:
+  namespace: dagster
+  git:
+    folder: services/apps/dagster
+    ref: main
+  repositoryRef:
+    kind: GitRepository
+    name: infra
+    namespace: infra
+  imports:
+    - stackRef:
+        name: dagster-{{ context.cluster }}
+        namespace: apps
+  configuration:
+    cluster: {{ context.cluster }}
+    hostname: {{ context.hostname }}
+  clusterRef:
+    kind: Cluster
+    name: {{ context.cluster }}
+    namespace: infra
diff --git a/catalogs/data/dagster/helm/dagster.yaml.liquid b/catalogs/data/dagster/helm/dagster.yaml.liquid
@@ -2,55 +2,61 @@
 {% assign imports_dagster_key =  'dagster-' | append: configuration.cluster %}
 
 dagsterWebserver:
-    annotations:
-        security.plural.sh/oauth-env-secret: dagster-proxy-config
-      {% if configuration["basicAuth"] %}
-        security.plural.sh/htpasswd-secret: httpaswd-users
-      {% endif %}
-    labels:
-        security.plural.sh/inject-oauth-sidecar: "true"
+  annotations:
+    security.plural.sh/oauth-env-secret: dagster-proxy-config
+    {% if configuration["basicAuth"] %}
+    security.plural.sh/htpasswd-secret: httpaswd-users
+    {% endif %}
+  labels:
+    security.plural.sh/inject-oauth-sidecar: "true"
 
 postgresql:
-    enabled: false
-    postgresqlHost: {{ imports[imports_dagster_key].postgres_host }}
-    postgresqlUsername: dagster
-    postgresqlPassword: {{ imports[imports_dagster_key].postgres_password }}
-    postgresqlDatabase: dagster
-    postgresqlParams:
-        sslmode: require
+  enabled: false
+  postgresqlHost: {{ imports[imports_dagster_key].postgres_host }}
+  postgresqlUsername: dagster
+  postgresqlPassword: {{ imports[imports_dagster_key].postgres_password }}
+  postgresqlDatabase: dagster
+  postgresqlParams:
+    sslmode: require
+
+computeLogManager:
+  type: S3ComputeLogManager
+  config:
+    s3ComputeLogManager:
+      bucket: {{ configuration.bucket }}
 
 ingress:
-    enabled: true
-    ingressClassName: nginx
-    dagsterWebserver:
-        host: {{ configuration.hostname }}
-        tls:
-            enabled: true
-            secretName: dagster-tls
-        precedingPaths:
-            - path: /.*
-              pathType: ImplementationSpecific
-              serviceName: dagster-oauth2-proxy
-              servicePort: http-oauth
-    annotations:
-        kubernetes.io/tls-acme: "true"
-        cert-manager.io/cluster-issuer: letsencrypt-prod
-        # Extend timeout to allow long running queries.
-        nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
-        nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
-        nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
-        nginx.ingress.kubernetes.io/use-regex: "true"
+  enabled: true
+  ingressClassName: nginx
+  dagsterWebserver:
+    host: {{ configuration.hostname }}
+    tls:
+      enabled: true
+      secretName: dagster-tls
+    precedingPaths:
+      - path: /.*
+        pathType: ImplementationSpecific
+        serviceName: dagster-oauth2-proxy
+        servicePort: http-oauth
+  annotations:
+    kubernetes.io/tls-acme: "true"
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    # Extend timeout to allow long running queries.
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+    nginx.ingress.kubernetes.io/use-regex: "true"
 
 dagster-user-deployments:
-    enabled: true
-    deployments:
-        - name: "k8s-example-user-code-1"
-          image:
-              repository: "docker.io/dagster/user-code-example"
-              tag: latest
-              pullPolicy: Always
-          dagsterApiGrpcArgs:
-              - "--python-file"
-              - "/example_project/example_repo/repo.py"
-          port: 3030
+  enabled: true
+  deployments:
+    - name: "k8s-example-user-code-1"
+      image:
+        repository: "docker.io/dagster/user-code-example"
+        tag: latest
+        pullPolicy: Always
+      dagsterApiGrpcArgs:
+        - "--python-file"
+        - "/example_project/example_repo/repo.py"
+      port: 3030
 {% endraw %}
diff --git a/catalogs/data/dagster/services/dagster-aws-env.yaml.liquid b/catalogs/data/dagster/services/dagster-aws-env.yaml.liquid
@@ -0,0 +1,11 @@
+{% raw %}
+{% assign imports_dagster_key =  'dagster-' | append: configuration.cluster %}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: dagster-aws-env
+  namespace: dagster
+stringData:
+  AWS_ACCESS_KEY_ID: {{ imports[imports_dagster_key].access_key_id }}
+  AWS_SECRET_ACCESS_KEY: {{ imports[imports_dagster_key].secret_access_key }}
+{% endraw %}
diff --git a/setup/catalogs/data/dagster.yaml b/setup/catalogs/data/dagster.yaml
@@ -26,6 +26,9 @@ spec:
       - source: oauth-proxy-config.yaml
         destination: "bootstrap/apps/dagster/{{ context.cluster }}/oauth-proxy-config.yaml"
         external: true
+      - source: services/dagster-aws-env.yaml.liquid
+        destination: services/apps/dagster/dagster-aws-env.yaml.liquid
+        external: true
   repositoryRef:
     name: scaffolds
   catalogRef: