feat: Initial service catalog setup

.github/workflows/test-catalogs.yaml

@@ -18,6 +18,6 @@ jobs:
       - uses: extractions/setup-just@v2
       - uses: pluralsh/[email protected]
         with:
-          vsn: 0.9.24
+          vsn: 0.11.0
 
       - run: just test

catalogs/data/airbyte/README.md

@@ -6,7 +6,7 @@ This is a baseline, prod ready airbyte installation using Plural.  It includes a
 * RDS/Google Cloud Sql, Azure Flexible Server to handle postgres. This gives you a robust RDBMS service to hold airbyte's core transactional data.
 * Plural OIDC to handle authentication to Airbyte. Airbyte does not support this natively, and so we use oauth-proxy as a middleware to handle authentication.
 
-In addtion, there are a few common customizations you might want to do.
+In addition, there are a few common customizations you might want to do.
 
 ## Configure Basic Auth
 
@@ -80,4 +80,4 @@ when building your application.
 
 ## Contributing
 
-If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds 
+If there are any features or documentation you'd like to add to this setup, please feel free to contribute back at https://github.com/pluralsh/scaffolds 

catalogs/data/airbyte/airbyte-raw-servicedeployment.yaml

@@ -0,0 +1,20 @@
+apiVersion: deployments.plural.sh/v1alpha1
+kind: ServiceDeployment
+metadata:
+  name: airbyte-raw-{{ context.cluster }}
+  namespace: apps
+spec:
+  namespace: airbyte
+  git:
+    folder: services/apps/airbyte
+    ref: main
+  repositoryRef:
+    kind: GitRepository
+    name: infra
+    namespace: infra
+  configuration:
+    hostname: {{ context.hostname }}
+  clusterRef:
+    kind: Cluster
+    name: {{ context.cluster }}
+    namespace: infra

catalogs/data/airbyte/airbyte-servicedeployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: deployments.plural.sh/v1alpha1
+kind: ServiceDeployment
+metadata:
+  name: airbyte-{{ context.cluster }}
+  namespace: apps
+spec:
+  namespace: airbyte
+  git:
+    folder: helm/airbyte/{{ context.cluster }}
+    ref: main
+  repositoryRef:
+    kind: GitRepository
+    name: infra
+    namespace: infra
+  helm:
+    version: "1.x.x"
+    chart: airbyte
+    release: airbyte
+    ignoreHooks: false
+    url: https://airbytehq.github.io/helm-charts
+    valuesFiles:
+    - airbyte.yaml.liquid
+  imports:
+  - stackRef:
+      name: airbyte-{{ context.cluster }}
+      namespace: apps
+  configuration:
+    cluster: {{ context.cluster }}
+    hostname: {{ context.hostname }}
+    bucket: {{ context.bucket }}
+    region: {{ context.region }}
+  clusterRef:
+    kind: Cluster
+    name: {{ context.cluster }}
+    namespace: infra

test/outputs/catalogs/data/airbyte/stack.yaml.liquid → catalogs/data/airbyte/airbyte-stack.yaml

@@ -2,20 +2,21 @@ apiVersion: deployments.plural.sh/v1alpha1
 kind: InfrastructureStack
 metadata:
   name: airbyte-{{ context.cluster }}
+  namespace: apps
 spec:
   detach: false
   type: TERRAFORM
   approval: true
   manageState: true
   actor: [email protected]
-  configuration:
-    version: '1.8'
+  git:
+    ref: main
+    folder: terraform/apps/airbyte/{{ context.cluster }}
   repositoryRef:
     name: infra
     namespace: infra
+  configuration:
+    version: '1.8'
   clusterRef:
-    name: mgmt
+    name: {{ context.stackCluster }}
     namespace: infra
-  git:
-    ref: main
-    folder: terraform/apps/airbyte/{{ context.cloud }}

catalogs/data/airbyte/helm/airbyte.yaml.liquid

@@ -0,0 +1,57 @@
+{% raw %}
+{% assign imports_airbyte_key =  'airbyte-' | append: configuration.cluster %}
+
+global:
+  deploymentMode: oss
+  edition: community
+
+  airbyteUrl: {{ configuration.hostname }}
+
+  storage:
+    type: S3
+    storageSecretName: airbyte-airbyte-secrets
+    s3:
+      region: {{ configuration.region }}
+      authenticationType: credentials
+      accessKeyId: {{ imports[imports_airbyte_key].access_key_id }}
+      accessKeyIdSecretKey: AWS_ACCESS_KEY_ID
+      secretAccessKey: {{ imports[imports_airbyte_key].secret_access_key }}
+      secretAccessKeySecretKey: AWS_SECRET_ACCESS_KEY
+    bucket:
+      log: {{ configuration.bucket }}
+      state: {{ configuration.bucket }}
+      workloadOutput: {{ configuration.bucket }}
+
+  database:
+    type: external
+    database: airbyte
+    host: {{ imports[imports_airbyte_key].postgres_host }}
+    port: "5432"
+    secretName: airbyte-airbyte-secrets
+    user: airbyte
+    userSecretKey: DATABASE_USER
+    password: {{ imports[imports_airbyte_key].postgres_password }}
+    passwordSecretKey: DATABASE_PASSWORD
+
+postgresql:
+  enabled: false
+
+externalDatabase:
+  database: airbyte
+  host: {{ imports[imports_airbyte_key].postgres_host }}
+  user: airbyte
+  existingSecret: ~
+  password: {{ imports[imports_airbyte_key].postgres_password }}
+  port: 5432
+
+webapp:
+  ingress:
+    enabled: false
+  podAnnotations:
+    security.plural.sh/oauth-env-secret: airbyte-proxy-config
+    {% if configuration["basicAuth"] %}
+    security.plural.sh/htpasswd-secret: httpaswd-users
+    {% endif %}
+  podLabels:
+    security.plural.sh/inject-oauth-sidecar: "true"
+{% endraw %}

catalogs/data/airbyte/helm/oauth-proxy-config.yaml.liquid

@@ -0,0 +1,28 @@
+{% raw %}
+{% assign imports_airbyte_key =  'airbyte-' | append: configuration.cluster %}
+
+service:
+  name: airbyte-oauth2-proxy
+  selector:
+    app.kubernetes.io/instance: airbyte
+    app.kubernetes.io/name: webapp
+
+secret:
+  clientID: {{ imports[imports_airbyte_key].oidc_client_id }}
+  clientSecret: {{ imports[imports_airbyte_key].oidc_client_secret }}
+  cookieSecret: {{ imports[imports_airbyte_key].oidc_cookie_secret }}
+  issuer: https://oidc.plural.sh/
+  upstream: http://localhost:8080
+  name: airbyte-proxy-config
+  env:
+    OAUTH2_PROXY_UPSTREAM_TIMEOUT: '120s'
+
+{% if configuration["basicAuth"] %}
+{% assign basicAuth = configuration["basicAuth"] | from_json %}
+users:
+{% for user in basicAuth %}
+  {{ user[0] }}: {{ user[1] }}
+{% endfor %}
+{% endif %}
+
+{% endraw %}

catalogs/data/airbyte/servicedeployment.yaml.liquid → catalogs/data/airbyte/oauth-proxy-config-servicedeployment.yaml

@@ -1,27 +1,30 @@
 apiVersion: deployments.plural.sh/v1alpha1
 kind: ServiceDeployment
 metadata:
-  name: airbyte-{{ context.cluster }}
-  namespace: infra
+  name: airbyte-oauth-proxy-config-{{ context.cluster }}
+  namespace: apps
 spec:
   namespace: airbyte
   git:
-    folder: helm/airbyte
+    folder: helm/airbyte/{{ context.cluster }}
     ref: main
   repositoryRef:
     kind: GitRepository
     name: infra
     namespace: infra
   helm:
     version: "x.x.x"
-    chart: airbyte
-    url: https://app.plural.sh/cm/airbyte
+    chart: oidc-config
+    url: https://pluralsh.github.io/module-library
     valuesFiles:
-    - {{ context.cluster }}.yaml.liquid
+    - oauth-proxy-config.yaml.liquid
   imports:
   - stackRef:
       name: airbyte-{{ context.cluster }}
-      namespace: infra
+      namespace: apps
+  configuration:
+    cluster: {{ context.cluster }}
+    hostname: {{ context.hostname }}
   clusterRef:
     kind: Cluster
     name: {{ context.cluster }}

catalogs/data/airbyte/services/oauth-proxy-ingress.yaml.liquid

@@ -0,0 +1,33 @@
+{% raw %}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: airbyte-webapp-proxy
+  namespace: airbyte
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    kubernetes.io/tls-acme: "true"
+    # Extend timeout to allow long running queries.
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/use-regex: "true"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: {{ configuration.hostname }}
+      http:
+        paths:
+          - backend:
+              service:
+                name: airbyte-oauth2-proxy
+                port:
+                  number: 80
+            path: /.*
+            pathType: ImplementationSpecific
+  tls:
+    - hosts:
+        - {{ configuration.hostname }}
+      secretName: airbyte-tls
+{% endraw %}

catalogs/data/airbyte/terraform/aws/iam.tf

@@ -6,7 +6,9 @@ resource "aws_iam_policy" "airbyte" {
 }
 
 resource "aws_iam_user" "airbyte" {
-  name = "${var.cluster_name}-airbyte"
+  name = "${data.plural_cluster.cluster.name}-airbyte"
+
+  depends_on = [ data.plural_cluster.cluster ]
 }
 
 resource "aws_iam_access_key" "airbyte" {
@@ -27,7 +29,9 @@ data "aws_iam_policy_document" "airbyte" {
 }
 
 resource "aws_iam_policy_attachment" "airbyte-user" {
-  name = "${var.cluster_name}-airbyte-policy"
+  name = "${data.plural_cluster.cluster.name}-airbyte-policy"
   users = [aws_iam_user.airbyte.name]
   policy_arn = aws_iam_policy.airbyte.arn
+
+  depends_on = [ data.plural_cluster.cluster ]
 }

catalogs/data/airbyte/terraform/aws/oidc.tf

@@ -1,5 +1,5 @@
 resource "random_password" "oidc_cookie" {
-  length      = 20
+  length      = 24
   min_lower   = 1
   min_numeric = 1
   min_upper   = 1
@@ -12,4 +12,4 @@ resource "plural_oidc_provider" "airbyte" {
   type = "PLURAL"
   description = "OIDC provider for airbyte deployed to the {{ context.cluster }} cluster"
   redirect_uris = ["https://{{ context.hostname }}/oauth2/callback"]
-}
+}