Fix-data

.github/workflows/sqlmesh.yml

@@ -0,0 +1,41 @@
+name: SQLMesh Bot
+run-name: 🚀SQLMesh Bot 🚀
+on:
+  pull_request:
+    types:
+    - synchronize
+    - opened
+# The latest commit is the one that will be used to create the PR environment and deploy to production
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+jobs:
+  sqlmesh:
+    name: SQLMesh Actions Workflow
+    runs-on: ubuntu-latest
+    permissions:
+      # Required to access code in PR
+      contents: write
+      # Required to post comments
+      issues: write
+      # Required to update check runs
+      checks: write
+      # Required to merge
+      pull-requests: write
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v4
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+        with:
+          ref: refs/pull/${{ github.event.issue.pull_request && github.event.issue.number || github.event.pull_request.number  }}/merge
+      - name: Install SQLMesh + Dependencies
+        run: pip install -r requirements.txt
+        shell: bash
+      - name: Run CI/CD Bot
+        run: |
+          sqlmesh_cicd -p ${{ github.workspace }} github --token ${{ secrets.GITHUB_TOKEN }} run-all
+        env: # TODO: update your GitHub secrets to include SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_PASSWORD, SNOWFLAKE_ROLE
+          SQLMESH_STATE_HOST: ${{ secrets.SQLMESH_STATE_HOST }}
+          SQLMESH_STATE_USERNAME: ${{ secrets.SQLMESH_STATE_USERNAME }}
+          SQLMESH_STATE_PASSWORD: ${{ secrets.SQLMESH_STATE_PASSWORD }}

README.md

@@ -2,8 +2,26 @@
 
 This is a simple, loveable, and complete SQLMesh demo project with the goal of running multiple, realistic scenarios in 1 minute. 
 
-This repo is going to try something a bit novel compared to your standard demo projects. This will take you through different stories that illustrate the data engineering workflow. It'll engage your inner dialogue along with look and feel of the development experience. The hope is for you to better internalize and translate these stories to your own reality. Don't worry, these stories won't be too long-winded!
+This repo is going to try something a bit novel compared to your standard demo projects. I'll take you through different stories that illustrate the data engineering workflow. It'll engage your inner dialogue along with look and feel of the development experience. The hope is for you to better internalize and translate these stories to your own reality. Don't worry, these stories won't be too long-winded!
+
+This is intentionally similar to what people may have experienced when I created this github repo at dbt Labs: [here](https://github.com/dbt-labs/jaffle_shop_duckdb)
 
 ## Story #1
 How do I go about making a big breaking change to the logic of a core sql model? 
-Context: You're tasked with a...
+Context: You're tasked with a...
+
+
+
+
+
+=========================================================================
+Portions of this project are modifications based on work created and shared by dbt Labs and used according to terms described in the Apache License, Version 2.0.
+
+The original work can be found at:
+https://github.com/dbt-labs/jaffle_shop_duckdb
+
+This project includes or is based on `jaffle_shop_duckdb` by dbt Labs which is available under a "Apache License 2.0" license.
+
+For the original work and its license, see:
+https://github.com/dbt-labs/jaffle_shop_duckdb/license
+=========================================================================

config.yaml

@@ -3,7 +3,13 @@ gateways:
         connection:
             type: duckdb
             database: db.db
-
+        state_connection:
+            type: postgres
+            host: {{ env_var('SQLMESH_STATE_HOST') }}
+            port: 5432
+            user: {{ env_var('SQLMESH_STATE_USERNAME') }}
+            password: {{ env_var('SQLMESH_STATE_PASSWORD') }}
+            database: sqlmesh_state
 default_gateway: local
 
 model_defaults:

models/customers.sql

@@ -0,0 +1,59 @@
+MODEL (
+  name demo.customers,
+  cron '@daily',
+  grain customer_id,
+  audits (UNIQUE_VALUES(columns = (
+    customer_id
+  )), NOT_NULL(columns = (
+    customer_id
+  )))
+);
+
+WITH customers AS (
+  SELECT
+    *
+  FROM demo.stg_customers
+), orders AS (
+  SELECT
+    *
+  FROM demo.stg_orders
+), payments AS (
+  SELECT
+    *
+  FROM demo.stg_payments
+), customer_orders AS (
+  SELECT
+    customer_id,
+    MIN(order_date) AS first_order,
+    MAX(order_date) AS most_recent_order,
+    COUNT(order_id) AS number_of_orders
+  FROM orders
+  GROUP BY
+    customer_id
+), customer_payments AS (
+  SELECT
+    orders.customer_id,
+    SUM(amount) AS total_amount
+  FROM payments
+  LEFT JOIN orders
+    ON payments.order_id = orders.order_id
+  GROUP BY
+    orders.customer_id
+), final AS (
+  SELECT
+    customers.customer_id,
+    customers.first_name,
+    customers.last_name,
+    customer_orders.first_order,
+    customer_orders.most_recent_order,
+    customer_orders.number_of_orders,
+    customer_payments.total_amount AS customer_lifetime_value
+  FROM customers
+  LEFT JOIN customer_orders
+    ON customers.customer_id = customer_orders.customer_id
+  LEFT JOIN customer_payments
+    ON customers.customer_id = customer_payments.customer_id
+)
+SELECT
+  *
+FROM final

models/orders.sql

@@ -0,0 +1,50 @@
+MODEL (
+  name demo.orders,
+  cron '@daily',
+  grain order_id,
+  audits (UNIQUE_VALUES(columns = (
+    order_id
+  )), NOT_NULL(columns = (
+    order_id
+  )))
+);
+
+@DEF(payment_methods, ['credit_card', 'coupon', 'bank_transfer', 'gift_card']);
+
+WITH orders AS (
+  SELECT
+    *
+  FROM demo.stg_orders
+), payments AS (
+  SELECT
+    *
+  FROM demo.stg_payments
+), order_payments AS (
+  SELECT
+    order_id,
+    @EACH(
+      @payment_methods,
+      x -> SUM(CASE WHEN payment_method = x THEN amount ELSE 0 END) AS @{x}_amount
+    ),
+    SUM(amount) AS total_amount
+  FROM payments
+  GROUP BY
+    order_id
+), final AS (
+  SELECT
+    orders.order_id,
+    orders.customer_id,
+    orders.order_date,
+    orders.status,
+    @EACH(
+      @payment_methods,
+      x -> order_payments.@{x}_amount
+    ),
+    order_payments.total_amount AS amount
+  FROM orders
+  LEFT JOIN order_payments
+    ON orders.order_id = order_payments.order_id
+)
+SELECT
+  *
+FROM final

models/seed_raw_customers.sql

@@ -0,0 +1,14 @@
+MODEL (
+  name demo.seed_raw_customers,
+  kind SEED (
+    path '../seeds/raw_customers.csv'
+  ),
+  columns (
+    id INT,
+    first_name TEXT(50),
+    last_name TEXT(50)
+  ),
+  grain (
+    id
+  )
+)

models/seed_raw_orders.sql

@@ -0,0 +1,13 @@
+MODEL (
+  name demo.seed_raw_orders,
+  kind SEED (
+    path '../seeds/raw_orders.csv'
+  ),
+  columns (
+    id INT,
+    user_id INT,
+    order_date DATE,
+    status TEXT(50)
+  ),
+  grain (id, user_id)
+)

models/seed_raw_payments.sql

@@ -0,0 +1,13 @@
+MODEL (
+  name demo.seed_raw_payments,
+  kind SEED (
+    path '../seeds/raw_payments.csv'
+  ),
+  columns (
+    id INT,
+    order_id INT,
+    payment_method TEXT(50),
+    amount INT
+  ),
+  grain (id, user_id)
+)