Lilian dbt training

transform/models/_models.yml

@@ -1,14 +1,26 @@
 version: 2
 
 models:
-  - name: sample_data
-    description: A sample model
+  - name: stg_pems__station_meta
+    description: reformating table columns
+    config:
+      materialized: table
+      schema: DBT_ZWU_TEST
     columns:
-      - name: name
-        description: A person's name
-      - name: city
-        description: Their residence
-      - name: license_plate
-        description: Their license plate
-      - name: email
-        description: Their email
+      - name: META_DATE
+        description: date
+      - name: ID
+        description: "station ID"
+        tests:
+          - not_null
+
+  - name: int_pems__stations_per_county_counted
+    description: intermediate model with 2023 station count by county
+    columns:
+      - name: County
+        description: county id
+        tests:
+          - not_null
+          - unique
+      - name: stat_counts
+        description: number of stations in this county

transform/models/_sources.yml

@@ -0,0 +1,85 @@
+version: 2
+
+sources:
+  - name: CLEARINGHOUSE
+    database: RAW_PRD
+    schema: CLEARINGHOUSE
+    description: Station Information
+    tables:
+      - name: STATION_META
+        description: meta data for stations
+        columns:
+          - name: FILENAME
+            description: file name
+          - name: ID
+            description: station ID
+          - name: FWY
+            description: freeway number
+          - name: DIR
+            description: traffic flow direction
+          - name: DISTRICT
+            description: district number
+          - name: COUNTY
+            description: county name
+          - name: CITY
+            description: city name
+          - name: STATE_PM
+            description: state postmile
+          - name: ABS_PM
+            description: absolute postmile
+          - name: LATITUDE
+            description: coordinate-latitude
+          - name: LONGITUDE
+            description: coordinate-longitude
+          - name: LENGTH
+            description: distance
+          - name: TYPE
+            description: <>
+          - name: LANES
+            description: number of lanes
+          - name: NAME
+            description: station name
+          - name: USER_ID_1
+            description: user id 1
+          - name: USER_ID_2
+            description: <>
+          - name: USER_ID_3
+            description: <>
+          - name: USER_ID_4
+            description: <>
+      - name: STATION_RAW
+        description: Raw traffic data collected at each station
+        columns:
+          - name: FILENAME
+            description: file name
+          - name: SAMPLE_TIMESTAMP
+            description: timestamp
+          - name: SAMPLE_DATE
+            description: date
+          - name: ID
+            description: station ID
+          - name: FLOW_1
+            description: flow in lane 1
+          - name: OCCUPANCY_1
+            description: lane 1 occupancy
+          - name: SPEED_1
+            description: average travel speed in lane 1
+          - name: FLOW_2
+            description: flow in lane 2
+          - name: OCCUPANCY_2
+            description: lane 2 occupancy
+          - name: SPEED_2
+            description: average travel speed in lane 2
+          - name: FLOW_3
+            description: flow in lane 3
+          - name: OCCUPANCY_3
+            description: lane 3 occupancy
+          - name: SPEED_3
+            description: average travel speed in lane 3
+      - name: STATION_STATUS
+        description: station status information
+        columns:
+          - name: FILENAME
+            description: file name
+          - name: CONTENT
+            description: content

transform/models/intermediate/int_pems__stations_per_county_counted.sql

@@ -0,0 +1,24 @@
+with
+
+source as (
+    select * from {{ ref("stg_pems__station_meta") }}
+),
+
+county_data as (
+    select
+        county,
+        Count(id) as sta_counts
+    from source
+    where Year(meta_date) = 2023
+    group by county
+
+)
+
+select *
+from county_data
+order by sta_counts desc
+
+
+-- select *
+-- from stg_pems__station_meta
+-- limit 10

transform/models/staging/stg_pems__station_meta.sql

@@ -0,0 +1,24 @@
+select
+    DATE_FROM_PARTS(
+        SUBSTRING(FILENAME, 18, 4),
+        SUBSTRING(FILENAME, 23, 2),
+        LEFT(RIGHT(FILENAME, 6), 2)
+    ) as META_DATE,
+    ID,
+    FWY as FREEWAY,
+    DIR as DIRECTION,
+    DISTRICT,
+    COUNTY,
+    CITY,
+    STATE_PM as STATE_POSTMILE,
+    ABS_PM as ABSOLUTE_POSTMILE,
+    LATITUDE,
+    LONGITUDE,
+    LENGTH,
+    TYPE,
+    LANES,
+    NAME
+
+-- from RAW_PRD.CLEARINGHOUSE.STATION_META
+
+from {{ source('CLEARINGHOUSE','STATION_META') }}