Merge pull request #440 from cagov/status_fixes

Fix data status incrementality
cagov · Oct 23, 2024 · 81e6607 · 81e6607
2 parents f58f4f2 + 484095e
commit 81e6607
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 19 deletions.
diff --git a/transform/models/intermediate/diagnostics/int_diagnostics__detector_status.sql b/transform/models/intermediate/diagnostics/int_diagnostics__detector_status.sql
@@ -9,6 +9,7 @@
 with
 source as (
     select * from {{ ref('int_diagnostics__samples_per_detector') }}
+    where {{ make_model_incremental('sample_date') }}
 ),
 
 detector_meta as (
@@ -17,7 +18,7 @@ detector_meta as (
 
 set_assgnmt as (
     select * from {{ ref('int_diagnostics__det_diag_set_assignment') }}
-    where {{ make_model_incremental('active_date') }}
+    where active_date between (select min(sample_date) from source) and (select max(sample_date) from source)
 ),
 
 assignment_with_meta as (

diff --git a/transform/models/intermediate/diagnostics/int_diagnostics__no_data_status.sql b/transform/models/intermediate/diagnostics/int_diagnostics__no_data_status.sql
@@ -7,9 +7,16 @@
 ) }}
 
 with
+source as (
+    select
+        sample_date as active_date,
+        * exclude sample_date
+    from {{ ref('int_diagnostics__samples_per_detector') }}
+    where {{ make_model_incremental('active_date') }}
+),
+
 detector_meta as (
     select * from {{ ref('int_vds__active_detectors') }}
-    where {{ make_model_incremental('active_date') }}
 ),
 
 station_meta as (
@@ -25,36 +32,32 @@ equipment_meta as (
         dm.*,
         sm.controller_id,
         sm.name,
-        sm.angle
+        sm.angle,
+        cm.line_num,
+        cm.stn_address,
+        cm.controller_type
     from detector_meta as dm
     inner join station_meta as sm
         on
             dm.station_id = sm.station_id
             and {{ get_scd_2_data('dm.active_date','sm._valid_from','sm._valid_to') }}
-),
-
-equipment_all_meta as (
-    select
-        em.*,
-        cm.line_num,
-        cm.stn_address,
-        cm.controller_type
-    from equipment_meta as em
     inner join controller_meta as cm
         on
-            em.controller_id = cm.controller_id
-            and {{ get_scd_2_data('em.active_date','cm._valid_from','cm._valid_to') }}
+            sm.controller_id = cm.controller_id
+            and {{ get_scd_2_data('dm.active_date','cm._valid_from','cm._valid_to') }}
+    -- Constrain dates to not exceed those in the samples_per_detector model.
+    where dm.active_date between (select min(active_date) from source) and (select max(active_date) from source)
 ),
 
 equipment_with_samples as (
     select
-        eam.*,
+        em.*,
         source.sample_ct
-    from equipment_all_meta as eam
-    left join {{ ref('int_diagnostics__samples_per_detector') }} as source
+    from equipment_meta as em
+    left join source
         on
-            eam.detector_id = source.detector_id
-            and eam.active_date = source.sample_date
+            em.detector_id = source.detector_id
+            and em.active_date = source.active_date
 ),
 
 district_feed_check as (