-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Created a model that performs various checks if data is missing that …
…is used in the detector status diagnostics tests. This results in more detailed statuses when data is not being relayed from detectors. Also, reorganized joins in the detector_status model to pass PR checks.
- Loading branch information
1 parent
1c972b5
commit ac7d895
Showing
2 changed files
with
237 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
170 changes: 170 additions & 0 deletions
170
transform/models/intermediate/diagnostics/int_diagnostics__no_data_status.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| {{ config( | ||
| materialized="incremental", | ||
| cluster_by=["active_date"], | ||
| unique_key=["detector_id", "active_date"], | ||
| on_schema_change="sync_all_columns", | ||
| snowflake_warehouse=get_snowflake_refresh_warehouse(small="XL") | ||
| ) }} | ||
|
|
||
| with | ||
| detector_meta as ( | ||
| select * from {{ ref('int_vds__active_detectors') }} | ||
| where {{ make_model_incremental('active_date') }} | ||
| ), | ||
|
|
||
| station_meta as ( | ||
| select * from {{ ref('int_vds__station_config') }} | ||
| ), | ||
|
|
||
| controller_meta as ( | ||
| select * from {{ ref('int_vds__controller_config') }} | ||
| ), | ||
|
|
||
| equipment_meta as ( | ||
| select | ||
| dm.*, | ||
| sm.controller_id, | ||
| sm.name, | ||
| sm.angle | ||
| from detector_meta as dm | ||
| inner join station_meta as sm | ||
| on | ||
| dm.station_id = sm.station_id | ||
| and {{ get_scd_2_data('dm.active_date','sm._valid_from','sm._valid_to') }} | ||
| ), | ||
|
|
||
| equipment_all_meta as ( | ||
| select | ||
| em.*, | ||
| cm.line_num, | ||
| cm.stn_address, | ||
| cm.controller_type | ||
| from equipment_meta as em | ||
| inner join controller_meta as cm | ||
| on | ||
| em.controller_id = cm.controller_id | ||
| and {{ get_scd_2_data('em.active_date','cm._valid_from','cm._valid_to') }} | ||
| ), | ||
|
|
||
| equipment_with_samples as ( | ||
| select | ||
| eam.*, | ||
| source.sample_ct | ||
| from equipment_all_meta as eam | ||
| left join {{ ref('int_diagnostics__samples_per_detector') }} as source | ||
| on | ||
| eam.detector_id = source.detector_id | ||
| and eam.active_date = source.sample_date | ||
| ), | ||
|
|
||
| district_feed_check as ( | ||
| select | ||
| ews.active_date, | ||
| ews.district, | ||
| case | ||
| when (count_if(ews.sample_ct > 0)) > 0 then 'Yes' | ||
| else 'No' | ||
| end as district_feed_working | ||
| from equipment_with_samples as ews | ||
| inner join {{ ref('districts') }} as d | ||
| on ews.district = d.district_id | ||
| group by ews.active_date, ews.district | ||
| ), | ||
|
|
||
| line_feed_check as ( | ||
| select | ||
| ews.active_date, | ||
| ews.district, | ||
| ews.line_num, | ||
| case | ||
| when ews.line_num is null then 'Yes' | ||
| when (count_if(ews.sample_ct > 0)) > 0 then 'Yes' | ||
| else 'No' | ||
| end as line_num_working | ||
| from equipment_with_samples as ews | ||
| group by ews.active_date, ews.district, ews.line_num | ||
| ), | ||
|
|
||
| controller_feed_check as ( | ||
| select | ||
| ews.active_date, | ||
| ews.district, | ||
| ews.controller_id, | ||
| case | ||
| when ews.controller_id is null then 'Yes' | ||
| when (count_if(ews.sample_ct > 0)) > 0 then 'Yes' | ||
| else 'No' | ||
| end as controller_feed_working | ||
| from equipment_with_samples as ews | ||
| group by ews.active_date, ews.district, ews.controller_id | ||
| ), | ||
|
|
||
| station_feed_check as ( | ||
| select | ||
| ews.active_date, | ||
| ews.district, | ||
| ews.station_id, | ||
| case | ||
| when ews.station_id is null then 'Yes' | ||
| when (count_if(ews.sample_ct > 0)) > 0 then 'Yes' | ||
| else 'No' | ||
| end as station_feed_working | ||
| from equipment_with_samples as ews | ||
| group by ews.active_date, ews.district, ews.station_id | ||
| ), | ||
|
|
||
| detector_feed_check as ( | ||
| select | ||
| ews.active_date, | ||
| ews.detector_id, | ||
| case | ||
| when ews.detector_id is null then 'Yes' | ||
| when (count_if(ews.sample_ct > 0)) > 0 then 'Yes' | ||
| else 'No' | ||
| end as detector_feed_working | ||
| from equipment_with_samples as ews | ||
| group by ews.active_date, ews.detector_id | ||
| ), | ||
|
|
||
| data_feed_check as ( | ||
| select | ||
| ews.active_date, | ||
| ews.district, | ||
| ews.line_num, | ||
| ews.controller_id, | ||
| ews.station_id, | ||
| ews.detector_id, | ||
| ews.sample_ct, | ||
| dfc.district_feed_working, | ||
| lfc.line_num_working, | ||
| cfc.controller_feed_working, | ||
| sfc.station_feed_working, | ||
| detfc.detector_feed_working | ||
| from equipment_with_samples as ews | ||
| left join district_feed_check as dfc | ||
| on | ||
| ews.active_date = dfc.active_date | ||
| and ews.district = dfc.district | ||
| left join line_feed_check as lfc | ||
| on | ||
| ews.active_date = lfc.active_date | ||
| and ews.district = lfc.district | ||
| and ews.line_num = lfc.line_num | ||
| left join controller_feed_check as cfc | ||
| on | ||
| ews.active_date = cfc.active_date | ||
| and ews.district = cfc.district | ||
| and ews.controller_id = cfc.controller_id | ||
| left join station_feed_check as sfc | ||
| on | ||
| ews.active_date = sfc.active_date | ||
| and ews.district = sfc.district | ||
| and ews.station_id = sfc.station_id | ||
| left join detector_feed_check as detfc | ||
| on | ||
| ews.active_date = detfc.active_date | ||
| and ews.detector_id = detfc.detector_id | ||
| ) | ||
|
|
||
| select * from data_feed_check | ||
| order by active_date, district, line_num, controller_id, station_id, detector_id |