From 3390cdc6225a6d99c5fa272beca3e60b5e3f5754 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Thu, 29 Aug 2024 20:38:04 +0000 Subject: [PATCH 01/15] #1053 rename file --- .../function-update-open_issues.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename volumes/miovision/sql/{views/create-view-open_issues.sql => function/function-update-open_issues.sql} (100%) diff --git a/volumes/miovision/sql/views/create-view-open_issues.sql b/volumes/miovision/sql/function/function-update-open_issues.sql similarity index 100% rename from volumes/miovision/sql/views/create-view-open_issues.sql rename to volumes/miovision/sql/function/function-update-open_issues.sql From f53cc145dd3babeff8ac1c876809286108a78a12 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Thu, 29 Aug 2024 21:20:35 +0000 Subject: [PATCH 02/15] #1053 new table and function to update --- .../function/function-update-open_issues.sql | 176 ++++++++++-------- .../sql/table/create-table-open_issues.sql | 38 ++++ 2 files changed, 138 insertions(+), 76 deletions(-) create mode 100644 volumes/miovision/sql/table/create-table-open_issues.sql diff --git a/volumes/miovision/sql/function/function-update-open_issues.sql b/volumes/miovision/sql/function/function-update-open_issues.sql index c250f08f4..a90afc22b 100644 --- a/volumes/miovision/sql/function/function-update-open_issues.sql +++ b/volumes/miovision/sql/function/function-update-open_issues.sql @@ -1,7 +1,5 @@ ---DROP VIEW miovision_api.open_issues; -CREATE OR REPLACE VIEW miovision_api.open_issues AS - ---DROP VIEW miovision_api.open_issues; +--DROP FUNCTION miovision_api.update_open_issues; +CREATE OR REPLACE FUNCTION miovision_api.update_open_issues() AS WITH alerts AS ( SELECT @@ -13,80 +11,106 @@ WITH alerts AS ( AND tsrange(alerts.start_time, alerts.end_time) && tsrange(ar.range_start, ar.range_end) GROUP BY ar.uid -) +), -SELECT - ar.uid, - ar.intersection_uid, - i.id AS intersection_id, - i.api_name AS intersection_name, - ar.classification_uid, - CASE - WHEN ar.classification_uid = 2 THEN 'Bicycle TMC' - WHEN ar.classification_uid = 10 THEN 'Bicycle Approach' - WHEN ar.classification_uid IS NULL THEN 'All modes' - ELSE c.classification - END, - ar.leg, - ar.range_start::date, - (current_timestamp AT TIME ZONE 'EST5EDT')::date - ar.range_start::date AS num_days, - ar.notes, - SUM(v.volume) AS volume, - alerts.alerts -FROM miovision_api.anomalous_ranges AS ar ---keep rows with null classification_uid -LEFT JOIN miovision_api.classifications AS c USING (classification_uid) ---omit null intersection_uids. These will go under discontinuities. -JOIN miovision_api.intersections AS i USING (intersection_uid) ---find last week volume -LEFT JOIN miovision_api.volumes AS v - ON ar.intersection_uid = v.intersection_uid - --volume within the last 7 days and after AR started - AND v.datetime_bin >= ar.range_start - --prune the partitions - AND v.datetime_bin >= current_date - interval '7 days' - AND ( - ar.classification_uid = v.classification_uid - OR ar.classification_uid IS NULL - ) - AND ( - ar.leg = v.leg - OR ar.leg IS NULL - ) -LEFT JOIN alerts ON alerts.uid = ar.uid -WHERE - ar.problem_level <> 'valid-caveat' - --currently active - AND ( - ar.range_end IS NULL - OR ( - ar.notes LIKE '%identified by a daily airflow process%' - AND ar.range_end = (current_timestamp AT TIME ZONE 'EST5EDT')::date --today +open_issues AS ( + SELECT + ar.uid, + ar.intersection_uid, + i.id AS intersection_id, + i.api_name AS intersection_name, + ar.classification_uid, + CASE + WHEN ar.classification_uid = 2 THEN 'Bicycle TMC' + WHEN ar.classification_uid = 10 THEN 'Bicycle Approach' + WHEN ar.classification_uid IS NULL THEN 'All modes' + ELSE c.classification + END, + ar.leg, + ar.range_start::date, + (current_timestamp AT TIME ZONE 'EST5EDT')::date - ar.range_start::date AS num_days, + ar.notes, + SUM(v.volume) AS volume, + alerts.alerts + FROM miovision_api.anomalous_ranges AS ar + --keep rows with null classification_uid + LEFT JOIN miovision_api.classifications AS c USING (classification_uid) + --omit null intersection_uids. These will go under discontinuities. + JOIN miovision_api.intersections AS i USING (intersection_uid) + --find last week volume + LEFT JOIN miovision_api.volumes AS v + ON ar.intersection_uid = v.intersection_uid + --volume within the last 7 days and after AR started + AND v.datetime_bin >= ar.range_start + --prune the partitions + AND v.datetime_bin >= current_date - interval '7 days' + AND ( + ar.classification_uid = v.classification_uid + OR ar.classification_uid IS NULL ) - ) -GROUP BY - ar.uid, - ar.intersection_uid, - i.id, - i.api_name, - ar.classification_uid, - c.classification, - ar.range_start, - ar.notes, - alerts.alerts -ORDER BY - ar.intersection_uid, - ar.range_start, - ar.classification_uid; + AND ( + ar.leg = v.leg + OR ar.leg IS NULL + ) + LEFT JOIN alerts ON alerts.uid = ar.uid + WHERE + ar.problem_level <> 'valid-caveat' + --currently active + AND ( + ar.range_end IS NULL + OR ( + ar.notes LIKE '%identified by a daily airflow process%' + AND ar.range_end = (current_timestamp AT TIME ZONE 'EST5EDT')::date --today + ) + ) + GROUP BY + ar.uid, + ar.intersection_uid, + i.id, + i.api_name, + ar.classification_uid, + c.classification, + ar.range_start, + ar.notes, + alerts.alerts + ORDER BY + ar.intersection_uid, + ar.range_start, + ar.classification_uid +), + +closed AS ( + DELETE FROM miovision_api.open_issues_review + WHERE uid NOT IN (SELECT uid FROM open_issues) +) -COMMENT ON VIEW miovision_api.open_issues -IS '''A view to export open ended anomalous_ranges for communication with Miovision. -Converts intersection_uid and classification_uid into formats familiar to Miovision -(intersections.id, classifications.classification). anomalous_ranges.id col can be -used to link response back to table.'''; +MERGE INTO miovision_api.open_issues_review AS oir +USING open_issues AS oi +ON oir.uid = oi.uid +WHEN MATCHED THEN +UPDATE SET + intersection_uid = oi.intersection_uid, + intersection_id = oi.intersection_id, + intersection_name = oi.intersection_name, + classification_uid = oi.classification_uid, + classification = oi.classification, + leg = oi.leg, + range_start = oi.range_start, + num_days = oi.num_days, + notes = oi.notes, + volume = oi.volume, + alerts = oi.alerts +WHEN NOT MATCHED THEN + INSERT ( + uid, intersection_uid, intersection_id, intersection_name, classification_uid, + classification, leg, range_start, num_days, notes, volume, alerts + ) VALUES ( + oi.uid, oi.intersection_uid, oi.intersection_id, oi.intersection_name, + oi.classification_uid, oi.classification, oi.leg, oi.range_start, oi.num_days, oi.notes, + oi.volume, oi.alerts + ); -COMMENT ON COLUMN miovision_api.open_issues.volume -IS 'Volume recorded within the last 7 days or since anomalous_range began, whichever is later.' -ALTER TABLE miovision_api.open_issues OWNER TO miovision_admins; -GRANT SELECT ON TABLE miovision_api.open_issues TO bdit_humans; +COMMENT ON FUNCTION miovision_api.update_open_issues +IS '''A function to update miovision_api.open_issues. +Run weekly via miovision_check Airflow DAG.'''; \ No newline at end of file diff --git a/volumes/miovision/sql/table/create-table-open_issues.sql b/volumes/miovision/sql/table/create-table-open_issues.sql new file mode 100644 index 000000000..ee1a562d8 --- /dev/null +++ b/volumes/miovision/sql/table/create-table-open_issues.sql @@ -0,0 +1,38 @@ +-- Table: miovision_api.open_issues_review +-- DROP TABLE IF EXISTS miovision_api.open_issues_review; + +CREATE TABLE IF NOT EXISTS miovision_api.open_issues_review +( + uid smallint NOT NULL, + intersection_uid smallint, + intersection_id text COLLATE pg_catalog."default", + intersection_name text COLLATE pg_catalog."default", + classification_uid smallint, + classification text COLLATE pg_catalog."default", + leg text COLLATE pg_catalog."default", + range_start date, + num_days integer, + notes text COLLATE pg_catalog."default", + volume bigint, + alerts text COLLATE pg_catalog."default", + logged boolean, + CONSTRAINT open_issues_review_pkey PRIMARY KEY (uid) +) + +TABLESPACE pg_default; + +ALTER TABLE IF EXISTS miovision_api.open_issues_review +OWNER to miovision_admins; + +REVOKE ALL ON TABLE miovision_api.open_issues_review FROM bdit_humans; +REVOKE ALL ON TABLE miovision_api.open_issues_review FROM ckousin; + +GRANT SELECT ON TABLE miovision_api.open_issues_review TO bdit_humans; + +GRANT UPDATE ON TABLE miovision_api.open_issues_review TO ckousin; + +GRANT ALL ON TABLE miovision_api.open_issues_review TO miovision_admins; + +COMMENT ON TABLE miovision_api.open_issues_review +IS 'Update this table using `SELECT miovision_api.update_open_issues();`. +This is performed automatically once a week on Mondays by `miovision_check` Airflow DAG. '; \ No newline at end of file From f665866423b83b275ea2bf52acdfbcd481927658 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Thu, 29 Aug 2024 21:43:03 +0000 Subject: [PATCH 03/15] Merge open_intersection_outages changes #1053 --- .../select-ongoing_intersection_outages.sql | 35 +++++++++---------- .../sql/data_checks/select-open_issues.sql | 2 +- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql b/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql index 18f706a9e..4af43679e 100644 --- a/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql +++ b/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql @@ -1,25 +1,24 @@ -WITH ongoing_outages AS ( +WITH all_outages AS ( + SELECT COUNT(*) AS cnt + FROM miovision_api.open_issues + WHERE COALESCE(classification_uid, 0) != 2 +), + +total_outages AS ( SELECT i.intersection_name || ' (id: `' || i.id || '`) - data last received: `' - || MAX(v.dt::date) || '` (' || '{{ macros.ds_add(ds, 6) }}'::date --noqa: TMP, LT05 - - MAX(v.dt::date) || ' days)' AS descrip - FROM miovision_api.volumes_daily_unfiltered AS v + || oi.range_start || '` (' || CURRENT_DATE - oi.range_start || ' days)' AS descrip + FROM miovision_api.open_issues AS oi JOIN miovision_api.intersections AS i USING (intersection_uid) - WHERE i.date_decommissioned IS NULL - GROUP BY - i.intersection_uid, - i.intersection_name - HAVING - MAX(v.dt) - < '{{ macros.ds_add(ds, 6) }}'::date --noqa: TMP - - interval '{{ params.min_duration }}' --noqa: TMP + WHERE classification_uid IS NULL ) SELECT - COUNT(ongoing_outages.*) < 1 AS _check, - CASE WHEN COUNT(ongoing_outages.*) = 1 THEN 'There is ' ELSE 'There are ' END - || COALESCE(COUNT(ongoing_outages.*), 0) - || CASE WHEN COUNT(ongoing_outages.*) = 1 THEN ' ongoing outage.' ELSE ' ongoing outages.' + (SELECT cnt FROM all_outages) < 1 AS _check, + CASE WHEN COUNT(total_outages.*) = 1 THEN 'There is ' ELSE 'There are ' END + || COALESCE(COUNT(total_outages.*), 0) + || CASE WHEN COUNT(total_outages.*) = 1 THEN ' full outages.' ELSE ' full outages ' + || 'and ' || (SELECT cnt FROM all_outages) || ' partial outages. See `miovision_api.open_issues_review`.' END AS summ, --gap_threshold - array_agg(ongoing_outages.descrip) AS gaps -FROM ongoing_outages + array_agg(total_outages.descrip) AS gaps +FROM total_outages diff --git a/volumes/miovision/sql/data_checks/select-open_issues.sql b/volumes/miovision/sql/data_checks/select-open_issues.sql index e9ccb3734..6d02d9119 100644 --- a/volumes/miovision/sql/data_checks/select-open_issues.sql +++ b/volumes/miovision/sql/data_checks/select-open_issues.sql @@ -9,7 +9,7 @@ WITH ars AS ( leg, volume, notes - FROM miovision_api.open_issues + FROM miovision_api.open_issues_review WHERE volume > 0 ORDER BY uid ) From d7b629d76baaf6cd0fd63e4e8a54dc9b8ea936f9 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:29:16 -0400 Subject: [PATCH 04/15] #1053 fix function formatting --- .../sql/function/function-update-open_issues.sql | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/volumes/miovision/sql/function/function-update-open_issues.sql b/volumes/miovision/sql/function/function-update-open_issues.sql index a90afc22b..86e3ef353 100644 --- a/volumes/miovision/sql/function/function-update-open_issues.sql +++ b/volumes/miovision/sql/function/function-update-open_issues.sql @@ -1,5 +1,9 @@ --DROP FUNCTION miovision_api.update_open_issues; -CREATE OR REPLACE FUNCTION miovision_api.update_open_issues() AS +CREATE OR REPLACE FUNCTION miovision_api.update_open_issues() +RETURNS VOID +LANGUAGE SQL + +AS $BODY$ WITH alerts AS ( SELECT @@ -110,7 +114,8 @@ WHEN NOT MATCHED THEN oi.volume, oi.alerts ); +$BODY$; COMMENT ON FUNCTION miovision_api.update_open_issues IS '''A function to update miovision_api.open_issues. -Run weekly via miovision_check Airflow DAG.'''; \ No newline at end of file +Run weekly via miovision_check Airflow DAG.'''; From d35ce3e8716d930d638f7d6e61eceb03e8fc6a85 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Tue, 17 Sep 2024 20:24:30 +0000 Subject: [PATCH 05/15] #1053 add a refresh of open_issues to anomalous_range refresh task --- volumes/miovision/api/intersection_tmc.py | 2 ++ volumes/miovision/api/readme.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/volumes/miovision/api/intersection_tmc.py b/volumes/miovision/api/intersection_tmc.py index a07fa9a16..7979335f2 100644 --- a/volumes/miovision/api/intersection_tmc.py +++ b/volumes/miovision/api/intersection_tmc.py @@ -455,6 +455,8 @@ def agg_zero_volume_anomalous_ranges(conn, time_period, intersections = None): cur.execute(anomalous_range_sql, query_params) logger.info('Aggregation of zero volume periods into anomalous_ranges table complete for intersections %s', [x.uid for x in intersections]) + #update the table used for manual QC + cur.execute("SELECT miovision_api.update_open_issues();") except psycopg2.Error as exc: logger.exception(exc) sys.exit(1) diff --git a/volumes/miovision/api/readme.md b/volumes/miovision/api/readme.md index 4c6211939..6d106b249 100644 --- a/volumes/miovision/api/readme.md +++ b/volumes/miovision/api/readme.md @@ -277,7 +277,7 @@ This task group completes various Miovision aggregations. - `find_gaps_task` clears and then populates `miovision_api.unacceptable_gaps` using `intersection_tmc.find_gaps` function. - `aggregate_15_min_mvt_task` clears and then populates `miovision_api.volumes_15min_mvt` using `intersection_tmc.aggregate_15_min_mvt` function. - `aggregate_15_min_task` clears and then populates `miovision_api.volumes_15min` using `intersection_tmc.aggregate_15_min` function. -- `zero_volume_anomalous_ranges_task` identifies intersection / classification combos with zero volumes and adds/updates `miovision_api.anomalous_ranges` accordingly. +- `zero_volume_anomalous_ranges_task` identifies intersection / classification combos with zero volumes and adds/updates `miovision_api.anomalous_ranges` accordingly. Also refreshes `miovision_api.open_issues` table used for manual QC. - `aggregate_volumes_daily_task` clears and then populates `miovision_api.volumes_daily` using `intersection_tmc.aggregate_volumes_daily` function. - `get_report_dates_task` clears and then populates `miovision_api.report_dates` using `intersection_tmc.get_report_dates` function. From 0f85397404d1af8ef6e941df7c5623a277da222f Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Tue, 17 Sep 2024 20:24:50 +0000 Subject: [PATCH 06/15] #1053 add a last updated comment to open_issues refresh function --- .../function/function-update-open_issues.sql | 222 +++++++++--------- 1 file changed, 116 insertions(+), 106 deletions(-) diff --git a/volumes/miovision/sql/function/function-update-open_issues.sql b/volumes/miovision/sql/function/function-update-open_issues.sql index 86e3ef353..7083d7fc2 100644 --- a/volumes/miovision/sql/function/function-update-open_issues.sql +++ b/volumes/miovision/sql/function/function-update-open_issues.sql @@ -1,121 +1,131 @@ --DROP FUNCTION miovision_api.update_open_issues; CREATE OR REPLACE FUNCTION miovision_api.update_open_issues() RETURNS VOID -LANGUAGE SQL +LANGUAGE PLPGSQL AS $BODY$ -WITH alerts AS ( - SELECT - ar.uid, - string_agg(DISTINCT alerts.alert, '; '::text) AS alerts - FROM miovision_api.anomalous_ranges AS ar - JOIN miovision_api.alerts - ON alerts.intersection_uid = ar.intersection_uid - AND tsrange(alerts.start_time, alerts.end_time) - && tsrange(ar.range_start, ar.range_end) - GROUP BY ar.uid -), +BEGIN -open_issues AS ( - SELECT - ar.uid, - ar.intersection_uid, - i.id AS intersection_id, - i.api_name AS intersection_name, - ar.classification_uid, - CASE - WHEN ar.classification_uid = 2 THEN 'Bicycle TMC' - WHEN ar.classification_uid = 10 THEN 'Bicycle Approach' - WHEN ar.classification_uid IS NULL THEN 'All modes' - ELSE c.classification - END, - ar.leg, - ar.range_start::date, - (current_timestamp AT TIME ZONE 'EST5EDT')::date - ar.range_start::date AS num_days, - ar.notes, - SUM(v.volume) AS volume, - alerts.alerts - FROM miovision_api.anomalous_ranges AS ar - --keep rows with null classification_uid - LEFT JOIN miovision_api.classifications AS c USING (classification_uid) - --omit null intersection_uids. These will go under discontinuities. - JOIN miovision_api.intersections AS i USING (intersection_uid) - --find last week volume - LEFT JOIN miovision_api.volumes AS v - ON ar.intersection_uid = v.intersection_uid - --volume within the last 7 days and after AR started - AND v.datetime_bin >= ar.range_start - --prune the partitions - AND v.datetime_bin >= current_date - interval '7 days' - AND ( - ar.classification_uid = v.classification_uid - OR ar.classification_uid IS NULL - ) - AND ( - ar.leg = v.leg - OR ar.leg IS NULL - ) - LEFT JOIN alerts ON alerts.uid = ar.uid - WHERE - ar.problem_level <> 'valid-caveat' - --currently active - AND ( - ar.range_end IS NULL - OR ( - ar.notes LIKE '%identified by a daily airflow process%' - AND ar.range_end = (current_timestamp AT TIME ZONE 'EST5EDT')::date --today + WITH alerts AS ( + SELECT + ar.uid, + string_agg(DISTINCT alerts.alert, '; '::text) AS alerts + FROM miovision_api.anomalous_ranges AS ar + JOIN miovision_api.alerts + ON alerts.intersection_uid = ar.intersection_uid + AND tsrange(alerts.start_time, alerts.end_time) + && tsrange(ar.range_start, ar.range_end) + GROUP BY ar.uid + ), + + open_issues AS ( + SELECT + ar.uid, + ar.intersection_uid, + i.id AS intersection_id, + i.api_name AS intersection_name, + ar.classification_uid, + CASE + WHEN ar.classification_uid = 2 THEN 'Bicycle TMC' + WHEN ar.classification_uid = 10 THEN 'Bicycle Approach' + WHEN ar.classification_uid IS NULL THEN 'All modes' + ELSE c.classification + END, + ar.leg, + ar.range_start::date, + (current_timestamp AT TIME ZONE 'EST5EDT')::date - ar.range_start::date AS num_days, + ar.notes, + SUM(v.volume) AS volume, + alerts.alerts + FROM miovision_api.anomalous_ranges AS ar + --keep rows with null classification_uid + LEFT JOIN miovision_api.classifications AS c USING (classification_uid) + --omit null intersection_uids. These will go under discontinuities. + JOIN miovision_api.intersections AS i USING (intersection_uid) + --find last week volume + LEFT JOIN miovision_api.volumes AS v + ON ar.intersection_uid = v.intersection_uid + --volume within the last 7 days and after AR started + AND v.datetime_bin >= ar.range_start + --prune the partitions + AND v.datetime_bin >= current_date - interval '7 days' + AND ( + ar.classification_uid = v.classification_uid + OR ar.classification_uid IS NULL + ) + AND ( + ar.leg = v.leg + OR ar.leg IS NULL + ) + LEFT JOIN alerts ON alerts.uid = ar.uid + WHERE + ar.problem_level <> 'valid-caveat' + --currently active + AND ( + ar.range_end IS NULL + OR ( + ar.notes LIKE '%identified by a daily airflow process%' + AND ar.range_end = (current_timestamp AT TIME ZONE 'EST5EDT')::date --today + ) ) - ) - GROUP BY - ar.uid, - ar.intersection_uid, - i.id, - i.api_name, - ar.classification_uid, - c.classification, - ar.range_start, - ar.notes, - alerts.alerts - ORDER BY - ar.intersection_uid, - ar.range_start, - ar.classification_uid -), + GROUP BY + ar.uid, + ar.intersection_uid, + i.id, + i.api_name, + ar.classification_uid, + c.classification, + ar.range_start, + ar.notes, + alerts.alerts + ORDER BY + ar.intersection_uid, + ar.range_start, + ar.classification_uid + ), -closed AS ( - DELETE FROM miovision_api.open_issues_review - WHERE uid NOT IN (SELECT uid FROM open_issues) -) + closed AS ( + DELETE FROM miovision_api.open_issues_review + WHERE uid NOT IN (SELECT uid FROM open_issues) + ) -MERGE INTO miovision_api.open_issues_review AS oir -USING open_issues AS oi -ON oir.uid = oi.uid -WHEN MATCHED THEN -UPDATE SET - intersection_uid = oi.intersection_uid, - intersection_id = oi.intersection_id, - intersection_name = oi.intersection_name, - classification_uid = oi.classification_uid, - classification = oi.classification, - leg = oi.leg, - range_start = oi.range_start, - num_days = oi.num_days, - notes = oi.notes, - volume = oi.volume, - alerts = oi.alerts -WHEN NOT MATCHED THEN - INSERT ( - uid, intersection_uid, intersection_id, intersection_name, classification_uid, - classification, leg, range_start, num_days, notes, volume, alerts - ) VALUES ( - oi.uid, oi.intersection_uid, oi.intersection_id, oi.intersection_name, - oi.classification_uid, oi.classification, oi.leg, oi.range_start, oi.num_days, oi.notes, - oi.volume, oi.alerts - ); + MERGE INTO miovision_api.open_issues_review AS oir + USING open_issues AS oi + ON oir.uid = oi.uid + WHEN MATCHED THEN + UPDATE SET + intersection_uid = oi.intersection_uid, + intersection_id = oi.intersection_id, + intersection_name = oi.intersection_name, + classification_uid = oi.classification_uid, + classification = oi.classification, + leg = oi.leg, + range_start = oi.range_start, + num_days = oi.num_days, + notes = oi.notes, + volume = oi.volume, + alerts = oi.alerts + WHEN NOT MATCHED THEN + INSERT ( + uid, intersection_uid, intersection_id, intersection_name, classification_uid, + classification, leg, range_start, num_days, notes, volume, alerts + ) VALUES ( + oi.uid, oi.intersection_uid, oi.intersection_id, oi.intersection_name, + oi.classification_uid, oi.classification, oi.leg, oi.range_start, oi.num_days, oi.notes, + oi.volume, oi.alerts + ); + EXECUTE FORMAT( + 'COMMENT ON TABLE miovision_api.open_issues_review IS %L', + 'Last updated ' || to_char(now() AT TIME ZONE 'EST5EDT', 'yyyy-mm-dd HH24:MI') + || ' using `SELECT miovision_api.update_open_issues();`.' + || ' This is performed automatically once per day by `miovision_pull` DAG.' + ); + +END $BODY$; COMMENT ON FUNCTION miovision_api.update_open_issues IS '''A function to update miovision_api.open_issues. -Run weekly via miovision_check Airflow DAG.'''; +Run daily via miovision_pull Airflow DAG.'''; From 1872fc43df148d9dbd6648f1d2877c6f9e8b7974 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Tue, 17 Sep 2024 20:28:20 +0000 Subject: [PATCH 07/15] #1053 give permissions to bot --- .../sql/table/create-table-open_issues.sql | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/volumes/miovision/sql/table/create-table-open_issues.sql b/volumes/miovision/sql/table/create-table-open_issues.sql index ee1a562d8..063c82855 100644 --- a/volumes/miovision/sql/table/create-table-open_issues.sql +++ b/volumes/miovision/sql/table/create-table-open_issues.sql @@ -24,15 +24,11 @@ TABLESPACE pg_default; ALTER TABLE IF EXISTS miovision_api.open_issues_review OWNER to miovision_admins; -REVOKE ALL ON TABLE miovision_api.open_issues_review FROM bdit_humans; -REVOKE ALL ON TABLE miovision_api.open_issues_review FROM ckousin; +GRANT ALL ON TABLE miovision_api.open_issues_review TO miovision_admins; +GRANT INSERT, SELECT, UPDATE, DELETE ON TABLE miovision_api.open_issues_review TO miovision_api_bot; +REVOKE ALL ON TABLE miovision_api.open_issues_review FROM bdit_humans; GRANT SELECT ON TABLE miovision_api.open_issues_review TO bdit_humans; -GRANT UPDATE ON TABLE miovision_api.open_issues_review TO ckousin; - -GRANT ALL ON TABLE miovision_api.open_issues_review TO miovision_admins; - -COMMENT ON TABLE miovision_api.open_issues_review -IS 'Update this table using `SELECT miovision_api.update_open_issues();`. -This is performed automatically once a week on Mondays by `miovision_check` Airflow DAG. '; \ No newline at end of file +REVOKE ALL ON TABLE miovision_api.open_issues_review FROM ckousin; +GRANT UPDATE ON TABLE miovision_api.open_issues_review TO ckousin; \ No newline at end of file From ff2fdd6b0b964cb92ac4633ae66820fc15a7f9fe Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:21:40 +0000 Subject: [PATCH 08/15] #1053 alter owner --- volumes/miovision/sql/table/create-table-open_issues.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volumes/miovision/sql/table/create-table-open_issues.sql b/volumes/miovision/sql/table/create-table-open_issues.sql index 063c82855..0307abd77 100644 --- a/volumes/miovision/sql/table/create-table-open_issues.sql +++ b/volumes/miovision/sql/table/create-table-open_issues.sql @@ -21,8 +21,7 @@ CREATE TABLE IF NOT EXISTS miovision_api.open_issues_review TABLESPACE pg_default; -ALTER TABLE IF EXISTS miovision_api.open_issues_review -OWNER to miovision_admins; +ALTER TABLE miovision_api.open_issues_review OWNER TO miovision_api_bot; GRANT ALL ON TABLE miovision_api.open_issues_review TO miovision_admins; GRANT INSERT, SELECT, UPDATE, DELETE ON TABLE miovision_api.open_issues_review TO miovision_api_bot; From fe7d3a818b4b1d6d4b73615d1ffe8a8e5fd38e4a Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:16:41 -0400 Subject: [PATCH 09/15] #1053 add notes column --- volumes/miovision/sql/table/create-table-open_issues.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volumes/miovision/sql/table/create-table-open_issues.sql b/volumes/miovision/sql/table/create-table-open_issues.sql index 0307abd77..05084dcde 100644 --- a/volumes/miovision/sql/table/create-table-open_issues.sql +++ b/volumes/miovision/sql/table/create-table-open_issues.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS miovision_api.open_issues_review volume bigint, alerts text COLLATE pg_catalog."default", logged boolean, + notes txt CONSTRAINT open_issues_review_pkey PRIMARY KEY (uid) ) @@ -30,4 +31,4 @@ REVOKE ALL ON TABLE miovision_api.open_issues_review FROM bdit_humans; GRANT SELECT ON TABLE miovision_api.open_issues_review TO bdit_humans; REVOKE ALL ON TABLE miovision_api.open_issues_review FROM ckousin; -GRANT UPDATE ON TABLE miovision_api.open_issues_review TO ckousin; \ No newline at end of file +GRANT UPDATE ON TABLE miovision_api.open_issues_review TO ckousin; From 3fc090b3b812b6fa98ed5394eda84a346b416523 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:18:11 -0400 Subject: [PATCH 10/15] #1053 don't use duplicate column names --- volumes/miovision/sql/table/create-table-open_issues.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volumes/miovision/sql/table/create-table-open_issues.sql b/volumes/miovision/sql/table/create-table-open_issues.sql index 05084dcde..61be6aa91 100644 --- a/volumes/miovision/sql/table/create-table-open_issues.sql +++ b/volumes/miovision/sql/table/create-table-open_issues.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS miovision_api.open_issues_review volume bigint, alerts text COLLATE pg_catalog."default", logged boolean, - notes txt + reviewer_notes text, CONSTRAINT open_issues_review_pkey PRIMARY KEY (uid) ) From 1996174ec273ac0df8a1386783a1d662d61179da Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:54:25 +0000 Subject: [PATCH 11/15] #1053 fluff --- .../data_checks/select-ongoing_intersection_outages.sql | 9 +++++++-- .../sql/function/function-update-open_issues.sql | 5 ++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql b/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql index 4af43679e..76ec4643e 100644 --- a/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql +++ b/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql @@ -17,8 +17,13 @@ SELECT (SELECT cnt FROM all_outages) < 1 AS _check, CASE WHEN COUNT(total_outages.*) = 1 THEN 'There is ' ELSE 'There are ' END || COALESCE(COUNT(total_outages.*), 0) - || CASE WHEN COUNT(total_outages.*) = 1 THEN ' full outages.' ELSE ' full outages ' - || 'and ' || (SELECT cnt FROM all_outages) || ' partial outages. See `miovision_api.open_issues_review`.' + || CASE + WHEN + COUNT(total_outages.*) = 1 THEN ' full outages.' ELSE ' full outages ' + || 'and ' + || (SELECT cnt FROM all_outages + ) + || ' partial outages. See `miovision_api.open_issues_review`.' END AS summ, --gap_threshold array_agg(total_outages.descrip) AS gaps FROM total_outages diff --git a/volumes/miovision/sql/function/function-update-open_issues.sql b/volumes/miovision/sql/function/function-update-open_issues.sql index 7083d7fc2..ffe93752e 100644 --- a/volumes/miovision/sql/function/function-update-open_issues.sql +++ b/volumes/miovision/sql/function/function-update-open_issues.sql @@ -1,6 +1,6 @@ --DROP FUNCTION miovision_api.update_open_issues; CREATE OR REPLACE FUNCTION miovision_api.update_open_issues() -RETURNS VOID +RETURNS void LANGUAGE PLPGSQL AS $BODY$ @@ -126,6 +126,5 @@ BEGIN END $BODY$; -COMMENT ON FUNCTION miovision_api.update_open_issues -IS '''A function to update miovision_api.open_issues. +COMMENT ON FUNCTION miovision_api.update_open_issuesIS '''A function to update miovision_api.open_issues. Run daily via miovision_pull Airflow DAG.'''; From bf457a93750f857e9c6db1f51f71929a67853b41 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Thu, 28 Nov 2024 20:07:15 +0000 Subject: [PATCH 12/15] #1053 add to readme --- volumes/miovision/sql/readme.md | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/volumes/miovision/sql/readme.md b/volumes/miovision/sql/readme.md index 7c9d6162a..ea78b4658 100644 --- a/volumes/miovision/sql/readme.md +++ b/volumes/miovision/sql/readme.md @@ -9,14 +9,15 @@ - [`movements`](#movements) - [`volumes`](#volumes) - [Aggregated Data](#aggregated-data) - - [`volumes_15min_mvt` (Use view `volumes_15min_mvt_filtered` to exclude anomalous\_ranges)](#volumes_15min_mvt-use-view-volumes_15min_mvt_filtered-to-exclude-anomalous_ranges) - - [`volumes_15min` (Use view `volumes_15min_filtered` to exclude anomalous\_ranges)](#volumes_15min-use-view-volumes_15min_filtered-to-exclude-anomalous_ranges) + - [`volumes_15min_mvt`](#volumes_15min_mvt) + - [`volumes_15min`](#volumes_15min) - [`miovision_api.volumes_daily`](#miovision_apivolumes_daily) - [`unacceptable_gaps`](#unacceptable_gaps) - [`gapsize_lookup`](#gapsize_lookup) - [Reference Tables](#reference-tables) - [`miovision_api.breaks`](#miovision_apibreaks) - [`miovision_api.anomalous_ranges`](#miovision_apianomalous_ranges) + - [`miovision_api.open_issues`](#miovision_apiopen_issues) - [`miovision_api.anomaly_investigation_levels` and `miovision_api.anomaly_problem_levels`](#miovision_apianomaly_investigation_levels-and-miovision_apianomaly_problem_levels) - [`movement_map`](#movement_map) - [`periods`](#periods) @@ -232,7 +233,8 @@ Data are aggregated from 1-minute volume data into two types of 15-minute volume } ``` -### `volumes_15min_mvt` (Use view `volumes_15min_mvt_filtered` to exclude anomalous_ranges) +### `volumes_15min_mvt` +**(Use view `volumes_15min_mvt_filtered` to exclude anomalous_ranges)** `volumes_15min_mvt` contains data aggregated into 15 minute bins. In order to make averaging hourly volumes simpler, the volume can be `NULL` (for all modes) or `0` for classifications 1, 2, 6, 10 (which corresponds to light vehicles, bicycles (classifications 2 and 10) and pedestrians). @@ -260,7 +262,8 @@ Please see [this diagram](../getting_started.md#Vehicle-Movements) for a visuali - A *Unique constraint* was added to `miovision_api.volumes_15min_mvt` table based on `intersection_uid`, `datetime_bin`, `classification_uid`, `leg` and `movement_uid`. -### `volumes_15min` (Use view `volumes_15min_filtered` to exclude anomalous_ranges) +### `volumes_15min` +**(Use view `volumes_15min_filtered` to exclude anomalous_ranges)** Data table storing ATR versions of the 15-minute turning movement data. Data in `volumes` is stored in TMC format, so must be converted to ATR to be included in @@ -368,6 +371,28 @@ The `anomalous_ranges` table is used to log issues related to specific intersect | problem_level | references `miovision_api.anomaly_problem_levels`; indicates the degree or nature of the problem. e.g. valid with a caveat vs do-not-use under any circumstance | | leg | Intersection leg which is affected by range. Null refers to all legs. | +### `miovision_api.open_issues` + +`open_issues` is a complementary table to `anomalous_ranges`. It is similar to a view of anomalous ranges which are currently open, except it is a table to allow interactive editing in PgAdmin. Each day via Airflow DAG, issues which are no longer active are removed, but the "logged" and "reviewer_notes" columns remain and other columns may be updated. + +| column_name | data_type | sample | +|:-------------------|:------------|:-------------------------------------------------------------------------------------------------------| +| uid | smallint | 1797 | +| intersection_uid | smallint | 1 | +| intersection_id | text | 8184ba7d-a2e3-4a1c-b70f-31da15e7462a | +| intersection_name | text | Bathurst Street and Adelaide Street West | +| classification_uid | smallint | 2 | +| classification | text | Bicycle TMC | +| leg | text | W | +| range_start | date | 2024-11-20 | +| num_days | integer | 8 | +| notes | text | Zero counts, identified by a daily airflow process running function miovision_api.identify_zero_counts | +| volume | bigint | | +| alerts | text | | +| logged | boolean | | +| reviewer_notes | text | | + + ### `miovision_api.anomaly_investigation_levels` and `miovision_api.anomaly_problem_levels` These two tables are used to enforce standardized descriptions in the `investigation_level` and `problem_level` columns of `anomalous_ranges`. From 335054f460333491c785693034088b5b2a4022d1 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Thu, 28 Nov 2024 20:07:32 +0000 Subject: [PATCH 13/15] Merge ongoing_intersection_outages changes #1053 --- .../select-ongoing_intersection_outages.sql | 2 +- .../sql/data_checks/select-open_issues.sql | 2 +- .../function/function-update-open_issues.sql | 6 +++--- .../sql/table/create-table-open_issues.sql | 20 +++++++++---------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql b/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql index 76ec4643e..03a4c2d8f 100644 --- a/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql +++ b/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql @@ -23,7 +23,7 @@ SELECT || 'and ' || (SELECT cnt FROM all_outages ) - || ' partial outages. See `miovision_api.open_issues_review`.' + || ' partial outages. See `miovision_api.open_issues`.' END AS summ, --gap_threshold array_agg(total_outages.descrip) AS gaps FROM total_outages diff --git a/volumes/miovision/sql/data_checks/select-open_issues.sql b/volumes/miovision/sql/data_checks/select-open_issues.sql index 6d02d9119..e9ccb3734 100644 --- a/volumes/miovision/sql/data_checks/select-open_issues.sql +++ b/volumes/miovision/sql/data_checks/select-open_issues.sql @@ -9,7 +9,7 @@ WITH ars AS ( leg, volume, notes - FROM miovision_api.open_issues_review + FROM miovision_api.open_issues WHERE volume > 0 ORDER BY uid ) diff --git a/volumes/miovision/sql/function/function-update-open_issues.sql b/volumes/miovision/sql/function/function-update-open_issues.sql index ffe93752e..3f4d2a73d 100644 --- a/volumes/miovision/sql/function/function-update-open_issues.sql +++ b/volumes/miovision/sql/function/function-update-open_issues.sql @@ -86,11 +86,11 @@ BEGIN ), closed AS ( - DELETE FROM miovision_api.open_issues_review + DELETE FROM miovision_api.open_issues WHERE uid NOT IN (SELECT uid FROM open_issues) ) - MERGE INTO miovision_api.open_issues_review AS oir + MERGE INTO miovision_api.open_issues AS oir USING open_issues AS oi ON oir.uid = oi.uid WHEN MATCHED THEN @@ -117,7 +117,7 @@ BEGIN ); EXECUTE FORMAT( - 'COMMENT ON TABLE miovision_api.open_issues_review IS %L', + 'COMMENT ON TABLE miovision_api.open_issues IS %L', 'Last updated ' || to_char(now() AT TIME ZONE 'EST5EDT', 'yyyy-mm-dd HH24:MI') || ' using `SELECT miovision_api.update_open_issues();`.' || ' This is performed automatically once per day by `miovision_pull` DAG.' diff --git a/volumes/miovision/sql/table/create-table-open_issues.sql b/volumes/miovision/sql/table/create-table-open_issues.sql index 61be6aa91..77139264a 100644 --- a/volumes/miovision/sql/table/create-table-open_issues.sql +++ b/volumes/miovision/sql/table/create-table-open_issues.sql @@ -1,7 +1,7 @@ --- Table: miovision_api.open_issues_review --- DROP TABLE IF EXISTS miovision_api.open_issues_review; +-- Table: miovision_api.open_issues +-- DROP TABLE IF EXISTS miovision_api.open_issues; -CREATE TABLE IF NOT EXISTS miovision_api.open_issues_review +CREATE TABLE IF NOT EXISTS miovision_api.open_issues ( uid smallint NOT NULL, intersection_uid smallint, @@ -22,13 +22,13 @@ CREATE TABLE IF NOT EXISTS miovision_api.open_issues_review TABLESPACE pg_default; -ALTER TABLE miovision_api.open_issues_review OWNER TO miovision_api_bot; +ALTER TABLE miovision_api.open_issues OWNER TO miovision_api_bot; -GRANT ALL ON TABLE miovision_api.open_issues_review TO miovision_admins; -GRANT INSERT, SELECT, UPDATE, DELETE ON TABLE miovision_api.open_issues_review TO miovision_api_bot; +GRANT ALL ON TABLE miovision_api.open_issues TO miovision_admins; +GRANT INSERT, SELECT, UPDATE, DELETE ON TABLE miovision_api.open_issues TO miovision_api_bot; -REVOKE ALL ON TABLE miovision_api.open_issues_review FROM bdit_humans; -GRANT SELECT ON TABLE miovision_api.open_issues_review TO bdit_humans; +REVOKE ALL ON TABLE miovision_api.open_issues FROM bdit_humans; +GRANT SELECT ON TABLE miovision_api.open_issues TO bdit_humans; -REVOKE ALL ON TABLE miovision_api.open_issues_review FROM ckousin; -GRANT UPDATE ON TABLE miovision_api.open_issues_review TO ckousin; +REVOKE ALL ON TABLE miovision_api.open_issues FROM ckousin; +GRANT UPDATE ON TABLE miovision_api.open_issues TO ckousin; From aa623ad893441d5f3f701d3558a2f1040f5d0cb6 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Thu, 28 Nov 2024 20:13:58 +0000 Subject: [PATCH 14/15] #1053 fix typo, add permissions --- .../sql/function/function-update-open_issues.sql | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/volumes/miovision/sql/function/function-update-open_issues.sql b/volumes/miovision/sql/function/function-update-open_issues.sql index 3f4d2a73d..a77ed4d60 100644 --- a/volumes/miovision/sql/function/function-update-open_issues.sql +++ b/volumes/miovision/sql/function/function-update-open_issues.sql @@ -126,5 +126,12 @@ BEGIN END $BODY$; -COMMENT ON FUNCTION miovision_api.update_open_issuesIS '''A function to update miovision_api.open_issues. -Run daily via miovision_pull Airflow DAG.'''; +COMMENT ON FUNCTION miovision_api.update_open_issues IS +'A function to update miovision_api.open_issues. ' +'Run daily via miovision_pull Airflow DAG.'; + +ALTER FUNCTION miovision_api.update_open_issues() OWNER TO miovision_admins; + +GRANT EXECUTE ON FUNCTION miovision_api.update_open_issues() TO miovision_api_bot; +GRANT EXECUTE ON FUNCTION miovision_api.update_open_issues() TO miovision_data_detectives; + From 5bc8ef5a760ec9d1fbdcca95b38bb6287f10394d Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Thu, 28 Nov 2024 20:55:19 +0000 Subject: [PATCH 15/15] #1053 sqlfluff --- .../sql/data_checks/select-ongoing_intersection_outages.sql | 2 +- .../miovision/sql/function/function-update-open_issues.sql | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql b/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql index 03a4c2d8f..e5b891002 100644 --- a/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql +++ b/volumes/miovision/sql/data_checks/select-ongoing_intersection_outages.sql @@ -10,7 +10,7 @@ total_outages AS ( || oi.range_start || '` (' || CURRENT_DATE - oi.range_start || ' days)' AS descrip FROM miovision_api.open_issues AS oi JOIN miovision_api.intersections AS i USING (intersection_uid) - WHERE classification_uid IS NULL + WHERE oi.classification_uid IS NULL ) SELECT diff --git a/volumes/miovision/sql/function/function-update-open_issues.sql b/volumes/miovision/sql/function/function-update-open_issues.sql index a77ed4d60..95d16e123 100644 --- a/volumes/miovision/sql/function/function-update-open_issues.sql +++ b/volumes/miovision/sql/function/function-update-open_issues.sql @@ -126,12 +126,12 @@ BEGIN END $BODY$; -COMMENT ON FUNCTION miovision_api.update_open_issues IS +COMMENT ON FUNCTION miovision_api.update_open_issues() IS 'A function to update miovision_api.open_issues. ' 'Run daily via miovision_pull Airflow DAG.'; ALTER FUNCTION miovision_api.update_open_issues() OWNER TO miovision_admins; -GRANT EXECUTE ON FUNCTION miovision_api.update_open_issues() TO miovision_api_bot; -GRANT EXECUTE ON FUNCTION miovision_api.update_open_issues() TO miovision_data_detectives; +GRANT EXECUTE ON FUNCTION miovision_api.update_open_issues() TO MIOVISION_API_BOT; +GRANT EXECUTE ON FUNCTION miovision_api.update_open_issues() TO MIOVISION_DATA_DETECTIVES;