From dd4da408da3c26951682afdd8d0e9d306fa190f7 Mon Sep 17 00:00:00 2001 From: gabrielwol <80077912+gabrielwol@users.noreply.github.com> Date: Tue, 13 Feb 2024 13:29:15 -0500 Subject: [PATCH 1/4] #436 add api_name to intersections table --- .../sql/table/create-table-intersections.sql | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/volumes/miovision/sql/table/create-table-intersections.sql b/volumes/miovision/sql/table/create-table-intersections.sql index 40e428c3f..fd7a1a111 100644 --- a/volumes/miovision/sql/table/create-table-intersections.sql +++ b/volumes/miovision/sql/table/create-table-intersections.sql @@ -15,9 +15,16 @@ n_leg_restricted boolean, e_leg_restricted boolean, s_leg_restricted boolean, - w_leg_restricted boolean + w_leg_restricted boolean, + api_name text COLLATE pg_catalog."default" ) WITH ( OIDS = FALSE ) -TABLESPACE pg_default; \ No newline at end of file +TABLESPACE pg_default; + +COMMENT ON COLUMN miovision_api.intersections.intersection_name +IS 'A short name for the intersection, following the convention [E / W street name] / [N / S street name].'; + +COMMENT ON COLUMN miovision_api.intersections.api_name +IS 'The intersection name used in the Miovision API, for communication with external parties.'; From 0c96107e94d0f2af91bbed67bc1f1320801d19d7 Mon Sep 17 00:00:00 2001 From: Gabe Wolofsky <80077912+gabrielwol@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:31:28 +0000 Subject: [PATCH 2/4] #436 update readme instructions for api_name --- .../miovision/update_intersections/Readme.md | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/volumes/miovision/update_intersections/Readme.md b/volumes/miovision/update_intersections/Readme.md index 6fec3af93..9097cf5e8 100644 --- a/volumes/miovision/update_intersections/Readme.md +++ b/volumes/miovision/update_intersections/Readme.md @@ -9,10 +9,10 @@ For the main Miovision readme, see [here](../README.md). - [Miovision Intersection Update Resources](#miovision-intersection-update-resources) - [Removing Intersections](#removing-intersections) - [Adding Intersections](#adding-intersections) - - [Update miovision_api.intersections:](#update-miovision_apiintersections) - - [Update miovision_api.intersection_movements](#update-miovision_apiintersection_movements) - - [Update miovision_api.centreline_miovision](#update-miovision_apicentreline_miovision) - - [Backfill/Aggregate new intersection data](#backfillaggregate-new-intersection-data) + - [Update `miovision_api.intersections`:](#update-miovision_apiintersections) + - [Update `miovision_api.intersection_movements`](#update-miovision_apiintersection_movements) + - [Update `miovision_api.centreline_miovision`](#update-miovision_apicentreline_miovision) + - [Backfill/Aggregate new intersection data](#backfillaggregate-new-intersection-data) - [New Intersection Activation Dates.ipynb](#new-intersection-activation-datesipynb) - [Adding many intersections](#adding-many-intersections) @@ -35,7 +35,7 @@ Adding intersections is not as simple as removing an intersection. We will first Look at the table [`miovision_api.intersections`](../README.md#intersections) to see what information about the new intersections is needed to update the table. The steps needed to find details such as id, coordinates, px, int_id, geom, which leg_restricted etc are described below. Once everything is done, have a member of `miovision_admins` do an INSERT INTO this table to include the new intersections. 1. **Name and ID** - The new intersection's `intersection_name`, `id`, can be found using the [Miovision API](https://api.miovision.com/intersections) /intersections endpoint. The key needed to authorize the API is the same one used by the Miovision Airflow user. + The new intersection's `api_name`, `id`, can be found using the [Miovision API](https://api.miovision.com/intersections) /intersections endpoint. The key needed to authorize the API is the same one used by the Miovision Airflow user. The `intersection_name` is an internal name following the convention `[E / W street name] / [N / S street name]`. 2. **date installed** `date_installed` is the *date of the first row of data from the location* (so if the first row has a `datetime_bin` of '2020-10-05 12:15', the `date_installed` is '2020-10-05'). `date_installed` can be found by by e-mailing Miovision, manually querying the Miovision API for the first available timestamp, or by running the [Jupyter notebook](new_intersection_activation_dates.ipynb) in this folder. @@ -74,21 +74,21 @@ Look at the table [`miovision_api.intersections`](../README.md#intersections) to ```sql INSERT INTO miovision_api.intersections(intersection_uid, id, intersection_name, date_installed, lat, lng, geom, street_main, street_cross, int_id, px, - n_leg_restricted, e_leg_restricted, s_leg_restricted, w_leg_restricted) + n_leg_restricted, e_leg_restricted, s_leg_restricted, w_leg_restricted, api_name) WITH new_intersections (intersection_uid, id, intersection_name, date_installed, px, - n_leg_restricted, e_leg_restricted, s_leg_restricted, w_leg_restricted) AS ( + n_leg_restricted, e_leg_restricted, s_leg_restricted, w_leg_restricted, api_name) AS ( VALUES - (67, '11dcfdc5-2b37-45c0-ac79-3d6926553582', 'Sheppard Avenue West and Keele Street', - '2021-06-16'::date, '0600', null, null, null, null), - (68, '9ed9e7f3-9edc-4f58-ae5b-8c9add746886', 'Steeles Avenue West and Jane Street', - '2021-05-12'::date, '0535', null, null, null, null) + (67, '11dcfdc5-2b37-45c0-ac79-3d6926553582', 'Sheppard / Keele', + '2021-06-16'::date, '0600', null, null, null, null, 'Sheppard Avenue West and Keele Street'), + (68, '9ed9e7f3-9edc-4f58-ae5b-8c9add746886', 'Steeles / Jane', + '2021-05-12'::date, '0535', null, null, null, null, 'Steeles Avenue West and Jane Street') ) SELECT ni.intersection_uid, --sequential ni.id, --from api - ni.intersection_name, --from api + ni.intersection_name, --cleaned name ni.date_installed, --identify via communication or new_intersection_activation_dates.ipynb ts.latitude, ts.longitude, @@ -100,7 +100,8 @@ Look at the table [`miovision_api.intersections`](../README.md#intersections) to ni.n_leg_restricted, ni.e_leg_restricted, ni.s_leg_restricted, - ni.w_leg_restricted + ni.w_leg_restricted. + api_name --from api FROM new_intersections AS ni JOIN gis.traffic_signal AS ts USING (px) ``` @@ -411,7 +412,7 @@ with psycopg2.connect(**postgres_settings) as conn: date_installed, lat, lng, street_main, street_cross, int_id, px, n_leg_restricted, e_leg_restricted, - s_leg_restricted, w_leg_restricted) VALUES %s""" + s_leg_restricted, w_leg_restricted, api_name) VALUES %s""" execute_values(cur, insert_data, df_list) update_geom = """UPDATE miovision_api.intersections a SET geom = ST_SetSRID(ST_MakePoint(b.lng, b.lat), 4326) From b2845d9fce0b70a1c4fb30e1d75ce21596141960 Mon Sep 17 00:00:00 2001 From: Gabe Wolofsky <80077912+gabrielwol@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:34:10 +0000 Subject: [PATCH 3/4] #436 update sql readme with api_name --- volumes/miovision/sql/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/volumes/miovision/sql/README.md b/volumes/miovision/sql/README.md index 8beb63e14..d569f25b5 100644 --- a/volumes/miovision/sql/README.md +++ b/volumes/miovision/sql/README.md @@ -1,4 +1,6 @@ - + + + - [1. Overview](#1-overview) - [2. `miovision_api` Table Structure](#2-miovision_api-table-structure) - [Miovision Data Relationships at a Glance](#miovision-data-relationships-at-a-glance) @@ -39,7 +41,6 @@ - [Identifying new anomalies](#identifying-new-anomalies) - # 1. Overview This folder contains sql scripts used in both the API and the old data dump process. The [`csv_data/`](csv_data/) sub-folder contains `sql` files unique to processing the data from csv dumps. @@ -72,7 +73,7 @@ Reference table for each unique intersection at which data has been collected, y :-----|:-----|:-----|:-----| intersection_uid|integer|Unique identifier for table|10| id|text|Unique id from Miovision API|990cd89a-430a-409a-b0e7-d37338394148| -intersection_name|text|Intersection in format of [main street] / [cross street]|King / Bathurst| +intersection_name|text|Intersection short name in format of [E / W street] / [N / S street]|King / Bathurst| date_installed|date|Installation date of the camera (date of the first available timestamp)|2017-10-03| date_decommissioned|date|Decommissioned date of the camera (date of the last available timestamp)|NULL| lat|numeric|Latitude of intersection location|43.643945| @@ -86,7 +87,7 @@ n_leg_restricted|boolean|Whether that leg is restricted to vehicles|NULL| e_leg_restricted|boolean|Whether that leg is restricted to vehicles|NULL| s_leg_restricted|boolean|Whether that leg is restricted to vehicles|NULL| w_leg_restricted|boolean|Whether that leg is restricted to vehicles|NULL| - +api_name|text|Intersection name from API|Bathurst Street and King Street West| ### `classifications` From f79fd6eee455f2ca35a002ba081270f8f13012c1 Mon Sep 17 00:00:00 2001 From: Gabe Wolofsky <80077912+gabrielwol@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:33:37 +0000 Subject: [PATCH 4/4] #436 fluff --- volumes/miovision/sql/table/create-table-intersections.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volumes/miovision/sql/table/create-table-intersections.sql b/volumes/miovision/sql/table/create-table-intersections.sql index fd7a1a111..429cbb06c 100644 --- a/volumes/miovision/sql/table/create-table-intersections.sql +++ b/volumes/miovision/sql/table/create-table-intersections.sql @@ -11,7 +11,7 @@ street_cross text COLLATE pg_catalog."default", int_id bigint, px integer, - geom geometry(Point,4326), + geom geometry (POINT, 4326), n_leg_restricted boolean, e_leg_restricted boolean, s_leg_restricted boolean, @@ -19,12 +19,13 @@ api_name text COLLATE pg_catalog."default" ) WITH ( - OIDS = FALSE + oids = FALSE ) TABLESPACE pg_default; COMMENT ON COLUMN miovision_api.intersections.intersection_name -IS 'A short name for the intersection, following the convention [E / W street name] / [N / S street name].'; +IS 'A short name for the intersection, following the convention +[E / W street name] / [N / S street name].'; COMMENT ON COLUMN miovision_api.intersections.api_name IS 'The intersection name used in the Miovision API, for communication with external parties.';