diff --git a/docs/reference.rst b/docs/api_reference.rst similarity index 64% rename from docs/reference.rst rename to docs/api_reference.rst index 529294b4..1fc0631c 100644 --- a/docs/reference.rst +++ b/docs/api_reference.rst @@ -4,7 +4,14 @@ API Reference / Postman Collection For reference and documentation of the API, please open the postman collection for the version of ripple1d -`v0.4.1: `_ This beta version contains the endpoints included in the first production testing release. Note that the following variables should be set in the postman environment + +`v0.5.0: `_ This beta version contains new endpoints: + - `geom_to_gpkg`: Extract the data from a model source dirctory to a gepoackage. + - `conflate`: Conflate all reaches from the NWM network corresponding to the source model. + - `conflation_metrics`: Apply conflation metrics for a conflated source model. + + +`v0.4.1-v0.4.2: `_ This beta version contains the endpoints included in the first production testing release. Note that the following variables should be set in the postman environment .. code-block:: YAML diff --git a/docs/index.rst b/docs/index.rst index 02c40b17..d6850747 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,8 @@ on the NOAA National Water Model network. :caption: For Users for_users.rst - reference.rst + api_reference.rst + tech_reference.rst users_change_log.rst diff --git a/docs/tech_reference.rst b/docs/tech_reference.rst new file mode 100644 index 00000000..ea280084 --- /dev/null +++ b/docs/tech_reference.rst @@ -0,0 +1,417 @@ +Technical Reference +=================== + +Preliminary conflation metrics are calcuated to quantify alignment between the source RAS model and the source network reaches. Definitions and examples: + ++------------------------+------------------------+-------------------+ +| term | type | definition | ++========================+========================+===================+ +| reaches | dict | Source network | +| | | reaches that | +| | | conflated with | +| | | the RAS model. | +| | | Reach ids are the | +| | | keys of the | +| | | nested dicts | +| | | which contain the | +| | | conflation | +| | | results/metrics. | ++------------------------+------------------------+-------------------+ +| us_xs | dict | Contains relevant | +| | | HEC-RAS info for | +| | | the most upstream | +| | | cross section | +| | | associated with | +| | | the reach. | ++------------------------+------------------------+-------------------+ +| ds_xs | dict | Contains relevant | +| | | HEC-RAS info for | +| | | the most | +| | | downstream cross | +| | | section | +| | | associated with | +| | | the reach. | ++------------------------+------------------------+-------------------+ +| river | str | The river name | +| | | for the cross | +| | | section as it was | +| | | specified in the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| reach | str | The reach name | +| | | for the cross | +| | | section as it was | +| | | specified in the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| xs_id | float | The river station | +| | | for the cross | +| | | section as it was | +| | | specified in the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| min_elevation | float | The minimum | +| | | elevation in the | +| | | cross section | +| | | station-elevation | +| | | data as it was | +| | | specified in the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| max_elevation | float | The maximum | +| | | elevation in the | +| | | cross section | +| | | station-elevation | +| | | data as it was | +| | | specified in the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| eclipsed | bool | Specifies if the | +| | | reach has been | +| | | eclipsed | +| | | (covered) by an | +| | | upstream or | +| | | downstream reach. | ++------------------------+------------------------+-------------------+ +| low_flow | int | The lower flow of | +| | | the source | +| | | network data or | +| | | the HEC-RAS data | +| | | and reduced by | +| | | 20%. | ++------------------------+------------------------+-------------------+ +| high_flow | int | The higher flow | +| | | of the source | +| | | network data or | +| | | the HEC-RAS data | +| | | and increased by | +| | | 20%. | ++------------------------+------------------------+-------------------+ +| network_to_id | str | The to_id as | +| | | specified in the | +| | | attributes of the | +| | | input hydrofabric | +| | | layer. | ++------------------------+------------------------+-------------------+ +| metrics | dict | Contains the | +| | | conflation | +| | | metrics for the | +| | | reach. | ++------------------------+------------------------+-------------------+ +| xs | dict | Metrics | +| | | describing the | +| | | deviations | +| | | between | +| | | centerlines at | +| | | cross section | +| | | locationss for | +| | | the source | +| | | HEC-RAS model and | +| | | the hydrofabric. | ++------------------------+------------------------+-------------------+ +| centerline_offset | dict | Metrics | +| | | describing the | +| | | distance between | +| | | the point where | +| | | the source | +| | | HEC-RAS | +| | | centerline | +| | | intersects the | +| | | cross sections | +| | | and the point | +| | | where the source | +| | | network reach | +| | | intersects the | +| | | cross section. | ++------------------------+------------------------+-------------------+ +| thalweg_offset | dict | Metrics | +| | | describing the | +| | | distance between | +| | | the source | +| | | HEC-RAS’s XS | +| | | thalweg and the | +| | | point where the | +| | | source network | +| | | reach intersects | +| | | the cross | +| | | section. | ++------------------------+------------------------+-------------------+ +| lengths | dict | Metrics | +| | | describing the | +| | | distance between | +| | | the source | +| | | HEC-RAS’s XS | +| | | thalweg and the | +| | | point where the | +| | | source network | +| | | reach intersects | +| | | the cross | +| | | section. | ++------------------------+------------------------+-------------------+ +| ras | int | Length of the | +| | | source HEC-RAS | +| | | centerline | +| | | between the most | +| | | upstream and most | +| | | downstream cross | +| | | sections | +| | | associated with | +| | | the source | +| | | network. | ++------------------------+------------------------+-------------------+ +| network | int | Length of source | +| | | network between | +| | | the most upstream | +| | | and most | +| | | downstream cross | +| | | sections | +| | | associated with | +| | | the source | +| | | network reach. | ++------------------------+------------------------+-------------------+ +| coverage | dict | Metrics | +| | | describing the | +| | | extent of | +| | | coverage of the | +| | | source network | +| | | reach that is | +| | | covered by the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| start | float | The upstream | +| | | limit where the | +| | | source HEC-RAS | +| | | model provides | +| | | coverage. | +| | | Provided as a | +| | | ratio of the | +| | | entire source | +| | | network reach | +| | | length. | ++------------------------+------------------------+-------------------+ +| end | float | The downstream | +| | | limit where the | +| | | source HEC-RAS | +| | | model provides | +| | | coverage. | +| | | Provided as a | +| | | ratio of the | +| | | entire source | +| | | network reach. | ++------------------------+------------------------+-------------------+ +| overlapped_reaches | list | Source network | +| | | reaches that | +| | | intersect the | +| | | downstream most | +| | | cross sections of | +| | | the current | +| | | reach. | ++------------------------+------------------------+-------------------+ +| eclipsed_reaches | list | Reaches that are | +| | | eclipsed | +| | | (covered) by the | +| | | cross sections of | +| | | the current | +| | | reach. | ++------------------------+------------------------+-------------------+ +| metadata | dict | Metadata for the | +| | | conflation. | ++------------------------+------------------------+-------------------+ +| source_network | dict | Data summarizing | +| | | the source | +| | | network. | ++------------------------+------------------------+-------------------+ +| file_name | str | Source network | +| | | file name. | ++------------------------+------------------------+-------------------+ +| version | str | Source network | +| | | version. | ++------------------------+------------------------+-------------------+ +| type | str | Source network | +| | | type. | ++------------------------+------------------------+-------------------+ +| conflation_png | str | A png depicting | +| | | the conflation | +| | | results. | ++------------------------+------------------------+-------------------+ +| confl | str | The version of | +| ation_ripple1d_version | | ripple1d used to | +| | | conflate the | +| | | source HEC-RAS | +| | | model with the | +| | | source network. | ++------------------------+------------------------+-------------------+ +| me | str | The version of | +| trics_ripple1d_version | | ripple1d used to | +| | | compute the | +| | | conflation | +| | | metric. | ++------------------------+------------------------+-------------------+ +| source_ras_model | dict | Metadata for the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| stac_api | str | The stac api url | +| | | containing the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| stac_collection_id | str | The stac | +| | | collection | +| | | containing the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| stac_item_id | str | The stac item for | +| | | the source | +| | | HEC-RAS model. | ++------------------------+------------------------+-------------------+ +| source_ras_files | dict | files for the | +| | | souce HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| geometry | str | The geometry file | +| | | used from the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| forcing | str | The forcing file | +| | | used from the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| project-file | str | The project file | +| | | used from the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| plan | str | The plan file | +| | | used from the | +| | | source HEC-RAS | +| | | model. | ++------------------------+------------------------+-------------------+ +| length_units | str | Units for the | +| | | lengths specified | +| | | in the conflation | +| | | json. | ++------------------------+------------------------+-------------------+ +| flow_units | str | Units for the | +| | | flows specified | +| | | in the conflation | +| | | json. | ++------------------------+------------------------+-------------------+ + +Example output containing conflation metrics: + +.. code:: json + + { + "reaches": { + "11908582": { + "us_xs": { + "river": "Patuxent River", + "reach": "1", + "min_elevation": -10.6, + "max_elevation": 93.12, + "xs_id": 32805.59 + }, + "ds_xs": { + "river": "Patuxent River", + "reach": "1", + "min_elevation": -8.5, + "max_elevation": 53.21, + "xs_id": 26469.46 + }, + "eclipsed": false, + "low_flow_cfs": 2025, + "high_flow_cfs": 19969, + "network_to_id": "11908588", + "metrics": { + "xs": { + "centerline_offset": { + "count": 10, + "mean": 13, + "std": 8, + "min": 1, + "10%": 2, + "20%": 4, + "30%": 9, + "40%": 12, + "50%": 14, + "60%": 16, + "70%": 20, + "80%": 22, + "90%": 23, + "100%": 24, + "max": 24 + }, + "thalweg_offset": { + "count": 10, + "mean": 40, + "std": 34, + "min": 1, + "10%": 2, + "20%": 8, + "30%": 16, + "40%": 28, + "50%": 39, + "60%": 45, + "70%": 49, + "80%": 64, + "90%": 92, + "100%": 98, + "max": 98 + } + }, + "lengths": { + "ras": 6377, + "network": 6334, + "network_to_ras_ratio": 0.99 + }, + "coverage": { + "start": 0.09, + "end": 1 + } + }, + "overlapped_reaches": [ + { + "id": "11908588", + "overlap": 757 + } + ], + "eclipsed_reaches": [ + "11908584", + "11908586" + ] + }, + }, + "metadata": { + "source_network": { + "file_name": "flows.parquet", + "version": "2.1", + "type": "nwm_hydrofabric" + }, + "conflation_png": "PatuxentRiver.conflation.png", + "conflation_ripple1d_version": "0.4.2", + "metrics_ripple1d_version": "0.4.2", + "source_ras_model": { + "stac_api": "https://stac2.dewberryanalytics.com", + "stac_collection_id": "ebfe-12090301_LowerColoradoCummins", + "stac_item_id": "137a9667-e5cf-4cea-b6ec-2e882a42fdc8", + "source_ras_files": { + "geometry": "PatuxentRiver.g01", + "forcing": "PatuxentRiver.f01", + "project-file": "PatuxentRiver.prj", + "plan": "PatuxentRiver.p06" + } + }, + "length_units": "feet", + "flow_units": "cfs" + } + } diff --git a/docs/users_change_log.rst b/docs/users_change_log.rst index be84f7d0..c372d5de 100644 --- a/docs/users_change_log.rst +++ b/docs/users_change_log.rst @@ -3,10 +3,72 @@ Change Log for Users Go to the `Releases `_ page for a list of all releases. -Bugfix Release 0.4.1 + +Feature Release 0.5.0 ~~~~~~~~~~~~~~~~~~~~~ Users Changelog ---------------- + +This release of `ripple1d` incorporates geometry extraction, conflation, and conflation metrics into the API, and fixes several bugs. + + +Features Added +---------------- +**Conflation improvements** + +- The source HEC-RAS river centerline is now clipped to the most upstream and most downstream cross sections prior to starting conflation. This helps prevent identifying network reaches that are far away from the cross sections and improves the accuracy of the conflation. +- Overlapped reaches are now tracked and documented in the conflation json file. +- A bbox column has been added to the network parquet file for faster reading. This was especially needed for the new conflation endpoint since each request needs to load the parquet file. Load times without the bbox column were between 5-20 seconds; this is reduced to 1-2 seconds with the bbox column. +- The conflation function now reads locally instead of from s3. +- The conflation function no longer creates a STAC item. +- RAS metadata is now added to the conflation json. +- The source network's metadata is now added to the conflation json. +- Length and flow units are now documented in the conflation json file. + +**Conflation Metrics** + +Three metrics are computed to asses the qualitiy of the conflation: + +- `Coverage`: The the start and end location of the reach coverage is computed as a ratio of the length of the network reach. +- `Lengths`: The lengths between the most upstream cross section and most downstream cross section along the network reach and source HEC-RAS Model's centerline is computed. The ratio of the two lengths is also provided. +- `XS`: The distance between where the network reach and HEC-RAS Model's centerline intersects the cross sections is computed. A similar comparison is performed using the cross section's thalweg location and the network reaches intersection location with the cross sections. The mean, min, max, std, and quartiles are provided as a summary for both comparisons. + + +**Geometry Extraction improvements** + +- A new function to verify .prj file is a HEC-RAS file has been added. +- The extracted geopackage now contians a non-spatial metadata table for the souce HEC-RAS model. +- Tests have been added for extracting geopackage from HEC-RAS model. +- Additional attributes are added to the source model gpkg for downstream use. +- Units are extracted from the source RAS model and added to metadata. + +**API** + +- An endpoint was added for extracting geometry and relevant metdata for the soure HEC-RAS models and storing it in in a geopackage. +- An endpoint to compute conflation metric was added. +- An endpoint for conflation (which includes metrics calculations) was added. +- Tests were added for the conflation, conflation metrics, and geopackage endpoints. + + +Bug Fixes +---------- + +- Reaches whose conflation results indicate upstream and downstream cross sections are the same are now considered a failed conflation. +- The function to create a concave hull for the cross sections has been improved when junctions are present. +- Eclipsed reaches are now better identified and are documented in the conflation json with a boolean. +- A check is now performed to ensure cross sections intersect the source HEC-RAS model's river centerline. If cross sections do not intersect the centerline they are dropped. +- A conflation json is no longer written for source HEC-RAS models that fail to conflate. +- Handling has been added to subset gpkg endpoint for river stationings of interpolated. These river stations contain an "*" to indicate interpolated cross section. +- Several issues with the automated API tests were identified and fixed. +- API tests no longer re-run gpkg_from_ras and conflate_model for every reach; just once per source test model. +- When API tests pass the resulting files are now removed automatically. Resulting files for tests that fail are not removed so that the tester can better trouble shoot. + + + +Bugfix Release 0.4.1-0.4.2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Users Changelog +---------------- This release of `ripple1d` fixes several bugs identified during testing. Features Added @@ -21,9 +83,7 @@ Bug Fixes - A bug due to a hard coded terrain path name causing an error on `create_fim_lib` has been resolved. - A bug associated with the `ripple_version` parameter has been resolved by removing the parameter from the body of requests (see note in API above). - An issue with including lateral structures (not yet implemented) in the ras geometry files causing hang ups has been resolved. This fix resolved another issue where stationing was mis-applied in the newly created ras geometry files. - - ------------ +- A bug which caused a failure when calling subset_gpkg in cases where the model geometries are simple (no structures / no junctions). Feature Release 0.4.0 @@ -59,11 +119,13 @@ Features Added **API** - - `ripple_version` is no longer a required argument for any endpoint. + +- `ripple_version` is no longer a required argument for any endpoint. Bug Fixes ---------- Numerous small bug fixes were made to enable the support of hydraulic structures. Other notable bugs include: - - HEC-RAS stations with length > 8 characters are now supported. - - Mangled profile names resulting from negative elevations producing FIM libraries has been fixed. \ No newline at end of file + +- HEC-RAS stations with length > 8 characters are now supported. +- Mangled profile names resulting from negative elevations producing FIM libraries has been fixed. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 278caf54..606274a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ripple1d" -version = "0.4.2" +version = "0.5.0" description = "HEC-RAS model automation" readme = "README.md" maintainers = [ diff --git a/ripple1d/api/postman_collection.json b/ripple1d/api/postman_collection.json index 7b0e2604..57743627 100644 --- a/ripple1d/api/postman_collection.json +++ b/ripple1d/api/postman_collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "c4afca18-6b93-4c83-aa82-30306d98dcb2", + "_postman_id": "a4de6034-22c7-4a2b-b91c-12494de19040", "name": "ripple1d", "description": "Collection for processing existing HEC-RAS models for use in the production of Flood Inundation Maps (FIMs) and rating curves for use in near-real time flood forecasting on the NOAA National Water Model", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", @@ -75,7 +75,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\r\n \"src_gpkg_path\": \"{{source_model_directory}}\\\\Baxter.gpkg\",\r\n \"conflation_json\": \"{{source_model_directory}}\\\\Baxter.conflation.json\",\r\n \"nwm_pq_path\": \"{{nwm_data_directory}}\\\\flows.parquet\"\r\n \r\n}", + "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"source_network\": {\"file_name\":\"{{nwm_data_directory}}\\\\flows.parquet\",\r\n \"version\":\"2.1\", // optional\r\n \"file_hash\": \"a6786ba02073f0233b9a69afc49d92bf-178\", // optional\r\n \"type\":\"nwm_hydrofabric\"} \r\n}", "options": { "raw": { "language": "json" @@ -93,7 +93,7 @@ "execution" ] }, - "description": "Create a new model for a given nwm_id (NWM reach) from an existing HEC-RAS model." + "description": "Compute conflation metrics for a given gpkg/conflation json." }, "response": [] }, @@ -104,7 +104,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"source_network\": {\"file_name\":\"{{nwm_data_directory}}\\\\flows.parquet\",\r\n \"version\":\"2.1\",\r\n \"type\":\"nwm_hydrofabric\"}\r\n \r\n}", + "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"source_network\": {\"file_name\":\"{{nwm_data_directory}}\\\\flows.parquet\",\r\n \"version\":\"2.1\", // optional\r\n \"type\":\"nwm_hydrofabric\"}\r\n \r\n}", "options": { "raw": { "language": "json" @@ -122,7 +122,7 @@ "execution" ] }, - "description": "Create a new model for a given nwm_id (NWM reach) from an existing HEC-RAS model." + "description": "Conflate a source HEC-RAS model with the NWM network." }, "response": [] }, @@ -162,7 +162,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"crs\": 2227,\r\n \"metadata\": {\"stac_item\":\"sldakj;ga\"}\r\n}\r\n", + "raw": "{\r\n \"source_model_directory\": \"{{source_model_directory}}\",\r\n \"crs\": 2227,\r\n \"metadata\": {\"stac_api\":\"https://stac2.dewberryanalytics.com\", // optional\r\n \"stac_collection_id\":\"ebfe-12090301_LowerColoradoCummins\", // optional\r\n \"stac_item_id\":\"137a9667-e5cf-4cea-b6ec-2e882a42fdc8\"} // optional\r\n}\r\n", "options": { "raw": { "language": "json" @@ -170,13 +170,13 @@ } }, "url": { - "raw": "{{url}}/processes/extract_gpkg/execution", + "raw": "{{url}}/processes/gpkg_from_ras/execution", "host": [ "{{url}}" ], "path": [ "processes", - "extract_gpkg", + "gpkg_from_ras", "execution" ] }, @@ -238,7 +238,7 @@ "execution" ] }, - "description": "Run a range of flows for Normal Depth on a HEC-RAS model. The results will be used to inform flows required to develop rating curves at a provided increment." + "description": "Run a range of flows for dormal depth on a HEC-RAS model. The results will be used to inform flows required to develop rating curves at a provided increment." }, "response": [] }, @@ -296,7 +296,7 @@ "execution" ] }, - "description": "Run a series of HEC-RAS simulations using Known Water Surface Elevations." + "description": "Run a HEC-RAS simulation using Known Water Surface Elevations." }, "response": [] }, @@ -353,7 +353,8 @@ "nwm_reach_model_stac", "execution" ] - } + }, + "description": "Create a STAC item for a HEC-RAS model representing a NWM reach." }, "response": [] }, @@ -381,7 +382,8 @@ "fim_lib_stac", "execution" ] - } + }, + "description": "Create a STAC item for a FIM library for a NWM reah." }, "response": [] } diff --git a/ripple1d/version.py b/ripple1d/version.py index af52623d..82ed525a 100644 --- a/ripple1d/version.py +++ b/ripple1d/version.py @@ -1,3 +1,3 @@ """ripple1d version.""" -__version__ = "0.4.2" +__version__ = "0.5.0"