From 46201581749c9be2030fd9fe71c1ba94f9f5cdbd Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Tue, 17 Sep 2024 13:07:03 -0400 Subject: [PATCH 01/27] add logging for debug --- ripple1d/ops/ras_terrain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ripple1d/ops/ras_terrain.py b/ripple1d/ops/ras_terrain.py index 750bccab..6322a6d7 100644 --- a/ripple1d/ops/ras_terrain.py +++ b/ripple1d/ops/ras_terrain.py @@ -66,6 +66,7 @@ def create_ras_terrain( if resolution_units not in ["Feet", "Meters"]: raise ValueError(f"Invalid resolution_units: {resolution_units}. expected 'Feet' or 'Meters'") + logging.debug("Call NwmReachModel") nwm_rm = NwmReachModel(submodel_directory) if not nwm_rm.file_exists(nwm_rm.ras_gpkg_file): @@ -75,8 +76,10 @@ def create_ras_terrain( os.makedirs(nwm_rm.terrain_directory, exist_ok=True) # get geometry mask + logging.debug("create gdf_xs") gdf_xs = gpd.read_file(nwm_rm.ras_gpkg_file, layer="XS", driver="GPKG").explode(ignore_index=True) crs = gdf_xs.crs + logging.debug("create get_geometry_mask") mask = get_geometry_mask(gdf_xs, terrain_source_url) # clip dem @@ -103,6 +106,7 @@ def create_ras_terrain( projection_file = write_projection_file(gdf_xs.crs, nwm_rm.terrain_directory) # Make the RAS mapping terrain locally + logging.debug("create_terrain") result = create_terrain( [src_dem_reprojected_localfile], projection_file, From fe6511da18b656787722188357e42ddfe4056c8d Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Wed, 18 Sep 2024 09:17:50 -0500 Subject: [PATCH 02/27] check if reach is eclipsed Closes #190 --- ripple1d/ops/subset_gpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ripple1d/ops/subset_gpkg.py b/ripple1d/ops/subset_gpkg.py index 655ac98b..eaea2f25 100644 --- a/ripple1d/ops/subset_gpkg.py +++ b/ripple1d/ops/subset_gpkg.py @@ -479,7 +479,7 @@ def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_i raise FileNotFoundError(f"cannot find conflation file {rsd.conflation_file}, please ensure file exists") ripple1d_parameters = rsd.nwm_conflation_parameters(str(nwm_id)) - if ripple1d_parameters["us_xs"]["xs_id"] == "-9999": + if ripple1d_parameters["eclipsed"]: ripple1d_parameters["messages"] = f"skipping {nwm_id}; no cross sections conflated." logging.warning(ripple1d_parameters["messages"]) From c71e8fb000aff6ac876abbed7cd78eaf23dc1181 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 06:58:12 -0500 Subject: [PATCH 03/27] check cross section direction when calling xs_concave_hull --- ripple1d/conflate/rasfim.py | 66 +++++++++++++--------------------- ripple1d/data_model.py | 10 ++++++ ripple1d/ops/metrics.py | 12 +++++-- ripple1d/ops/ras_terrain.py | 13 +++---- ripple1d/ops/subset_gpkg.py | 13 +++++-- ripple1d/ras_to_gpkg.py | 3 +- ripple1d/utils/ripple_utils.py | 57 +++++++++++++++++++++++++++-- 7 files changed, 116 insertions(+), 58 deletions(-) diff --git a/ripple1d/conflate/rasfim.py b/ripple1d/conflate/rasfim.py index 97efcc5a..473a5668 100644 --- a/ripple1d/conflate/rasfim.py +++ b/ripple1d/conflate/rasfim.py @@ -348,9 +348,14 @@ def ras_xs_bbox(self) -> Polygon: def ras_xs_concave_hull(self, river_reach_name: str = None) -> Polygon: """Return the concave hull of the cross sections.""" if river_reach_name is None: - return xs_concave_hull(self.ras_xs,self.ras_junctions) + return xs_concave_hull(fix_reversed_xs(self.ras_xs, self.ras_centerlines), self.ras_junctions) else: - return xs_concave_hull(self.ras_xs[self.ras_xs["river_reach"] == river_reach_name]) + xs = self.ras_xs[self.ras_xs["river_reach"] == river_reach_name] + return xs_concave_hull( + fix_reversed_xs( + self.ras_xs, self.ras_centerlines.loc[self.ras_centerlines["river_reach"] == river_reach_name] + ) + ) def ras_xs_convex_hull(self, river_reach_name: str = None): """Return the convex hull of the cross sections.""" @@ -438,7 +443,11 @@ def ras_start_end_points(self, river_reach_name: str = None, centerline=None,cli def ras_centerline_by_river_reach_name(self, river_reach_name: str,clip_to_xs=False) -> LineString: """Return the centerline for the specified river reach.""" if clip_to_xs: - return self.ras_centerlines[self.ras_centerlines["river_reach"] == river_reach_name].geometry.iloc[0].intersection(self.ras_xs_concave_hull(river_reach_name).geometry.iloc[0].buffer(1)) + return ( + self.ras_centerlines[self.ras_centerlines["river_reach"] == river_reach_name] + .geometry.iloc[0] + .intersection(self.ras_xs_concave_hull(river_reach_name).geometry.iloc[0].buffer(1)) + ) else: return self.ras_centerlines[self.ras_centerlines["river_reach"] == river_reach_name].geometry.iloc[0] @@ -674,40 +683,6 @@ def get_us_most_xs_from_junction(rfc, us_river, us_reach): return ds_xs_id -def validate_point(geom): - """Validate that point is of type Point. If Multipoint or Linestring create point from first coordinate pair.""" - if isinstance(geom, Point): - return geom - elif isinstance(geom, MultiPoint): - return geom.geoms[0] - elif isinstance(geom, LineString) and list(geom.coords): - return Point(geom.coords[0]) - else: - raise TypeError(f"expected point at xs-river intersection got: {type(geom)}") - - -def check_xs_direction(cross_sections: gpd.GeoDataFrame, reach: LineString): - """Return only cross sections that are drawn right to left looking downstream.""" - ids = [] - for _, xs in cross_sections.iterrows(): - try: - point = reach.intersection(xs["geometry"]) - point = validate_point(point) - xs_rs = reach.project(point) - - offset = xs.geometry.offset_curve(-1) - point = reach.intersection(offset) - point = validate_point(point) - - offset_rs = reach.project(point) - if xs_rs > offset_rs: - ids.append(xs["ID"]) - except TypeError as e: - logging.warning(f"could not validate xs-river intersection for: {xs["river"]} {xs['reach']} {xs['river_station']}") - continue - return cross_sections.loc[cross_sections["ID"].isin(ids)] - - def map_reach_xs(rfc: RasFimConflater, reach: MultiLineString) -> dict: """ Map the upstream and downstream cross sections for the nwm reach. @@ -716,11 +691,20 @@ def map_reach_xs(rfc: RasFimConflater, reach: MultiLineString) -> dict: """ # get the xs that intersect the nwm reach intersected_xs = rfc.ras_xs[rfc.ras_xs.intersects(reach.geometry)] - intersected_xs = check_xs_direction(intersected_xs, reach.geometry) - has_junctions = rfc.ras_junctions is not None - if intersected_xs.empty: - return {"eclipsed":True} + return {"eclipsed": True} + + not_reversed_xs = check_xs_direction(intersected_xs, reach.geometry) + logging.info(len(intersected_xs)) + logging.info(len(not_reversed_xs)) + intersected_xs["geometry"] = intersected_xs.apply( + lambda row: ( + row.geometry if row["river_reach_rs"] in list(not_reversed_xs["river_reach_rs"]) else reverse(row.geometry) + ), + axis=1, + ) + + has_junctions = rfc.ras_junctions is not None # get start and end points of the nwm reach start, end = endpoints_from_multiline(reach.geometry) diff --git a/ripple1d/data_model.py b/ripple1d/data_model.py index 2ade8001..599708b1 100644 --- a/ripple1d/data_model.py +++ b/ripple1d/data_model.py @@ -16,10 +16,12 @@ from ripple1d.utils.ripple_utils import ( data_pairs_from_text_block, + fix_reversed_xs, search_contents, text_block_from_start_end_str, text_block_from_start_str_length, text_block_from_start_str_to_empty_line, + xs_concave_hull, ) from ripple1d.utils.s3_utils import init_s3_resources, read_json_from_s3 @@ -56,6 +58,14 @@ def ras_gpkg_file(self): """RAS GeoPackage file.""" return self.derive_path(".gpkg") + + @property + def xs_concave_hull(self): + """XS Concave Hull.""" + if self._xs_concave_hull is None: + self._xs_concave_hull = xs_concave_hull(fix_reversed_xs(self.ras_xs, self.ras_rivers)) + return self._xs_concave_hull + @property def assets(self): """Model assets.""" diff --git a/ripple1d/ops/metrics.py b/ripple1d/ops/metrics.py index 5be5122c..ad126dad 100644 --- a/ripple1d/ops/metrics.py +++ b/ripple1d/ops/metrics.py @@ -14,7 +14,7 @@ from ripple1d.consts import HYDROFABRIC_CRS, METERS_PER_FOOT from ripple1d.data_model import XS from ripple1d.ops.subset_gpkg import RippleGeopackageSubsetter -from ripple1d.utils.ripple_utils import xs_concave_hull +from ripple1d.utils.ripple_utils import fix_reversed_xs, xs_concave_hull class ConflationMetrics: @@ -155,7 +155,9 @@ def overlapped_reaches(self, to_reaches: gpd.GeoDataFrame) -> dict: for i, row in to_reaches.iterrows(): if row[geom_name].intersects(self.xs_gdf.union_all()): overlap = ( - row[geom_name].intersection(xs_concave_hull(self.xs_gdf)["geometry"].iloc[0]).length + row[geom_name] + .intersection(xs_concave_hull(fix_reversed_xs(self.xs_gdf, self.river_gdf))["geometry"].iloc[0]) + .length / METERS_PER_FOOT ) return [{"id": str(row["ID"]), "overlap": int(overlap)}] @@ -165,7 +167,11 @@ def eclipsed_reaches(self, network_reaches: gpd.GeoDataFrame) -> dict: """Calculate the overlap between the network reach and the cross sections.""" if network_reaches.empty: return [] - eclipsed_reaches = network_reaches[network_reaches.covered_by(xs_concave_hull(self.xs_gdf)["geometry"].iloc[0])] + eclipsed_reaches = network_reaches[ + network_reaches.covered_by( + xs_concave_hull(fix_reversed_xs(self.xs_gdf, self.river_gdf))["geometry"].iloc[0] + ) + ] return [str(row["ID"]) for _, row in eclipsed_reaches.iterrows()] diff --git a/ripple1d/ops/ras_terrain.py b/ripple1d/ops/ras_terrain.py index 750bccab..bc441643 100644 --- a/ripple1d/ops/ras_terrain.py +++ b/ripple1d/ops/ras_terrain.py @@ -20,13 +20,12 @@ from ripple1d.data_model import NwmReachModel from ripple1d.ras import create_terrain from ripple1d.utils.dg_utils import clip_raster, reproject_raster -from ripple1d.utils.ripple_utils import xs_concave_hull +from ripple1d.utils.ripple_utils import fix_reversed_xs, xs_concave_hull -def get_geometry_mask(gdf_xs: str, MAP_DEM_UNCLIPPED_SRC_URL: str) -> gpd.GeoDataFrame: +def get_geometry_mask(gdf_xs_conc_hull: str, MAP_DEM_UNCLIPPED_SRC_URL: str) -> gpd.GeoDataFrame: """Get a geometry mask for the DEM based on the cross sections.""" # build a DEM mask polygon based on the XS extents - gdf_xs_conc_hull = xs_concave_hull(gdf_xs) # Buffer the concave hull by transforming it to Albers, buffering it, then transforming it to the src raster crs with rasterio.open(MAP_DEM_UNCLIPPED_SRC_URL) as src: @@ -75,9 +74,7 @@ def create_ras_terrain( os.makedirs(nwm_rm.terrain_directory, exist_ok=True) # get geometry mask - gdf_xs = gpd.read_file(nwm_rm.ras_gpkg_file, layer="XS", driver="GPKG").explode(ignore_index=True) - crs = gdf_xs.crs - mask = get_geometry_mask(gdf_xs, terrain_source_url) + mask = get_geometry_mask(nwm_rm.xs_concave_hull, terrain_source_url) # clip dem src_dem_clipped_localfile = os.path.join(nwm_rm.terrain_directory, "temp.tif") @@ -96,11 +93,11 @@ def create_ras_terrain( # reproject/resample dem logging.debug(f"Reprojecting/Resampling DEM {src_dem_clipped_localfile} to {src_dem_clipped_localfile}") - reproject_raster(src_dem_clipped_localfile, src_dem_reprojected_localfile, crs, resolution, resolution_units) + reproject_raster(src_dem_clipped_localfile, src_dem_reprojected_localfile, nwm_rm.crs, resolution, resolution_units) os.remove(src_dem_clipped_localfile) # write projection file - projection_file = write_projection_file(gdf_xs.crs, nwm_rm.terrain_directory) + projection_file = write_projection_file(nwm_rm.crs, nwm_rm.terrain_directory) # Make the RAS mapping terrain locally result = create_terrain( diff --git a/ripple1d/ops/subset_gpkg.py b/ripple1d/ops/subset_gpkg.py index 655ac98b..3738cb75 100644 --- a/ripple1d/ops/subset_gpkg.py +++ b/ripple1d/ops/subset_gpkg.py @@ -12,7 +12,7 @@ import ripple1d from ripple1d.consts import METERS_PER_FOOT from ripple1d.data_model import NwmReachModel, RippleSourceDirectory, RippleSourceModel -from ripple1d.utils.ripple_utils import xs_concave_hull +from ripple1d.utils.ripple_utils import fix_reversed_xs, xs_concave_hull class RippleGeopackageSubsetter: @@ -161,7 +161,9 @@ def write_ripple_gpkg( if gdf.shape[0] > 0: gdf.to_file(self.ripple_gpkg_file, layer=layer) if layer == "XS": - xs_concave_hull(gdf).to_file(self.ripple_gpkg_file, driver="GPKG", layer="XS_concave_hull") + xs_concave_hull(fix_reversed_xs(gdf, self.subset_gdfs["River"])).to_file( + self.ripple_gpkg_file, driver="GPKG", layer="XS_concave_hull" + ) @property def min_flow(self) -> float: @@ -432,7 +434,12 @@ def clip_river(self, xs_subset_gdf: gpd.GeoDataFrame, river_subset_gdf: gpd.GeoD buffer = 10 while True: - concave_hull = xs_concave_hull(xs_subset_gdf).to_crs(epsg=5070).buffer(buffer * METERS_PER_FOOT).to_crs(crs) + concave_hull = ( + xs_concave_hull(fix_reversed_xs(xs_subset_gdf, river_subset_gdf)) + .to_crs(epsg=5070) + .buffer(buffer * METERS_PER_FOOT) + .to_crs(crs) + ) clipped_river_subset_gdf = river_subset_gdf.clip(concave_hull) buffer += 10 diff --git a/ripple1d/ras_to_gpkg.py b/ripple1d/ras_to_gpkg.py index 4ef3475c..58327bb0 100644 --- a/ripple1d/ras_to_gpkg.py +++ b/ripple1d/ras_to_gpkg.py @@ -30,7 +30,7 @@ reproject, write_thumbnail_to_s3, ) -from ripple1d.utils.ripple_utils import get_path, prj_is_ras, xs_concave_hull +from ripple1d.utils.ripple_utils import fix_reversed_xs, get_path, prj_is_ras, xs_concave_hull from ripple1d.utils.s3_utils import ( get_basic_object_metadata, init_s3_resources, @@ -48,6 +48,7 @@ def geom_flow_to_gpkg( layers, metadata = geom_flow_to_gdfs(ras_project, crs, metadata, client, bucket) for layer, gdf in layers.items(): if layer == "XS": + gdf = fix_reversed_xs(layers["XS"], layers["River"]) if "Junction" in layers.keys(): xs_concave_hull(gdf, layers["Junction"]).to_file(gpkg_file, driver="GPKG", layer="XS_concave_hull") else: diff --git a/ripple1d/utils/ripple_utils.py b/ripple1d/utils/ripple_utils.py index bd50b7ce..8504328e 100644 --- a/ripple1d/utils/ripple_utils.py +++ b/ripple1d/utils/ripple_utils.py @@ -3,6 +3,7 @@ from __future__ import annotations import glob +import logging import os from pathlib import Path @@ -10,8 +11,18 @@ import geopandas as gpd import pandas as pd from dotenv import find_dotenv, load_dotenv -from shapely import Polygon, concave_hull, line_merge, make_valid, union_all -from shapely.geometry import MultiPolygon, Point, Polygon +from shapely import ( + LineString, + MultiPoint, + MultiPolygon, + Point, + Polygon, + concave_hull, + line_merge, + make_valid, + reverse, + union_all, +) from ripple1d.errors import ( RASComputeError, @@ -62,6 +73,48 @@ def get_path(expected_path: str, client: boto3.client = None, bucket: str = None if path.endswith(Path(expected_path).suffix.upper()): return path +def fix_reversed_xs(xs:gpd.GeoDataFrame,river:gpd.GeoDataFrame)->gpd.GeoDataFrame: + """Check if cross sections are drawn right to left looking downstream. If not reverse them.""" + subsets=[] + for _,reach in river.iterrows(): + subset_xs=xs.loc[xs["river_reach"]== reach["river_reach"]] + not_reversed_xs=check_xs_direction(subset_xs,reach.geometry) + subset_xs["geometry"]=subset_xs.apply(lambda row: row.geometry if row["river_reach_rs"] in list(not_reversed_xs["river_reach_rs"]) else reverse(row.geometry),axis=1) + subsets.append(subset_xs) + return pd.concat(subsets) + +def validate_point(geom): + """Validate that point is of type Point. If Multipoint or Linestring create point from first coordinate pair.""" + if isinstance(geom, Point): + return geom + elif isinstance(geom, MultiPoint): + return geom.geoms[0] + elif isinstance(geom, LineString) and list(geom.coords): + return Point(geom.coords[0]) + else: + raise TypeError(f"expected point at xs-river intersection got: {type(geom)}") + +def check_xs_direction(cross_sections: gpd.GeoDataFrame, reach: LineString): + """Return only cross sections that are drawn right to left looking downstream.""" + river_reach_rs = [] + for _, xs in cross_sections.iterrows(): + try: + point = reach.intersection(xs["geometry"]) + point = validate_point(point) + xs_rs = reach.project(point) + + offset = xs.geometry.offset_curve(-1) + point = reach.intersection(offset) + point = validate_point(point) + + offset_rs = reach.project(point) + if xs_rs > offset_rs: + river_reach_rs.append(xs["river_reach_rs"]) + + except TypeError as e: + logging.warning(f"could not validate xs-river intersection for: {xs["river"]} {xs['reach']} {xs['river_station']}") + continue + return cross_sections.loc[cross_sections["river_reach_rs"].isin(river_reach_rs)] def xs_concave_hull(xs: gpd.GeoDataFrame, junction: gpd.GeoDataFrame = None) -> gpd.GeoDataFrame: """Compute and return the concave hull (polygon) for a set of cross sections (lines all facing the same direction).""" From 5178bd2622f157bb30941048ff08a46faf503d42 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 06:59:22 -0500 Subject: [PATCH 04/27] load gdfs only once. --- ripple1d/data_model.py | 34 +++++++++++++++++ ripple1d/ops/subset_gpkg.py | 75 ++++++++++++++++++++++--------------- 2 files changed, 79 insertions(+), 30 deletions(-) diff --git a/ripple1d/data_model.py b/ripple1d/data_model.py index 599708b1..d0da3a2b 100644 --- a/ripple1d/data_model.py +++ b/ripple1d/data_model.py @@ -32,6 +32,11 @@ class RasModelStructure: def __init__(self, model_directory: str): self.model_directory = model_directory self.model_basename = Path(model_directory).name + self._ras_junctions = None + self._ras_structures = None + self._ras_xs = None + self._ras_rivers = None + self._xs_concave_hull = None @property def model_name(self): @@ -58,6 +63,35 @@ def ras_gpkg_file(self): """RAS GeoPackage file.""" return self.derive_path(".gpkg") + @property + def ras_xs(self): + """RAS XS Geodataframe.""" + if self._ras_xs is None: + self._ras_xs = gpd.read_file(self.ras_gpkg_file, layer="XS") + return self._ras_xs + + @property + def ras_junctions(self): + """RAS Junctions Geodataframe.""" + if "Junction" in fiona.listlayers(self.ras_gpkg_file): + if self._ras_junctions is None: + self._ras_junctions = gpd.read_file(self.ras_gpkg_file, layer="Junction") + return self._ras_junctions + + @property + def ras_structures(self): + """RAS Structures Geodataframe.""" + if "Structure" in fiona.listlayers(self.ras_gpkg_file): + if self._ras_structures is None: + self._ras_structures = gpd.read_file(self.ras_gpkg_file, layer="Structure") + return self._ras_structures + + @property + def ras_rivers(self): + """RAS Rivers Geodataframe.""" + if self._ras_rivers is None: + self._ras_rivers = gpd.read_file(self.ras_gpkg_file, layer="River") + return self._ras_rivers @property def xs_concave_hull(self): diff --git a/ripple1d/ops/subset_gpkg.py b/ripple1d/ops/subset_gpkg.py index 3738cb75..9d33e16e 100644 --- a/ripple1d/ops/subset_gpkg.py +++ b/ripple1d/ops/subset_gpkg.py @@ -24,6 +24,11 @@ def __init__(self, src_gpkg_path: str, conflation_json: str, dst_project_dir: st self.conflation_json = conflation_json self.dst_project_dir = dst_project_dir self.nwm_id = nwm_id + self._subset_gdf = None + self._source_junction = None + self._source_river = None + self._source_xs = None + self._source_structure = None @property def conflation_parameters(self) -> dict: @@ -69,27 +74,34 @@ def ds_rs(self) -> str: @property def source_xs(self) -> gpd.GeoDataFrame: """Extract cross sections from the source geopackage.""" - xs = gpd.read_file(self.src_gpkg_path, layer="XS") - return xs[xs.intersects(self.source_river.union_all())] + if self._source_xs is None: + xs = gpd.read_file(self.src_gpkg_path, layer="XS") + self._source_xs = xs[xs.intersects(self.source_river.union_all())] + return self._source_xs @property def source_river(self) -> gpd.GeoDataFrame: """Extract river geometry from the source geopackage.""" - return gpd.read_file(self.src_gpkg_path, layer="River") + if self._source_river is None: + self._source_river = gpd.read_file(self.src_gpkg_path, layer="River") + return self._source_river @property def source_structure(self) -> gpd.GeoDataFrame: """Extract structures from the source geopackage.""" if "Structure" in fiona.listlayers(self.src_gpkg_path): - structures = gpd.read_file(self.src_gpkg_path, layer="Structure") - return structures[structures.intersects(self.source_river.union_all())] + if self._source_structure is None: + structures = gpd.read_file(self.src_gpkg_path, layer="Structure") + self._source_structure = structures[structures.intersects(self.source_river.union_all())] + return self._source_structure @property def source_junction(self) -> gpd.GeoDataFrame: """Extract junctions from the source geopackage.""" if "Junction" in fiona.listlayers(self.src_gpkg_path): - return gpd.read_file(self.src_gpkg_path, layer="Junction") - return None + if self._source_junction is None: + self._source_junction = gpd.read_file(self.src_gpkg_path, layer="Junction") + return self._source_junction @property def ripple_xs(self) -> gpd.GeoDataFrame: @@ -109,29 +121,32 @@ def ripple_structure(self) -> gpd.GeoDataFrame: @property def subset_gdfs(self) -> dict: """Subset the cross sections, structues, and river geometry for a given NWM reach.""" - # subset data - if self.us_river == self.ds_river and self.us_reach == self.ds_reach: - ripple_xs, ripple_structure, ripple_river = self.process_as_one_ras_reach() - else: - ripple_xs, ripple_structure, ripple_river = self.process_as_multiple_ras_reach() - - # check if only 1 cross section for nwm_reach - if len(ripple_xs) <= 1: - logging.warning(f"Only 1 cross section conflated to NWM reach {self.nwm_id}. Skipping this reach.") - return None - - # update fields - ripple_xs = self.update_fields(ripple_xs) - ripple_river = self.update_fields(ripple_river) - ripple_structure = self.update_fields(ripple_structure) - - # clip river to cross sections - ripple_river = self.clip_river(ripple_xs, ripple_river) - - if ripple_structure is not None and len(ripple_structure) > 0: - return {"XS": ripple_xs, "River": ripple_river, "Structure": ripple_structure} - else: - return {"XS": ripple_xs, "River": ripple_river} + if self._subset_gdf is None: + # subset data + if self.us_river == self.ds_river and self.us_reach == self.ds_reach: + ripple_xs, ripple_structure, ripple_river = self.process_as_one_ras_reach() + else: + ripple_xs, ripple_structure, ripple_river = self.process_as_multiple_ras_reach() + + # check if only 1 cross section for nwm_reach + if len(ripple_xs) <= 1: + logging.warning(f"Only 1 cross section conflated to NWM reach {self.nwm_id}. Skipping this reach.") + return None + + # update fields + ripple_xs = self.update_fields(ripple_xs) + ripple_river = self.update_fields(ripple_river) + ripple_structure = self.update_fields(ripple_structure) + + # clip river to cross sections + ripple_river = self.clip_river(ripple_xs, ripple_river) + + if ripple_structure is not None and len(ripple_structure) > 0: + self._subset_gdf = {"XS": ripple_xs, "River": ripple_river, "Structure": ripple_structure} + else: + self._subset_gdf = {"XS": ripple_xs, "River": ripple_river} + + return self._subset_gdf @property def ripple_gpkg_file(self) -> str: From e1aa8c8e899ad5c311fb7932261c3b21c35a8b5d Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 06:59:44 -0500 Subject: [PATCH 05/27] update formatting --- ripple1d/conflate/rasfim.py | 80 +++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/ripple1d/conflate/rasfim.py b/ripple1d/conflate/rasfim.py index 473a5668..2644241c 100644 --- a/ripple1d/conflate/rasfim.py +++ b/ripple1d/conflate/rasfim.py @@ -13,11 +13,11 @@ import pandas as pd import pyproj from fiona.errors import DriverError -from shapely.geometry import LineString, MultiLineString, MultiPoint, Point, Polygon, box +from shapely import LineString, MultiLineString, MultiPoint, Point, Polygon, box, reverse from shapely.ops import linemerge, nearest_points, transform from ripple1d.consts import METERS_PER_FOOT -from ripple1d.utils.ripple_utils import xs_concave_hull +from ripple1d.utils.ripple_utils import check_xs_direction, fix_reversed_xs, validate_point, xs_concave_hull HIGH_FLOW_FACTOR = 1.2 @@ -74,7 +74,7 @@ def __init__( self.nwm_pq = nwm_pq self.source_model_directory = source_model_directory self.ras_model_name = os.path.basename(source_model_directory) - self.ras_gpkg = os.path.join(source_model_directory,f"{self.ras_model_name}.gpkg") + self.ras_gpkg = os.path.join(source_model_directory, f"{self.ras_model_name}.gpkg") self.output_concave_hull_path = output_concave_hull_path self._nwm_reaches = None @@ -83,7 +83,7 @@ def __init__( self._ras_xs = None self._ras_structures = None self._ras_junctions = None - self._ras_metadata=None + self._ras_metadata = None self._common_crs = None self._xs_hulls = None self.__data_loaded = False @@ -91,28 +91,25 @@ def __init__( self._common_crs = NWM_CRS self.load_data() self.__data_loaded = True - def __repr__(self): """Return the string representation of the object.""" return f"RasFimConflater(nwm_pq={self.nwm_pq}, ras_gpkg={self.ras_gpkg})" - @property def stac_api(self): """The stac_api for the HEC-RAS Model.""" if self.ras_metadata: if "stac_api" in self.ras_metadata.keys(): return self.ras_metadata["stac_api"] - + @property def stac_collection_id(self): """The stac_collection_id for the HEC-RAS Model.""" if self.ras_metadata: if "stac_collection_id" in self.ras_metadata.keys(): return self.ras_metadata["stac_collection_id"] - - + @property def stac_item_id(self): """The stac_item_id for the HEC-RAS Model.""" @@ -131,7 +128,7 @@ def primary_flow_file(self): """The primary flow file for the HEC-RAS Model.""" if self.ras_metadata: return self.ras_metadata["primary_flow_file"] - + @property def primary_plan_file(self): """The primary plan file for the HEC-RAS Model.""" @@ -142,7 +139,7 @@ def primary_plan_file(self): def ras_project_file(self): """The source HEC-RAS project file.""" if self.ras_metadata: - return self.ras_metadata["ras_project_file"] + return self.ras_metadata["ras_project_file"] # @property # def xs_length_units(self): @@ -186,8 +183,7 @@ def ras_project_file(self): # return "miles" # else: # raise ValueError(f"Unable to determine reach length units from reach length r values") - - + # @property # def flow_units(self): # """Flow units of the source HEC-RAS model.""" @@ -198,11 +194,11 @@ def ras_project_file(self): # elif self.ras_metadata["units"] == "English": # return "cfs" - def populate_r_station(self, row: pd.Series,assume_ft:bool=True) -> str: + def populate_r_station(self, row: pd.Series, assume_ft: bool = True) -> str: """Populate the r value for a cross section. The r value is the ratio of the station to actual cross section length.""" - #TODO check if this is the correct way to calculate r + # TODO check if this is the correct way to calculate r df = pd.DataFrame(row["station_elevation_points"], index=["elevation"]).T - return df.index.max()/(row.geometry.length/METERS_PER_FOOT) + return df.index.max() / (row.geometry.length / METERS_PER_FOOT) @property def _gpkg_metadata(self): @@ -212,24 +208,24 @@ def _gpkg_metadata(self): cur.execute("select * from metadata") return dict(cur.fetchall()) - def determine_station_order(self, xs_gdf: gpd.GeoDataFrame,reach:LineString): + def determine_station_order(self, xs_gdf: gpd.GeoDataFrame, reach: LineString): """Detemine the order based on stationing of the cross sections along the reach.""" - rs=[] + rs = [] for _, xs in xs_gdf.iterrows(): - geom=reach.intersection(xs.geometry) + geom = reach.intersection(xs.geometry) try: - point=validate_point(geom) + point = validate_point(geom) rs.append(reach.project(point)) except TypeError as e: rs.append(rs[-1]) - - xs_gdf["rs"]=rs - return xs_gdf.sort_values(by="rs",ignore_index=True) - def add_hull(self, xs_gdf: gpd.GeoDataFrame,reach:LineString): + xs_gdf["rs"] = rs + return xs_gdf.sort_values(by="rs", ignore_index=True) + + def add_hull(self, xs_gdf: gpd.GeoDataFrame, reach: LineString): """Add the concave hull to the GeoDataFrame.""" if len(xs_gdf) > 1: - xs_gdf=self.determine_station_order(xs_gdf,reach) + xs_gdf = self.determine_station_order(xs_gdf, reach) hull = xs_concave_hull(xs_gdf) if self._xs_hulls is None: self._xs_hulls = hull @@ -253,20 +249,20 @@ def load_gpkg(self, gpkg: str): if "River" in layers: self._ras_centerlines = gpd.read_file(self.ras_gpkg, layer="River") if "XS" in layers: - xs=gpd.read_file(self.ras_gpkg, layer="XS") + xs = gpd.read_file(self.ras_gpkg, layer="XS") self._ras_xs = xs[xs.intersects(self._ras_centerlines.union_all())] if "Junction" in layers: self._ras_junctions = gpd.read_file(self.ras_gpkg, layer="Junction") if "Structure" in layers: - structures=gpd.read_file(self.ras_gpkg, layer="Structure") + structures = gpd.read_file(self.ras_gpkg, layer="Structure") self._ras_structures = structures[structures.intersects(self._ras_centerlines.union_all())] if "metadata" in layers: - self._ras_metadata=self._gpkg_metadata + self._ras_metadata = self._gpkg_metadata def load_pq(self, nwm_pq: str): """Load the NWM data from the Parquet file.""" try: - nwm_reaches = gpd.read_parquet(nwm_pq,bbox=self._ras_xs.to_crs(self.common_crs).total_bounds) + nwm_reaches = gpd.read_parquet(nwm_pq, bbox=self._ras_xs.to_crs(self.common_crs).total_bounds) nwm_reaches = nwm_reaches.rename(columns={"geom": "geometry"}) self._nwm_reaches = nwm_reaches.set_geometry("geometry") except Exception as e: @@ -322,7 +318,7 @@ def ras_structures(self) -> gpd.GeoDataFrame: return self._ras_structures.to_crs(self.common_crs) except AttributeError: return None - + @property @ensure_data_loaded def ras_junctions(self) -> gpd.GeoDataFrame: @@ -434,13 +430,15 @@ def wrapper(self, *args, **kwargs): return wrapper @check_centerline - def ras_start_end_points(self, river_reach_name: str = None, centerline=None,clip_to_xs=False) -> Tuple[Point, Point]: + def ras_start_end_points( + self, river_reach_name: str = None, centerline=None, clip_to_xs=False + ) -> Tuple[Point, Point]: """River_reach_name used by the decorator to get the centerline.""" if river_reach_name: - centerline = self.ras_centerline_by_river_reach_name(river_reach_name,clip_to_xs) + centerline = self.ras_centerline_by_river_reach_name(river_reach_name, clip_to_xs) return endpoints_from_multiline(centerline) - def ras_centerline_by_river_reach_name(self, river_reach_name: str,clip_to_xs=False) -> LineString: + def ras_centerline_by_river_reach_name(self, river_reach_name: str, clip_to_xs=False) -> LineString: """Return the centerline for the specified river reach.""" if clip_to_xs: return ( @@ -749,16 +747,12 @@ def map_reach_xs(rfc: RasFimConflater, reach: MultiLineString) -> dict: # add xs concave hull if rfc.output_concave_hull_path: xs_gdf = pd.concat([intersected_xs, rfc.ras_xs[rfc.ras_xs["ID"] == ds_xs]], ignore_index=True) - rfc.add_hull(xs_gdf,reach.geometry) - - if us_data==ds_data: - return {"eclipsed":True} - - return { - "us_xs": us_data, - "ds_xs": ds_data, - "eclipsed":False - } + rfc.add_hull(xs_gdf, reach.geometry) + + if us_data == ds_data: + return {"eclipsed": True} + + return {"us_xs": us_data, "ds_xs": ds_data, "eclipsed": False} def ras_reaches_metadata(rfc: RasFimConflater, candidate_reaches: gpd.GeoDataFrame): From 6af0bbe8b82d1a2627511092f9a69a925bc85954 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 07:00:23 -0500 Subject: [PATCH 06/27] add FIM_LIB_DIRECTORY to test data --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index c203ca71..2dd063d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,3 +57,4 @@ def setup_data(request): request.cls.min_elevation = MIN_ELEVATION request.cls.conflation_file = os.path.join(SOURCE_RAS_MODEL_DIRECTORY, f"{RAS_MODEL}.conflation.json") request.cls.crs = CRS[RAS_MODEL] + request.cls.FIM_LIB_DIRECTORY = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}\\fims") From 7cbacd90103dc4246d15913311a76711078b25a7 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 09:05:47 -0500 Subject: [PATCH 07/27] pass library _directory to NwmReachModel --- ripple1d/data_model.py | 5 +++-- ripple1d/ops/fim_lib.py | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ripple1d/data_model.py b/ripple1d/data_model.py index 2ade8001..1ce8a4ac 100644 --- a/ripple1d/data_model.py +++ b/ripple1d/data_model.py @@ -198,8 +198,9 @@ def nwm_conflation_parameters(self, nwm_id: str): class NwmReachModel(RasModelStructure): """National Water Model reach-based HEC-RAS Model files and directory structure.""" - def __init__(self, model_directory: str): + def __init__(self, model_directory: str, library_directory: str = ""): super().__init__(model_directory) + self.library_directory = library_directory @property def terrain_directory(self): @@ -214,7 +215,7 @@ def ras_terrain_hdf(self): @property def fim_results_directory(self): """FIM results directory.""" - return str(Path(self.model_directory) / "fims") + return str(Path(self.library_directory) / self.model_name) @property def fim_lib_assets(self): diff --git a/ripple1d/ops/fim_lib.py b/ripple1d/ops/fim_lib.py index dff267e0..a3492658 100644 --- a/ripple1d/ops/fim_lib.py +++ b/ripple1d/ops/fim_lib.py @@ -139,13 +139,14 @@ def create_fim_lib( resolution_units: str = "Meters", ): """Create a new FIM library for a NWM id.""" - nwm_rm = NwmReachModel(submodel_directory) - if not nwm_rm.file_exists(nwm_rm.ras_gpkg_file): - raise FileNotFoundError(f"cannot find ras_gpkg_file file {nwm_rm.ras_gpkg_file}, please ensure file exists") + nwm_rm = NwmReachModel(submodel_directory, library_directory) - crs = gpd.read_file(nwm_rm.ras_gpkg_file, layer="XS").crs - - rm = RasManager(nwm_rm.ras_project_file, version=ras_version, terrain_path=nwm_rm.ras_terrain_hdf, crs=crs) + rm = RasManager( + nwm_rm.ras_project_file, + version=ras_version, + terrain_path=nwm_rm.ras_terrain_hdf, + crs=nwm_rm.crs, + ) ras_plans = [f"{nwm_rm.model_name}_{plan}" for plan in plans] missing_grids_kwse, missing_grids_nd = post_process_depth_grids( From e32875169eb0fc3961dce570f559be5e8ba558b8 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 09:07:26 -0500 Subject: [PATCH 08/27] handle empty folder during cleanup --- ripple1d/ops/fim_lib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ripple1d/ops/fim_lib.py b/ripple1d/ops/fim_lib.py index a3492658..fefb07cd 100644 --- a/ripple1d/ops/fim_lib.py +++ b/ripple1d/ops/fim_lib.py @@ -174,7 +174,7 @@ def create_fim_lib( table_name, ) if cleanup: - shutil.rmtree(os.path.join(rm.ras_project._ras_dir, f"{nwm_rm.model_name}_kwse")) + shutil.rmtree(os.path.join(rm.ras_project._ras_dir, f"{nwm_rm.model_name}_kwse"), ignore_errors=True) if f"nd" in plan: zero_depth_to_sqlite( rm, @@ -185,7 +185,7 @@ def create_fim_lib( table_name, ) if cleanup: - shutil.rmtree(os.path.join(rm.ras_project._ras_dir, f"{nwm_rm.model_name}_nd")) + shutil.rmtree(os.path.join(rm.ras_project._ras_dir, f"{nwm_rm.model_name}_nd"), ignore_errors=True) return {"fim_results_directory": nwm_rm.fim_results_directory, "fim_results_database": nwm_rm.fim_results_database} From 54438dadf4b4f44c52920416d7c7edbd42e66967 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 09:07:44 -0500 Subject: [PATCH 09/27] check eclipsed reaches --- ripple1d/ops/ras_run.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/ripple1d/ops/ras_run.py b/ripple1d/ops/ras_run.py index 7532bbd4..fdcb2bfe 100644 --- a/ripple1d/ops/ras_run.py +++ b/ripple1d/ops/ras_run.py @@ -31,7 +31,7 @@ def create_model_run_normal_depth( if not nwm_rm.file_exists(nwm_rm.ras_gpkg_file): raise FileNotFoundError(f"cannot find ras_gpkg_file file {nwm_rm.ras_gpkg_file}, please ensure file exists") - if nwm_rm.ripple1d_parameters["us_xs"]["xs_id"] == "-9999": + if nwm_rm.ripple1d_parameters["eclipsed"] == True: logging.warning(f"skipping {nwm_rm.model_name}; no cross sections conflated.") else: logging.info(f"Working on initial normal depth run for nwm_id: {nwm_rm.model_name}") @@ -80,18 +80,15 @@ def run_incremental_normal_depth( if not nwm_rm.file_exists(nwm_rm.conflation_file): raise FileNotFoundError(f"cannot find conflation file {nwm_rm.conflation_file}, please ensure file exists") - if not nwm_rm.file_exists(nwm_rm.ras_gpkg_file): - raise FileNotFoundError(f"cannot find ras_gpkg_file file {nwm_rm.ras_gpkg_file}, please ensure file exists") - logging.info(f"Working on normal depth run for nwm_id: {nwm_rm.model_name}") - if nwm_rm.ripple1d_parameters["us_xs"]["xs_id"] == "-9999": + if nwm_rm.ripple1d_parameters["eclipsed"] == True: logging.warning(f"skipping {nwm_rm.model_name}; no cross sections conflated.") rm = RasManager( nwm_rm.ras_project_file, version=ras_version, terrain_path=nwm_rm.ras_terrain_hdf, - crs=nwm_rm.ripple1d_parameters["crs"], + crs=nwm_rm.crs, ) # determine flow increments @@ -139,18 +136,13 @@ def run_known_wse( if not nwm_rm.file_exists(nwm_rm.conflation_file): raise FileNotFoundError(f"cannot find conflation file {nwm_rm.conflation_file}, please ensure file exists") - if not nwm_rm.file_exists(nwm_rm.ras_gpkg_file): - raise FileNotFoundError(f"cannot find ras_gpkg_file file {nwm_rm.ras_gpkg_file}, please ensure file exists") - logging.info(f"Working on known water surface elevation run for nwm_id: {nwm_rm.model_name}") start_elevation = np.floor(min_elevation * 2) / 2 # round down to nearest .0 or .5 known_water_surface_elevations = np.arange(start_elevation, max_elevation + depth_increment, depth_increment) - crs = gpd.read_file(nwm_rm.ras_gpkg_file, layer="XS").crs - # write and compute flow/plans for known water surface elevation runs - rm = RasManager(nwm_rm.ras_project_file, version=ras_version, terrain_path=nwm_rm.ras_terrain_hdf, crs=crs) + rm = RasManager(nwm_rm.ras_project_file, version=ras_version, terrain_path=nwm_rm.ras_terrain_hdf, crs=nwm_rm.crs) # get resulting depths from the second normal depth runs_nd rm.plan = rm.plans[f"{nwm_rm.model_name}_nd"] From a0fd230996075e4995c59839daaffd12d70d40a7 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 09:08:04 -0500 Subject: [PATCH 10/27] update tests --- tests/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c203ca71..dca38812 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,7 @@ def setup_data(request): SOURCE_RAS_MODEL_DIRECTORY = os.path.join(TEST_DIR, f"ras-data\\{RAS_MODEL}") SUBMODELS_BASE_DIRECTORY = os.path.join(SOURCE_RAS_MODEL_DIRECTORY, "submodels") SUBMODELS_DIRECTORY = os.path.join(SUBMODELS_BASE_DIRECTORY, REACH_ID) + FIM_LIB_DIRECTORY = os.path.join(SUBMODELS_DIRECTORY, "fim") request.cls.REACH_ID = RAS_MODEL request.cls.REACH_ID = REACH_ID @@ -33,6 +34,7 @@ def setup_data(request): request.cls.SOURCE_RAS_MODEL_DIRECTORY = SOURCE_RAS_MODEL_DIRECTORY request.cls.SUBMODELS_BASE_DIRECTORY = SUBMODELS_BASE_DIRECTORY request.cls.SUBMODELS_DIRECTORY = SUBMODELS_DIRECTORY + request.cls.FIM_LIB_DIRECTORY = FIM_LIB_DIRECTORY request.cls.GPKG_FILE = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}.gpkg") request.cls.SOURCE_GPKG_FILE = os.path.join(SOURCE_RAS_MODEL_DIRECTORY, f"{RAS_MODEL}.gpkg") request.cls.TERRAIN_HDF = os.path.join(SUBMODELS_DIRECTORY, f"Terrain\\{REACH_ID}.hdf") @@ -48,10 +50,10 @@ def setup_data(request): request.cls.PLAN3_FILE = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}.p03") request.cls.FLOW3_FILE = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}.f03") request.cls.RESULT3_FILE = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}.r03") - request.cls.FIM_LIB_DB = os.path.join(SUBMODELS_DIRECTORY, f"fims\\{REACH_ID}.db") - request.cls.DEPTH_GRIDS_ND = os.path.join(SUBMODELS_DIRECTORY, f"fims\\z_nd") + request.cls.FIM_LIB_DB = os.path.join(FIM_LIB_DIRECTORY, f"{REACH_ID}\\{REACH_ID}.db") + request.cls.DEPTH_GRIDS_ND = os.path.join(FIM_LIB_DIRECTORY, f"{REACH_ID}\\z_nd") integer, decimal = str(np.floor((MIN_ELEVATION + 41) * 2) / 2).split(".") - request.cls.DEPTH_GRIDS_KWSE = os.path.join(SUBMODELS_DIRECTORY, f"fims\\z_{integer}_{decimal}") + request.cls.DEPTH_GRIDS_KWSE = os.path.join(FIM_LIB_DIRECTORY, f"{REACH_ID}\\z_{integer}_{decimal}") request.cls.MODEL_STAC_ITEM = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}.model.stac.json") request.cls.FIM_LIB_STAC_ITEM = os.path.join(SUBMODELS_DIRECTORY, f"fims\\{REACH_ID}.fim_lib.stac.json") request.cls.min_elevation = MIN_ELEVATION From 636090308041ee5d7b2f24cd171dbec0ad4e05c2 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 10:45:05 -0500 Subject: [PATCH 11/27] append to ratingcurve db if it already exists --- ripple1d/ops/fim_lib.py | 3 ++- ripple1d/utils/sqlite_utils.py | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ripple1d/ops/fim_lib.py b/ripple1d/ops/fim_lib.py index fefb07cd..5d3a92f2 100644 --- a/ripple1d/ops/fim_lib.py +++ b/ripple1d/ops/fim_lib.py @@ -161,7 +161,8 @@ def create_fim_lib( ) # create dabase and table - create_db_and_table(nwm_rm.fim_results_database, table_name) + if not os.path.exists(nwm_rm.fim_results_database): + create_db_and_table(nwm_rm.fim_results_database, table_name) for plan in plans: if f"kwse" in plan: diff --git a/ripple1d/utils/sqlite_utils.py b/ripple1d/utils/sqlite_utils.py index f3092e62..5b909843 100644 --- a/ripple1d/utils/sqlite_utils.py +++ b/ripple1d/utils/sqlite_utils.py @@ -10,9 +10,6 @@ def create_db_and_table(db_name: str, table_name: str): """Create sqlite database and table.""" - if os.path.exists(db_name): - os.remove(db_name) - sql_query = f""" CREATE TABLE {table_name}( reach_id INTEGER, From e120712881931cc3cf6f0e6c211018de39beff5a Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 13:33:37 -0400 Subject: [PATCH 12/27] add ProcessPoolExecutor to fix long waits for ras_terrain --- ripple1d/ops/ras_terrain.py | 39 +++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/ripple1d/ops/ras_terrain.py b/ripple1d/ops/ras_terrain.py index 6322a6d7..8c02793f 100644 --- a/ripple1d/ops/ras_terrain.py +++ b/ripple1d/ops/ras_terrain.py @@ -5,6 +5,7 @@ import json import logging import os +from concurrent.futures import ProcessPoolExecutor from pathlib import Path import geopandas as gpd @@ -53,34 +54,36 @@ def create_ras_terrain( vertical_units: str = MAP_DEM_VERT_UNITS, resolution: float = None, resolution_units: str = None, + task_id: str = "", ) -> None: """Create a RAS terrain file.""" - logging.info(f"Processing: {submodel_directory}") + logging.info(f"{task_id}: create_ras_terrain start") if resolution and not resolution_units: raise ValueError( - f"The 'resolution' arg has been provided but 'resolution_units' arg has not been provided. Please provide both" + f"{task_id} | 'resolution' arg has been provided but 'resolution_units' arg has not been provided. Please provide both" ) if resolution_units: if resolution_units not in ["Feet", "Meters"]: - raise ValueError(f"Invalid resolution_units: {resolution_units}. expected 'Feet' or 'Meters'") + raise ValueError(f"{task_id} | invalid resolution_units: {resolution_units}. expected 'Feet' or 'Meters'") - logging.debug("Call NwmReachModel") nwm_rm = NwmReachModel(submodel_directory) if not nwm_rm.file_exists(nwm_rm.ras_gpkg_file): - raise FileNotFoundError(f"Expecting {nwm_rm.ras_gpkg_file}, file not found") + raise FileNotFoundError( + f"{task_id} | NwmReachModel class expecting ras_gpkg_file {nwm_rm.ras_gpkg_file}, file not found" + ) if not os.path.exists(nwm_rm.terrain_directory): os.makedirs(nwm_rm.terrain_directory, exist_ok=True) - # get geometry mask - logging.debug("create gdf_xs") gdf_xs = gpd.read_file(nwm_rm.ras_gpkg_file, layer="XS", driver="GPKG").explode(ignore_index=True) crs = gdf_xs.crs - logging.debug("create get_geometry_mask") - mask = get_geometry_mask(gdf_xs, terrain_source_url) + + with ProcessPoolExecutor() as executor: + future = executor.submit(get_geometry_mask, gdf_xs=gdf_xs, MAP_DEM_UNCLIPPED_SRC_URL=terrain_source_url) + mask = future.result() # clip dem src_dem_clipped_localfile = os.path.join(nwm_rm.terrain_directory, "temp.tif") @@ -89,13 +92,15 @@ def create_ras_terrain( nwm_rm.terrain_directory, map_dem_clipped_basename.replace(".vrt", ".tif") ) - logging.debug(f"Clipping DEM {terrain_source_url} to {src_dem_clipped_localfile}") - clip_raster( - src_path=terrain_source_url, - dst_path=src_dem_clipped_localfile, - mask_polygon=mask, - vertical_units=vertical_units, - ) + with ProcessPoolExecutor() as executor: + future = executor.submit( + clip_raster, + src_path=terrain_source_url, + dst_path=src_dem_clipped_localfile, + mask_polygon=mask, + vertical_units=vertical_units, + ) + future.result() # reproject/resample dem logging.debug(f"Reprojecting/Resampling DEM {src_dem_clipped_localfile} to {src_dem_clipped_localfile}") @@ -106,7 +111,6 @@ def create_ras_terrain( projection_file = write_projection_file(gdf_xs.crs, nwm_rm.terrain_directory) # Make the RAS mapping terrain locally - logging.debug("create_terrain") result = create_terrain( [src_dem_reprojected_localfile], projection_file, @@ -115,4 +119,5 @@ def create_ras_terrain( ) os.remove(src_dem_reprojected_localfile) nwm_rm.update_write_ripple1d_parameters({"source_terrain": terrain_source_url}) + logging.info(f"{task_id}: create_ras_terrain complete") return result From 6ea328c382cf1700aa3412125a5aa1815b62e4c5 Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 13:37:33 -0400 Subject: [PATCH 13/27] add task context to _process for logging --- ripple1d/api/tasks.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ripple1d/api/tasks.py b/ripple1d/api/tasks.py index c6ab4b4e..375062ff 100644 --- a/ripple1d/api/tasks.py +++ b/ripple1d/api/tasks.py @@ -199,12 +199,15 @@ def sleep15(): return ("Slept for 15", 123.456) -# If needing the worker to know about its own task, then use `@huey.task(context=True)`` -# and add `task=None` to the task function definition. -@huey.task() +@huey.task(context=True) @tracerbacker -def _process(func: typing.Callable, kwargs: dict = {}): - """Execute generic huey task that calls the provided func with provided kwargs, asynchronously.""" +def _process(func: typing.Callable, kwargs: dict = {}, task=None): + """Execute generic huey task that calls the provided func with provided kwargs, asynchronously. + + task expected for all funcitons in the ops module to pass through task id for logging + """ + if task: + kwargs["task_id"] = task.id return func(**kwargs) From 0827d41698290ede090abe83678b100970531bc0 Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 13:51:03 -0400 Subject: [PATCH 14/27] add flags to huey consumer --- ripple1d/api/manager.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/ripple1d/api/manager.py b/ripple1d/api/manager.py index 490fc1b5..3d36c866 100644 --- a/ripple1d/api/manager.py +++ b/ripple1d/api/manager.py @@ -52,18 +52,6 @@ def start(self): print("Error: huey consumer script was not discoverable.") exit(1) - # if os.path.exists(self.logs_dir): - # shutil.rmtree(self.logs_dir, onerror=self._handle_remove_readonly) - - # if os.path.exists(self.logs_dir): - # print(f"Error: could not delete {self.logs_dir}") - # exit(1) - - # os.makedirs(self.logs_dir, exist_ok=True) - # if not os.path.exists(self.logs_dir): - # print(f"Error: could not create {self.logs_dir}") - # exit(1) - python_executable = sys.executable print("Starting ripple1d-huey") @@ -74,6 +62,8 @@ def start(self): "ripple1d.api.tasks.huey", "-w", str(self.huey["thread_count"]), + "--flush-locks", + "--no-periodic", ] if self.huey["hide_huey_shell"]: subprocess.Popen(["start", "cmd", "/k", " ".join(huey_command)], shell=False) From 003f7289169df99b05edfe17f9449978caf28c8e Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 16:03:52 -0400 Subject: [PATCH 15/27] add suppression of noisy library logs and add log_process wrapper --- ripple1d/api/log.py | 10 ++++++---- ripple1d/consts.py | 2 ++ ripple1d/ripple1d_logger.py | 18 +++++++++++++++++- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/ripple1d/api/log.py b/ripple1d/api/log.py index 4661eadf..8a98dde3 100644 --- a/ripple1d/api/log.py +++ b/ripple1d/api/log.py @@ -4,15 +4,17 @@ import json import logging import os +import time import traceback from datetime import datetime, timezone +from ripple1d.consts import SUPPRESS_LOGS from ripple1d.ripple1d_logger import RippleLogFormatter LOGS: dict[str, logging.Logger] = {} # global that is modified by initialize_log() -def initialize_log(log_dir: str = "") -> logging.Logger: +def initialize_log(log_dir: str = "", log_level: int = logging.INFO) -> logging.Logger: """Initialize log with JSON-LD style formatting and throttled level for AWS libs. By default sends to StreamHandler (stdout/stderr), but can provide a filename to log to disk instead. @@ -25,11 +27,11 @@ def initialize_log(log_dir: str = "") -> logging.Logger: if filename in LOGS: return LOGS[filename] - logging.getLogger("boto3").setLevel(logging.WARNING) - logging.getLogger("botocore").setLevel(logging.WARNING) + for module in SUPPRESS_LOGS: + logging.getLogger(module).setLevel(logging.ERROR) log = logging.getLogger() - log.setLevel(logging.INFO) + log.setLevel(log_level) formatter = RippleLogFormatter() if filename: diff --git a/ripple1d/consts.py b/ripple1d/consts.py index 6d6a674d..f232a458 100644 --- a/ripple1d/consts.py +++ b/ripple1d/consts.py @@ -2,6 +2,8 @@ from collections import OrderedDict +SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio", "shapely"] + MAP_DEM_UNCLIPPED_SRC_URL = ( "https://rockyweb.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/TIFF/USGS_Seamless_DEM_13.vrt" ) diff --git a/ripple1d/ripple1d_logger.py b/ripple1d/ripple1d_logger.py index 9c161d0b..efc520e3 100644 --- a/ripple1d/ripple1d_logger.py +++ b/ripple1d/ripple1d_logger.py @@ -3,16 +3,32 @@ import inspect import json import logging +import time import traceback from logging.handlers import RotatingFileHandler -SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterios"] +SUPPRESS_LOGS = ["boto3", "botocore", "geopandas", "fiona", "rasterio", "pyogrio"] import inspect import json import logging import traceback +def log_process(func): + """Log time to run function (called by huey task).""" + + def wrapper(*args, **kwargs): + if logging.getLogger().isEnabledFor(logging.INFO): + start = time.time() + result = func(*args, **kwargs) + if logging.getLogger().isEnabledFor(logging.INFO): + elapsed_time = time.time() - start + logging.info(f"{kwargs.get('task_id')} | {func.__name__} | process time {elapsed_time:.2f} seconds") + return result + + return wrapper + + class RippleLogFormatter(logging.Formatter): """Format log messages as JSON-LD.""" From d4009ca77ce528e4b9e691e2f348e6bbfc608dce Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 16:17:26 -0400 Subject: [PATCH 16/27] add task logging --- ripple1d/api/app.py | 1 + ripple1d/ops/fim_lib.py | 15 +++++++++++---- ripple1d/ops/metrics.py | 5 ++++- ripple1d/ops/ras_conflate.py | 14 ++++++++------ ripple1d/ops/ras_run.py | 8 ++++++++ ripple1d/ops/ras_terrain.py | 4 ++-- ripple1d/ops/subset_gpkg.py | 20 +++++++++++++------- ripple1d/ras_to_gpkg.py | 4 +++- 8 files changed, 50 insertions(+), 21 deletions(-) diff --git a/ripple1d/api/app.py b/ripple1d/api/app.py index 7bb6a945..6f688c9d 100644 --- a/ripple1d/api/app.py +++ b/ripple1d/api/app.py @@ -10,6 +10,7 @@ from ripple1d.api import tasks from ripple1d.api.utils import get_unexpected_and_missing_args +from ripple1d.consts import SUPPRESS_LOGS from ripple1d.ops.fim_lib import create_fim_lib, fim_lib_stac, nwm_reach_model_stac from ripple1d.ops.metrics import compute_conflation_metrics from ripple1d.ops.ras_conflate import conflate_model diff --git a/ripple1d/ops/fim_lib.py b/ripple1d/ops/fim_lib.py index dff267e0..1d8d9c8d 100644 --- a/ripple1d/ops/fim_lib.py +++ b/ripple1d/ops/fim_lib.py @@ -137,8 +137,10 @@ def create_fim_lib( overviews: bool = False, resolution: float = 3, resolution_units: str = "Meters", + task_id: str = "", ): """Create a new FIM library for a NWM id.""" + logging.info(f"{task_id} | create_fim_lib starting") nwm_rm = NwmReachModel(submodel_directory) if not nwm_rm.file_exists(nwm_rm.ras_gpkg_file): raise FileNotFoundError(f"cannot find ras_gpkg_file file {nwm_rm.ras_gpkg_file}, please ensure file exists") @@ -186,15 +188,15 @@ def create_fim_lib( if cleanup: shutil.rmtree(os.path.join(rm.ras_project._ras_dir, f"{nwm_rm.model_name}_nd")) + logging.info(f"{task_id} | create_fim_lib complete") return {"fim_results_directory": nwm_rm.fim_results_directory, "fim_results_database": nwm_rm.fim_results_database} def nwm_reach_model_stac( - ras_project_directory: str, - ras_model_s3_prefix: str = None, - bucket: str = None, + ras_project_directory: str, ras_model_s3_prefix: str = None, bucket: str = None, task_id: str = "" ): """Convert a FIM RAS model to a STAC item.""" + logging.info(f"{task_id} | nwm_reach_model_stac starting") nwm_rm = NwmReachModel(ras_project_directory) # create new stac item @@ -229,6 +231,7 @@ def nwm_reach_model_stac( nwm_rm.update_write_ripple1d_parameters({"model_stac_item": f"https://{bucket}.s3.amazonaws.com/{key}"}) else: nwm_rm.update_write_ripple1d_parameters({"model_stac_item": nwm_rm.model_stac_json_file}) + logging.info(f"{task_id} | nwm_reach_model_stac complete") def update_stac_s3_location(stac_item_file: pystac.Item, bucket: str, s3_prefix: str): @@ -312,8 +315,11 @@ def fim_lib_item(item_id: str, assets: list, stac_json: str, metadata: dict) -> return item -def fim_lib_stac(ras_project_directory: str, nwm_reach_id: str, s3_prefix: str = None, bucket: str = None): +def fim_lib_stac( + ras_project_directory: str, nwm_reach_id: str, s3_prefix: str = None, bucket: str = None, task_id: str = "" +): """Create a stac item for a fim library.""" + logging.info(f"{task_id} | fim_lib_stac starting") nwm_rm = NwmReachModel(ras_project_directory) metadata = { @@ -335,3 +341,4 @@ def fim_lib_stac(ras_project_directory: str, nwm_reach_id: str, s3_prefix: str = if s3_prefix and bucket: nwm_rm.upload_fim_lib_assets(s3_prefix, bucket) update_stac_s3_location(nwm_rm.fim_lib_stac_json_file, bucket, s3_prefix) + logging.info(f"{task_id} | fim_lib_stac complete") diff --git a/ripple1d/ops/metrics.py b/ripple1d/ops/metrics.py index 5be5122c..5ad57203 100644 --- a/ripple1d/ops/metrics.py +++ b/ripple1d/ops/metrics.py @@ -197,8 +197,9 @@ def compute_coverage_metrics(self, xs_gdf: gpd.GeoDataFrame) -> dict: logging.error(f"traceback: {traceback.format_exc()}") -def compute_conflation_metrics(source_model_directory: str, source_network: str): +def compute_conflation_metrics(source_model_directory: str, source_network: str, task_id: str = ""): """Compute metrics for a network reach.""" + logging.info(f"{task_id} | compute_conflation_metrics starting") network_pq_path = source_network["file_name"] model_name = os.path.basename(source_model_directory) src_gpkg_path = os.path.join(source_model_directory, f"{model_name}.gpkg") @@ -249,6 +250,8 @@ def compute_conflation_metrics(source_model_directory: str, source_network: str) conflation_parameters["reaches"][network_id].update({"metrics": {}}) with open(conflation_json, "w") as f: f.write(json.dumps(conflation_parameters, indent=4)) + + logging.info(f"{task_id} | compute_conflation_metrics complete") return conflation_parameters diff --git a/ripple1d/ops/ras_conflate.py b/ripple1d/ops/ras_conflate.py index 616e7f7b..b786b0da 100644 --- a/ripple1d/ops/ras_conflate.py +++ b/ripple1d/ops/ras_conflate.py @@ -57,7 +57,7 @@ def conflate_single_nwm_reach(rfc: RasFimConflater, nwm_reach_id: int): raise ValueError(f"nwm_reach_id {nwm_reach_id} not conflating to the ras model geometry.") -def conflate_model(source_model_directory: str, source_network: dict): +def conflate_model(source_model_directory: str, source_network: dict, task_id: str = ""): """Conflate a HEC-RAS model with NWM reaches. source_network example: @@ -67,6 +67,7 @@ def conflate_model(source_model_directory: str, source_network: dict): "type": "nwm_hydrofabric" // required } """ + logging.info(f"{task_id} | conflate_model starting") try: nwm_pq_path = source_network["file_name"] except KeyError: @@ -109,7 +110,7 @@ def conflate_model(source_model_directory: str, source_network: dict): continue logging.info( - f"{river_reach_name} | us_most_reach_id ={us_most_reach_id} and ds_most_reach_id = {ds_most_reach_id}" + f"{task_id} | {river_reach_name} | us_most_reach_id ={us_most_reach_id} and ds_most_reach_id = {ds_most_reach_id}" ) # walk network to get the potential reach ids @@ -122,7 +123,7 @@ def conflate_model(source_model_directory: str, source_network: dict): metadata["reaches"].update(ras_reaches_metadata(rfc, candidate_reaches)) if not conflated(metadata): - return "no reaches conflated" + return f"{task_id} | no reaches conflated" ids = list(metadata["reaches"].keys()) fim_stream = rfc.local_nwm_reaches()[rfc.local_nwm_reaches()["ID"].isin(ids)] @@ -135,7 +136,7 @@ def conflate_model(source_model_directory: str, source_network: dict): limit_plot_to_nearby_reaches=True, ) - logging.info(f"Conflation results: {metadata}") + logging.debug(f"{task_id} | Conflation results: {metadata}") conflation_file = f"{rfc.ras_gpkg.replace('.gpkg','.conflation.json')}" metadata["metadata"] = {} @@ -162,9 +163,10 @@ def conflate_model(source_model_directory: str, source_network: dict): try: compute_conflation_metrics(source_model_directory, source_network) except Exception as e: - logging.error(f"Error: {e}") - logging.error(f"traceback: {traceback.format_exc()}") + logging.error(f"{task_id} | Error: {e}") + logging.error(f"{task_id} | traceback: {traceback.format_exc()}") + logging.info(f"{task_id} | conflate_model complete") return conflation_file diff --git a/ripple1d/ops/ras_run.py b/ripple1d/ops/ras_run.py index 7532bbd4..6fcda86a 100644 --- a/ripple1d/ops/ras_run.py +++ b/ripple1d/ops/ras_run.py @@ -21,8 +21,10 @@ def create_model_run_normal_depth( num_of_discharges_for_initial_normal_depth_runs: int = 10, ras_version: str = "631", show_ras: bool = False, + task_id: str = "", ): """Write and compute initial normal depth runs to develop initial rating curves.""" + logging.info(f"{task_id} | create_model_run_normal_depth starting") nwm_rm = NwmReachModel(submodel_directory) if not nwm_rm.file_exists(nwm_rm.conflation_file): @@ -63,6 +65,8 @@ def create_model_run_normal_depth( show_ras=show_ras, run_ras=True, ) + + logging.info(f"{task_id} | create_model_run_normal_depth complete") return {f"{nwm_rm.model_name}_{plan_suffix}": asdict(fcl)} @@ -73,8 +77,10 @@ def run_incremental_normal_depth( depth_increment=0.5, write_depth_grids: str = True, show_ras: bool = False, + task_id: str = "", ): """Write and compute incremental normal depth runs to develop rating curves and depth grids.""" + logging.info(f"{task_id} | run_incremental_normal_depth starting") nwm_rm = NwmReachModel(submodel_directory) if not nwm_rm.file_exists(nwm_rm.conflation_file): @@ -120,6 +126,7 @@ def run_incremental_normal_depth( show_ras=show_ras, run_ras=True, ) + logging.info(f"{task_id} | run_incremental_normal_depth complete") return {f"{nwm_rm.model_name}_{plan_suffix}": asdict(fcl)} @@ -132,6 +139,7 @@ def run_known_wse( ras_version: str = "631", write_depth_grids: str = True, show_ras: bool = False, + task_id: str = "", ): """Write and compute known water surface elevation runs to develop rating curves and depth grids.""" nwm_rm = NwmReachModel(submodel_directory) diff --git a/ripple1d/ops/ras_terrain.py b/ripple1d/ops/ras_terrain.py index 8c02793f..95a133b0 100644 --- a/ripple1d/ops/ras_terrain.py +++ b/ripple1d/ops/ras_terrain.py @@ -57,7 +57,7 @@ def create_ras_terrain( task_id: str = "", ) -> None: """Create a RAS terrain file.""" - logging.info(f"{task_id}: create_ras_terrain start") + logging.info(f"{task_id} | create_ras_terrain starting") if resolution and not resolution_units: raise ValueError( @@ -119,5 +119,5 @@ def create_ras_terrain( ) os.remove(src_dem_reprojected_localfile) nwm_rm.update_write_ripple1d_parameters({"source_terrain": terrain_source_url}) - logging.info(f"{task_id}: create_ras_terrain complete") + logging.info(f"{task_id} | create_ras_terrain complete") return result diff --git a/ripple1d/ops/subset_gpkg.py b/ripple1d/ops/subset_gpkg.py index 655ac98b..3fc6d527 100644 --- a/ripple1d/ops/subset_gpkg.py +++ b/ripple1d/ops/subset_gpkg.py @@ -12,6 +12,7 @@ import ripple1d from ripple1d.consts import METERS_PER_FOOT from ripple1d.data_model import NwmReachModel, RippleSourceDirectory, RippleSourceModel +from ripple1d.ripple1d_logger import log_process from ripple1d.utils.ripple_utils import xs_concave_hull @@ -143,9 +144,8 @@ def nwm_reach_model(self) -> NwmReachModel: """Return the new NWM reach model object.""" return NwmReachModel(self.dst_project_dir) - def write_ripple_gpkg( - self, - ) -> None: + @log_process + def write_ripple_gpkg(self, task_id: str = None) -> None: """Write the subsetted geopackage to the destination project directory.""" os.makedirs(self.dst_project_dir, exist_ok=True) @@ -467,16 +467,21 @@ def write_ripple1d_parameters(self, ripple1d_parameters: dict): json.dump(ripple1d_parameters, f, indent=4) -def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_id: int): +def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_id: int, task_id: str = ""): """Use ripple conflation data to create a new GPKG from an existing ras geopackage.""" + logging.info(f"{task_id} | extract_submodel starting") rsd = RippleSourceDirectory(source_model_directory) - logging.info(f"Preparing to extract NWM ID {nwm_id} from {rsd.ras_project_file}") + logging.debug(f"{task_id} | preparing to extract NWM ID {nwm_id} from {rsd.ras_project_file}") if not rsd.file_exists(rsd.ras_gpkg_file): - raise FileNotFoundError(f"cannot find file ras-geometry file {rsd.ras_gpkg_file}, please ensure file exists") + raise FileNotFoundError( + f"{task_id} | cannot find file ras-geometry file {rsd.ras_gpkg_file}, please ensure file exists" + ) if not rsd.file_exists(rsd.conflation_file): - raise FileNotFoundError(f"cannot find conflation file {rsd.conflation_file}, please ensure file exists") + raise FileNotFoundError( + f"{task_id} | cannot find conflation file {rsd.conflation_file}, please ensure file exists" + ) ripple1d_parameters = rsd.nwm_conflation_parameters(str(nwm_id)) if ripple1d_parameters["us_xs"]["xs_id"] == "-9999": @@ -489,4 +494,5 @@ def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_i ripple1d_parameters = rgs.update_ripple1d_parameters(rsd) rgs.write_ripple1d_parameters(ripple1d_parameters) + logging.info(f"{task_id} | extract_submodel complete") return ripple1d_parameters diff --git a/ripple1d/ras_to_gpkg.py b/ripple1d/ras_to_gpkg.py index 4ef3475c..7ee5798f 100644 --- a/ripple1d/ras_to_gpkg.py +++ b/ripple1d/ras_to_gpkg.py @@ -272,8 +272,9 @@ def detemine_primary_plan( return candidate_plans[0] -def gpkg_from_ras(source_model_directory: str, crs: str, metadata: dict): +def gpkg_from_ras(source_model_directory: str, crs: str, metadata: dict, task_id: str = ""): """Write geometry and flow data to a geopackage locally.""" + logging.info(f"{task_id} | gpkg_from_ras starting") prjs = glob.glob(f"{source_model_directory}/*.prj") ras_text_file_path = None @@ -287,6 +288,7 @@ def gpkg_from_ras(source_model_directory: str, crs: str, metadata: dict): output_gpkg_path = ras_text_file_path.replace(".prj", ".gpkg") rp = RasProject(ras_text_file_path) + logging.info(f"{task_id} | gpkg_from_ras complete") return geom_flow_to_gpkg(rp, crs, output_gpkg_path, metadata) From f00ca0c10383654ad45d0be0a9bf76cb1145e124 Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 16:42:24 -0400 Subject: [PATCH 17/27] add log to kwses_run --- ripple1d/ops/ras_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ripple1d/ops/ras_run.py b/ripple1d/ops/ras_run.py index 05af9605..84b02b0f 100644 --- a/ripple1d/ops/ras_run.py +++ b/ripple1d/ops/ras_run.py @@ -139,6 +139,7 @@ def run_known_wse( task_id: str = "", ): """Write and compute known water surface elevation runs to develop rating curves and depth grids.""" + logging.info(f"{task_id} | run_known_wse starting") nwm_rm = NwmReachModel(submodel_directory) if not nwm_rm.file_exists(nwm_rm.conflation_file): @@ -194,6 +195,7 @@ def run_known_wse( show_ras=show_ras, run_ras=True, ) + logging.info(f"{task_id} | run_known_wse complete") return {f"{nwm_rm.model_name}_{plan_suffix}": {"kwse": known_water_surface_elevations.tolist()}} From eeee22868470d97753ad384d27359769f03b8d87 Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 17:08:49 -0500 Subject: [PATCH 18/27] drop driver arg to prevent warnings --- ripple1d/ops/ras_terrain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ripple1d/ops/ras_terrain.py b/ripple1d/ops/ras_terrain.py index 28b637ae..699ed0fc 100644 --- a/ripple1d/ops/ras_terrain.py +++ b/ripple1d/ops/ras_terrain.py @@ -77,7 +77,7 @@ def create_ras_terrain( if not os.path.exists(nwm_rm.terrain_directory): os.makedirs(nwm_rm.terrain_directory, exist_ok=True) - gdf_xs = gpd.read_file(nwm_rm.ras_gpkg_file, layer="XS", driver="GPKG").explode(ignore_index=True) + gdf_xs = gpd.read_file(nwm_rm.ras_gpkg_file, layer="XS").explode(ignore_index=True) crs = gdf_xs.crs with ProcessPoolExecutor() as executor: From c63110453d3d9e726c9394fd3ec629fb80fb2e2f Mon Sep 17 00:00:00 2001 From: Matt Deshotel Date: Thu, 19 Sep 2024 17:09:21 -0500 Subject: [PATCH 19/27] update tests --- tests/api_tests.py | 2 +- tests/conftest.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/api_tests.py b/tests/api_tests.py index a515c0e6..4b1b7006 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -166,7 +166,7 @@ def test_g_create_fim_lib(self): "submodel_directory": self.SUBMODELS_DIRECTORY, "plans": ["nd", "kwse"], "library_directory": self.FIM_LIB_DIRECTORY, - "cleanup": True, + "cleanup": False, } process = "create_fim_lib" files = [self.FIM_LIB_DB, self.DEPTH_GRIDS_ND, self.DEPTH_GRIDS_KWSE] diff --git a/tests/conftest.py b/tests/conftest.py index 5f820aea..32ab9c4a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,15 +26,14 @@ def setup_data(request): SOURCE_RAS_MODEL_DIRECTORY = os.path.join(TEST_DIR, f"ras-data\\{RAS_MODEL}") SUBMODELS_BASE_DIRECTORY = os.path.join(SOURCE_RAS_MODEL_DIRECTORY, "submodels") SUBMODELS_DIRECTORY = os.path.join(SUBMODELS_BASE_DIRECTORY, REACH_ID) - FIM_LIB_DIRECTORY = os.path.join(SUBMODELS_DIRECTORY, "fim") + FIM_LIB_DIRECTORY = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}\\fims\\{REACH_ID}") + request.cls.FIM_LIB_DIRECTORY = FIM_LIB_DIRECTORY request.cls.REACH_ID = RAS_MODEL request.cls.REACH_ID = REACH_ID - request.cls.SOURCE_NETWORK = SOURCE_NETWORK request.cls.SOURCE_RAS_MODEL_DIRECTORY = SOURCE_RAS_MODEL_DIRECTORY request.cls.SUBMODELS_BASE_DIRECTORY = SUBMODELS_BASE_DIRECTORY request.cls.SUBMODELS_DIRECTORY = SUBMODELS_DIRECTORY - request.cls.FIM_LIB_DIRECTORY = FIM_LIB_DIRECTORY request.cls.GPKG_FILE = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}.gpkg") request.cls.SOURCE_GPKG_FILE = os.path.join(SOURCE_RAS_MODEL_DIRECTORY, f"{RAS_MODEL}.gpkg") request.cls.TERRAIN_HDF = os.path.join(SUBMODELS_DIRECTORY, f"Terrain\\{REACH_ID}.hdf") @@ -50,6 +49,7 @@ def setup_data(request): request.cls.PLAN3_FILE = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}.p03") request.cls.FLOW3_FILE = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}.f03") request.cls.RESULT3_FILE = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}.r03") + request.cls.FIM_LIB_DB = os.path.join(FIM_LIB_DIRECTORY, f"{REACH_ID}\\{REACH_ID}.db") request.cls.DEPTH_GRIDS_ND = os.path.join(FIM_LIB_DIRECTORY, f"{REACH_ID}\\z_nd") integer, decimal = str(np.floor((MIN_ELEVATION + 41) * 2) / 2).split(".") @@ -59,4 +59,3 @@ def setup_data(request): request.cls.min_elevation = MIN_ELEVATION request.cls.conflation_file = os.path.join(SOURCE_RAS_MODEL_DIRECTORY, f"{RAS_MODEL}.conflation.json") request.cls.crs = CRS[RAS_MODEL] - request.cls.FIM_LIB_DIRECTORY = os.path.join(SUBMODELS_DIRECTORY, f"{REACH_ID}\\fims") From 33c778781772bf7faccdc3bfc7cc698e279d237a Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 18:13:27 -0400 Subject: [PATCH 20/27] fix merge conflict --- ripple1d/ops/ras_terrain.py | 4 +++- ripple1d/ops/subset_gpkg.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ripple1d/ops/ras_terrain.py b/ripple1d/ops/ras_terrain.py index 28b637ae..70e1cc76 100644 --- a/ripple1d/ops/ras_terrain.py +++ b/ripple1d/ops/ras_terrain.py @@ -20,11 +20,13 @@ ) from ripple1d.data_model import NwmReachModel from ripple1d.ras import create_terrain + +# from ripple1d.ripple1d_logger import log_process from ripple1d.utils.dg_utils import clip_raster, reproject_raster from ripple1d.utils.ripple_utils import fix_reversed_xs, xs_concave_hull -def get_geometry_mask(gdf_xs_conc_hull: str, MAP_DEM_UNCLIPPED_SRC_URL: str) -> gpd.GeoDataFrame: +def get_geometry_mask(gdf_xs_conc_hull: str, MAP_DEM_UNCLIPPED_SRC_URL: str, task_id: str = None) -> gpd.GeoDataFrame: """Get a geometry mask for the DEM based on the cross sections.""" # build a DEM mask polygon based on the XS extents diff --git a/ripple1d/ops/subset_gpkg.py b/ripple1d/ops/subset_gpkg.py index 4cea08a1..443958a7 100644 --- a/ripple1d/ops/subset_gpkg.py +++ b/ripple1d/ops/subset_gpkg.py @@ -512,7 +512,7 @@ def extract_submodel(source_model_directory: str, submodel_directory: str, nwm_i else: rgs = RippleGeopackageSubsetter(rsd.ras_gpkg_file, rsd.conflation_file, submodel_directory, nwm_id) - rgs.write_ripple_gpkg() + rgs.write_ripple_gpkg(task_id=task_id) ripple1d_parameters = rgs.update_ripple1d_parameters(rsd) rgs.write_ripple1d_parameters(ripple1d_parameters) From fceae8dd69651122938cabd3a22e1b341556ad0b Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 18:49:08 -0400 Subject: [PATCH 21/27] turn off png for conflation --- ripple1d/ops/ras_conflate.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ripple1d/ops/ras_conflate.py b/ripple1d/ops/ras_conflate.py index b786b0da..39e5c763 100644 --- a/ripple1d/ops/ras_conflate.py +++ b/ripple1d/ops/ras_conflate.py @@ -127,14 +127,14 @@ def conflate_model(source_model_directory: str, source_network: dict, task_id: s ids = list(metadata["reaches"].keys()) fim_stream = rfc.local_nwm_reaches()[rfc.local_nwm_reaches()["ID"].isin(ids)] - conflation_png = f"{rfc.ras_gpkg.replace('.gpkg','.conflation.png')}" + # conflation_png = f"{rfc.ras_gpkg.replace('.gpkg','.conflation.png')}" - plot_conflation_results( - rfc, - fim_stream, - conflation_png, - limit_plot_to_nearby_reaches=True, - ) + # plot_conflation_results( + # rfc, + # fim_stream, + # conflation_png, + # limit_plot_to_nearby_reaches=True, + # ) logging.debug(f"{task_id} | Conflation results: {metadata}") conflation_file = f"{rfc.ras_gpkg.replace('.gpkg','.conflation.json')}" @@ -142,7 +142,7 @@ def conflate_model(source_model_directory: str, source_network: dict, task_id: s metadata["metadata"] = {} metadata["metadata"]["source_network"] = source_network.copy() metadata["metadata"]["source_network"]["file_name"] = os.path.basename(nwm_pq_path) - metadata["metadata"]["conflation_png"] = os.path.basename(conflation_png) + # metadata["metadata"]["conflation_png"] = os.path.basename(conflation_png) metadata["metadata"]["conflation_ripple1d_version"] = ripple1d.__version__ metadata["metadata"]["metrics_ripple1d_version"] = ripple1d.__version__ metadata["metadata"]["source_ras_model"] = { From dc3b5ab11f381f202bd99de91005fe21c924f499 Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 20:27:06 -0400 Subject: [PATCH 22/27] remove uneeded logs --- ripple1d/conflate/rasfim.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ripple1d/conflate/rasfim.py b/ripple1d/conflate/rasfim.py index 2644241c..de857b4d 100644 --- a/ripple1d/conflate/rasfim.py +++ b/ripple1d/conflate/rasfim.py @@ -313,7 +313,7 @@ def ras_xs(self) -> gpd.GeoDataFrame: @ensure_data_loaded def ras_structures(self) -> gpd.GeoDataFrame: """RAS structures.""" - logging.info("RAS structures") + # logging.info("RAS structures") try: return self._ras_structures.to_crs(self.common_crs) except AttributeError: @@ -693,8 +693,6 @@ def map_reach_xs(rfc: RasFimConflater, reach: MultiLineString) -> dict: return {"eclipsed": True} not_reversed_xs = check_xs_direction(intersected_xs, reach.geometry) - logging.info(len(intersected_xs)) - logging.info(len(not_reversed_xs)) intersected_xs["geometry"] = intersected_xs.apply( lambda row: ( row.geometry if row["river_reach_rs"] in list(not_reversed_xs["river_reach_rs"]) else reverse(row.geometry) From 16d13c6977870d14e8916fc6390ca95eb4729f8e Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Thu, 19 Sep 2024 20:31:32 -0400 Subject: [PATCH 23/27] remove legacy production dir --- production/README.md | 52 ------ production/collection_summary.py | 60 ------- production/db_utils.py | 111 ------------ production/headers.py | 38 ----- production/mip_catalog.py | 161 ------------------ production/step_10_nwm_reach_model_stac.py | 16 -- production/step_11_fim_lib_stac.py | 15 -- production/step_1_extract_geometry.py | 86 ---------- production/step_2_create_stac.py | 65 ------- production/step_3_5_metrics.py | 10 -- production/step_3_conflate_model.py | 95 ----------- production/step_4_subset_gpkg.py | 21 --- production/step_5_create_terrain.py | 14 -- production/step_6_initial_normal_depth.py | 17 -- production/step_7_incremental_normal_depth.py | 15 -- .../step_8_known_water_surface_elevation.py | 30 ---- production/step_9_create_fim_lib.py | 18 -- 17 files changed, 824 deletions(-) delete mode 100644 production/README.md delete mode 100644 production/collection_summary.py delete mode 100644 production/db_utils.py delete mode 100644 production/headers.py delete mode 100644 production/mip_catalog.py delete mode 100644 production/step_10_nwm_reach_model_stac.py delete mode 100644 production/step_11_fim_lib_stac.py delete mode 100644 production/step_1_extract_geometry.py delete mode 100644 production/step_2_create_stac.py delete mode 100644 production/step_3_5_metrics.py delete mode 100644 production/step_3_conflate_model.py delete mode 100644 production/step_4_subset_gpkg.py delete mode 100644 production/step_5_create_terrain.py delete mode 100644 production/step_6_initial_normal_depth.py delete mode 100644 production/step_7_incremental_normal_depth.py delete mode 100644 production/step_8_known_water_surface_elevation.py delete mode 100644 production/step_9_create_fim_lib.py diff --git a/production/README.md b/production/README.md deleted file mode 100644 index 25e8215d..00000000 --- a/production/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Production Scripts -The scripts contained in this `production` directory are example scripts that run all of the processes necessary to leverage FEMA MIP models to develop FIM libraries using ripple1d. Each script/process is outlined below. - -### [step_1_extract_geometry.py](step_1_extract_geometry.py) ---- -Extract the geometry of an existing HEC-RAS model and create a geopackage. Currently, this script reads from an s3 location but ripple1d does have built in capacity to read from local. - -The discharges applied to the existing model are also extracted. - -*NOTE*: Structures are not currently supported. - -### [step_2_create_stac.py](step_2_create_stac.py) ---- -Create a Spatial-Temporal Asset Catalog (STAC) item for an existing HEC-RAS model. Use the geopackage from step 1 to define the spatial location of the model. - -### [step_3_conflate_model.py](step_3_conflate_model.py) ---- -Conflate an existing HEC-RAS model with the NWM reaches. Outputs a json of conflation parameters. - -### [step_4_subset_gpkg.py](step_4_subset_gpkg.py) ---- -Selects XS's and river reaches from an existing HEC-RAS model using the conflation output to create a geopackage constrained to a given reach of the NWM hydrofabric. Multiple new geopackages are typically produced; each one corresponding to a single NWM reach. - -### [step_5_create_terrain.py](step_5_create_terrain.py) ---- -Create a HEC-RAS terrain for each NWM-based geopackage. The concave hull of the geopackage's cross section layer is buffered and then used to clip a source DEM. - -### [step_6_initial_normal_depth.py](step_6_initial_normal_depth.py) ---- -Write a HEC-RAS model for each NWM-based geopackage, HEC-RAS terrain, and conflation parameters. Create a flow and plan for an initial normal depth run. The discharges applied are derived from the min/max flow from the NWM for this reach and the min/max flows pulled from the existing HEC-RAS model. 10 discharges are incremented evenly between the smallest and largest flows from the two aforementioned sources. - -This run is computed to develop a rating curve which will inform incremental discharges that produce depth increments provided by the user in the next step. - -Compute the initial normal depth plan. - -### [step_7_incremental_normal_depth.py](step_7_incremental_normal_depth.py) ---- -Based on rating curve derived from the initial-normal-depth-run and a target depth increment provided by the user, create a second normal depth run where the discharges are read from the rating curve at the specified depth increments. - -Compute the incremental normal depth plan. - -### [step_8_known_water_surface_elevation.py](step_8_known_water_surface_elevation.py) ---- -Create a new flow/plan for a known water surface elevation (kwse) run. This run uses the range of flows applied for the incremental-normal-depth-run and a min elevation, max elevation, and depth increment provded by the user to simulated all flow-kwse scenarios. The provided min/max elevations should represent the min/max water surface elevation expected at the downstream end of the NWM reach. - -For each flow-kwse combination, the kwse is compared to the water surface elevation resulting from the normal depth run whose flow is the same as the current flow. If the kwse is lower than the water surface elevation from the normal depth run, then the kwse will not control downstream portion of the model and thus the flow-kwse combination is removed from the list of necessary simulations. - -### [step_9_create_fim_lib.py](step_9_create_fim_lib.py) ---- -Create a library of depth grids resulting from the known-water-surface-elevation-runs and the incremental-normal-depth-runs. - -A database is also produced which contains the flows and kwse that were applied as well as the computed water surface elevations and depths at the upstrema end of NWM reach. diff --git a/production/collection_summary.py b/production/collection_summary.py deleted file mode 100644 index e386b34e..00000000 --- a/production/collection_summary.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Summarize collection.""" - -import logging - -import pystac_client - -from production.headers import get_auth_header -from ripple1d.ripple1d_logger import configure_logging -from ripple1d.utils.stac_utils import ( - upsert_collection, -) - -STAC_ENDPOINT = "https://stac2.dewberryanalytics.com" - -client = pystac_client.Client.open(STAC_ENDPOINT) -collections = client.get_collections() - - -def main(): - """Update STAC collections to include item summaries.""" - for collection in client.get_collections(): - logging.info(f"Adding summary to {collection.id}") - version_summary = {} - coverage_summary = { - "1D_HEC-RAS_models": 0, - "1D_HEC-RAS_river_miles": 0, - "2D_HEC-RAS_models": 0, - } - - for item in collection.get_all_items(): - river_miles = float(item.properties["river miles"]) - - coverage_summary["1D_HEC-RAS_river_miles"] += river_miles - coverage_summary["1D_HEC-RAS_models"] += 1 - - case_no = item.properties["MIP:case_ID"] - ras_version = item.properties["ras version"] - - if ras_version not in version_summary: - version_summary[ras_version] = {case_no: {"river_miles": 0, "ras_models": 0}} - - if case_no not in version_summary[ras_version]: - version_summary[ras_version][case_no] = {"river_miles": 0, "ras_models": 0} - - version_summary[ras_version][case_no]["river_miles"] += river_miles - version_summary[ras_version][case_no]["ras_models"] += 1 - - coverage_summary["1D_HEC-RAS_river_miles"] = int(coverage_summary["1D_HEC-RAS_river_miles"]) - - # collection.summaries.remove("ras_version_summary_by_MIP_case_ID") - # collection.summaries.remove("ras_version_summary") - collection.summaries.add("coverage", coverage_summary) - collection.summaries.add("ras_version_summary_with_MIP_case_IDs", version_summary) - header = get_auth_header() - upsert_collection(STAC_ENDPOINT, collection, header) - - -if __name__ == "__main__": - configure_logging(level=logging.INFO, logfile="add_collectiuon_summary.log") - main() diff --git a/production/db_utils.py b/production/db_utils.py deleted file mode 100644 index e21403b6..00000000 --- a/production/db_utils.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Database utils.""" - -import os - -import psycopg2 -from dotenv import load_dotenv -from psycopg2 import sql - -load_dotenv() - - -class PGFim: - """Class to interact with the FIM database.""" - - def __init__(self): - self.dbuser = os.getenv("DBUSER") - self.dbpass = os.getenv("DBPASS") - self.dbhost = os.getenv("DBHOST") - self.dbport = os.getenv("DBPORT") - self.dbname = os.getenv("DBNAME") - - def __conn_string(self): - conn_string = f"dbname='{self.dbname}' user='{self.dbuser}' password='{self.dbpass}' host='{self.dbhost}' port='{self.dbport}'" - return conn_string - - def read_cases(self, table: str, fields: list[str], mip_group: str, optional_condition: str = ""): - """Read cases from the cases schema.""" - approved_conditons = [ - "AND gpkg_complete=false AND gpkg_exc='expected 1 result, no results found'", - "AND gpkg_complete=false", - "AND gpkg_complete IS NULL", - "AND stac_complete=true AND conflation_complete=true", - "AND gpkg_complete=true AND stac_complete=false", - "AND gpkg_complete=true AND stac_complete IS NULL", - "AND stac_complete=true AND conflation_complete IS NULL", - "AND stac_complete=true AND conflation_complete = false", - "AND stac_complete=true AND (conflation_complete=false or conflation_complete is NULL)", - "", - ] - if optional_condition not in approved_conditons: - raise ValueError(f"optional_condition must be one of {approved_conditons} or None") - - with psycopg2.connect(self.__conn_string()) as connection: - cursor = connection.cursor() - fields_str = "" - for field in fields: - fields_str += f"{field}, " - sql_query = sql.SQL( - f"SELECT {fields_str.rstrip(', ')} FROM cases.{table} WHERE mip_group='{mip_group}' {optional_condition};" - ) - - cursor.execute(sql_query) - return cursor.fetchall() - - def update_case_status( - self, - table_name: str, - mip_group: str, - mip_case: str, - key: str, - status: bool, - exc: str, - traceback: str, - process: str, - ): - """Update the status of a table in the cases schema.""" - with psycopg2.connect(self.__conn_string()) as connection: - cursor = connection.cursor() - insert_query = sql.SQL( - f""" - INSERT INTO cases.{table_name}(s3_key,mip_group, case_id, {process}_complete, {process}_exc, {process}_traceback) - VALUES (%s, %s, %s, %s, %s, %s) - ON CONFLICT (s3_key) - DO UPDATE SET - case_id = EXCLUDED.case_id, - {process}_complete = EXCLUDED.{process}_complete, - {process}_exc = EXCLUDED.{process}_exc, - {process}_traceback = EXCLUDED.{process}_traceback; - """ - ) - cursor.execute(insert_query, (key, mip_group, mip_case, status, exc, traceback)) - connection.commit() - - def create_table(self, table_name: str): - """Create a table in the cases schema.""" - with psycopg2.connect(self.__conn_string()) as connection: - cursor = connection.cursor() - sql_query = sql.SQL( - f""" - CREATE TABLE cases.{table_name}( - mip_group TEXT, - s3_key TEXT, - crs TEXT, - ratio_of_best_crs TEXT); - """ - ) - cursor.execute(f"DROP TABLE IF EXISTS cases.{table_name}") - cursor.execute(sql_query) - connection.commit() - - def populate_crs_table(self, table_name: str, mip_group: str, key: str, crs: str, ratio_of_best_crs: str): - """Populate the crs table in the cases schema.""" - with psycopg2.connect(self.__conn_string()) as connection: - cursor = connection.cursor() - cursor.execute( - f""" - INSERT INTO cases.{table_name} (mip_group, s3_key, crs, ratio_of_best_crs) VALUES (%s, %s, %s, %s); - """, - (mip_group, key, crs, ratio_of_best_crs), - ) - connection.commit() diff --git a/production/headers.py b/production/headers.py deleted file mode 100644 index f01b21a6..00000000 --- a/production/headers.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Manage Headers.""" - -import json -import os - -import requests - - -def get_auth_header(): - """Get auth header for a given user.""" - auth_server = os.getenv("AUTH_ISSUER") - client_id = os.getenv("AUTH_ID") - client_secret = os.getenv("AUTH_SECRET") - - username = os.getenv("AUTH_USER") - password = os.getenv("AUTH_USER_PASSWORD") - - auth_payload = f"username={username}&password={password}&client_id={client_id}&grant_type=password&client_secret={client_secret}" - headers = { - "Content-Type": "application/x-www-form-urlencoded", - "Authorization": "Bearer null", - } - - auth_response = requests.request("POST", auth_server, headers=headers, data=auth_payload) - - try: - token = json.loads(auth_response.text)["access_token"] - except KeyError: - logging.debug(auth_response.text) - raise KeyError - - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - "X-ProcessAPI-User-Email": username, - } - - return headers diff --git a/production/mip_catalog.py b/production/mip_catalog.py deleted file mode 100644 index a1f916e6..00000000 --- a/production/mip_catalog.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Build STAC Catalog for MIP HEC-RAS models.""" - -# mip_catalog.py -import logging -from typing import List - -import pystac -import pystac_client -from dotenv import load_dotenv - -from production.db_utils import PGFim -from production.headers import get_auth_header -from ripple1d.ripple1d_logger import configure_logging -from ripple1d.utils.stac_utils import ( - collection_exists, - create_collection, - delete_collection, - key_to_uri, - upsert_collection, - upsert_item, -) - -load_dotenv() - -STAC_ENDPOINT = "https://stac2.dewberryanalytics.com" - - -GROUP_1a = { - "pg_group": "a", - "sql_condition": "AND stac_complete=true AND conflation_complete=true", - "collection_id": "owp_30_pct_mip_cv1_g1", - "collection_title": "OWP 30% Group 1a Conflated FEMA MIP Models", - "description": "HEC-RAS models collected from FEMA's Mapping Information Platform. Group 1a collection contains \ - models within the OWP 30% coverage area that have \ - been auto-georeferenced with modest confidence and \ - version 1.alpha conflation process was successful.", -} - -GROUP_1b = { - "pg_group": "a", - "sql_condition": "AND stac_complete=true AND (conflation_complete=false or conflation_complete is null)", - "collection_id": "owp_30_pct_mip_noc_g1", - "collection_title": "OWP 30% Group 1b FEMA MIP Models (pending conflation)", - "description": "HEC-RAS models collected from FEMA's Mapping Information Platform. Group 1b collection contains \ - models within the OWP 30% coverage area that have \ - been auto-georeferenced with modest confidence but \ - version 1.alpha conflation process failed.", -} - - -GROUP_2a = { - "pg_group": "b", - "sql_condition": "AND stac_complete=true AND conflation_complete=true", - "collection_id": "owp_30_pct_mip_cv1_g2", - "collection_title": "OWP 30% Group 2a Conflated FEMA MIP Models", - "description": "HEC-RAS models collected from FEMA's Mapping Information Platform. Group 2a collection contains \ - models within the OWP 30% coverage area that have \ - been auto-georeferenced with low confidence and \ - version 1.alpha conflation process was successful.", -} - -GROUP_2b = { - "pg_group": "b", - "sql_condition": "AND stac_complete=true AND (conflation_complete=false or conflation_complete is null)", - "collection_id": "owp_30_pct_mip_noc_g2", - "collection_title": "OWP 30% Group 2b FEMA MIP Models (pending conflation)", - "description": "HEC-RAS models collected from FEMA's Mapping Information Platform. Group 2b collection contains \ - models within the OWP 30% coverage area that have \ - been auto-georeferenced with low confidence but \ - version 1.alpha conflation process failed.", -} - - -GROUP_3 = { - "pg_group": "tx_ble", - "sql_condition": "AND stac_complete=true AND conflation_complete=true", - "collection_id": "ripple1d_test_data", - "collection_title": "Test collection for Ripple using Texas BLE data.", - "description": "Test collection for Ripple using Texas BLE data accessed via https://webapps.usgs.gov/infrm/estbfe/", -} - - -def list_items_on_s3(table_name: str, mip_group: str, bucket: str = "fim", condition: str = None) -> List[pystac.Item]: - """Read from database a list of geopackages to create stac items.""" - db = PGFim() - - data = db.read_cases( - table_name, - [ - "case_id", - "s3_key", - ], - mip_group, - condition, - ) - - stac_items = [] - logging.info(f"Found {len(data)} items for group {mip_group}") - for i, (mip_case, s3_ras_project_key) in enumerate(data): - stac_item_s3_key = s3_ras_project_key.replace("mip", "stac").replace(".prj", ".json") - - try: - logging.info(f"loading from case: {mip_case} | key found: {stac_item_s3_key}") - item = pystac.Item.from_file(key_to_uri(stac_item_s3_key, bucket=bucket)) - stac_items.append(item) - except Exception: - logging.error(f"Error reading item: {stac_item_s3_key}") - continue - - return stac_items - - -if __name__ == "__main__": - configure_logging(level=logging.INFO, logfile="create_stac.log") - table_name = "processing" - bucket = "fim" - - for group in [GROUP_1a, GROUP_1b, GROUP_2a, GROUP_2b, GROUP_3]: - mip_group = group["pg_group"] - collection_id = group["collection_id"] - collection_title = group["collection_title"] - - header = get_auth_header() - - # TODO: Be careful temporarily deteleting collections - r = collection_exists(STAC_ENDPOINT, collection_id) - if r.ok: - delete_collection(STAC_ENDPOINT, collection_id, header) - - stac_items = list_items_on_s3(table_name, mip_group, bucket, condition=group["sql_condition"]) - - if len(stac_items) == 0: - continue - - else: - first_items = stac_items[0:2] - collection = create_collection( - first_items, - collection_id, - group["description"], - collection_title, - ) - logging.info(f"Created collection {collection.id}") - upsert_collection(STAC_ENDPOINT, collection, header) - logging.info("Added collection to catalog") - - for item in stac_items: - try: - item.collection_id = collection_id - upsert_item(STAC_ENDPOINT, collection_id, item, header) - logging.info(f"upserting item: {item.id}") - - except Exception as e: - logging.error(f"Error upserting item: {e}") - - client = pystac_client.Client.open(STAC_ENDPOINT) - collection = client.get_collection(collection_id) - collection.update_extent_from_items() - header = get_auth_header() - upsert_collection(STAC_ENDPOINT, collection, header) - logging.info("Updated collection extent from items") diff --git a/production/step_10_nwm_reach_model_stac.py b/production/step_10_nwm_reach_model_stac.py deleted file mode 100644 index d0403229..00000000 --- a/production/step_10_nwm_reach_model_stac.py +++ /dev/null @@ -1,16 +0,0 @@ -import os - -from ripple1d.ops.fim_lib import nwm_reach_model_stac -from ripple1d.ripple1d_logger import configure_logging -import logging - -if __name__ == "__main__": - configure_logging(level=logging.INFO) - reach_id = "2823932" - - ras_project_directory = os.path.dirname(__file__).replace( - "production", f"tests\\ras-data\\Baxter\\submodels\\{reach_id}" - ) - ras_model_s3_prefix = f"stac/test-data/nwm_reach_models/{reach_id}" - bucket = "fim" - nwm_reach_model_stac(ras_project_directory, ras_model_s3_prefix, bucket) diff --git a/production/step_11_fim_lib_stac.py b/production/step_11_fim_lib_stac.py deleted file mode 100644 index 5c68f527..00000000 --- a/production/step_11_fim_lib_stac.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -import logging -from ripple1d.ops.fim_lib import fim_lib_stac -from ripple1d.ripple1d_logger import configure_logging - -if __name__ == "__main__": - configure_logging(level=logging.INFO) - nwm_reach_id = "2823932" - ras_project_directory = os.path.dirname(__file__).replace( - "production", f"tests\\ras-data\\Baxter\\submodels\\{nwm_reach_id}" - ) - s3_prefix = "stac/test-data/fim_libs" - bucket = "fim" - - fim_lib_stac(ras_project_directory, nwm_reach_id, s3_prefix, bucket) diff --git a/production/step_1_extract_geometry.py b/production/step_1_extract_geometry.py deleted file mode 100644 index 929b96a3..00000000 --- a/production/step_1_extract_geometry.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Extract geopackage from HEC-RAS geometry file.""" - -import logging -import traceback - -from dotenv import find_dotenv, load_dotenv - -from production.db_utils import PGFim -from ripple1d.errors import ( - NotAPrjFile, -) -from ripple1d.ras_to_gpkg import gpkg_from_ras_s3 -from ripple1d.ripple1d_logger import configure_logging -from ripple1d.utils.s3_utils import init_s3_resources, list_keys - -load_dotenv() - - -def process_one_geom( - key: str, - crs: str, - bucket: str = None, -): - """Process one geometry file and convert it to geopackage.""" - # create path name for gpkg - if key.endswith(".prj"): - gpkg_path = key.replace("prj", "gpkg").replace("cases", "gpkg_s1") - else: - raise NotAPrjFile(f"{key} does not have a '.prj' extension") - - # read the geometry and write the geopackage - if bucket: - gpkg_from_ras_s3(key, crs, gpkg_path, bucket) - return f"s3://{bucket}/{gpkg_path}" - - -def main(crs_table_name: str, processing_table: str, mip_group: str, bucket: str = None): - """Read from database a list of ras files to convert to geopackage.""" - db = PGFim() - processing_data = db.read_cases( - processing_table, - ["s3_key"], - mip_group, - ) - processing_data = [i[0] for i in processing_data] - - _, client, _ = init_s3_resources() - prefix = "mip/new-cases" - new_cases_s3_keys = list_keys(client, bucket, prefix, suffix=".prj") - - crs_data = db.read_cases(crs_table_name, ["s3_key", "crs"], mip_group) - - for i, (key, crs) in enumerate(crs_data): - - key = key.replace(f"s3://{bucket}/", "") - if key not in processing_data: - mip_case = key.split("cases/")[1].split("/")[0] - - if key.replace("cases", "new-cases") in new_cases_s3_keys: - key = key.replace("cases", "new-cases") - - logging.info( - f"Working on ({i+1}/{len(crs_data)} | {round(100*(i+1)/len(crs_data),1)}% | key: {key} | mip case: {mip_case}" - ) - try: - _ = process_one_geom(key, crs, bucket) - db.update_case_status(processing_table, mip_group, mip_case, key, True, None, None, "gpkg") - - except Exception as e: - exc = str(e) - tb = str(traceback.format_exc()) - logging.error(exc) - logging.error(tb) - db.update_case_status(processing_table, mip_group, mip_case, key, False, exc, tb, "gpkg") - - -if __name__ == "__main__": - configure_logging(level=logging.INFO, logfile="extract_geometry.log") - load_dotenv(find_dotenv()) - - crs_table_name = "inferred_crs_v2" - bucket = "fim" - mip_group = "b" - processing_table = "processing_s1" - - main(crs_table_name, processing_table, mip_group, bucket) diff --git a/production/step_2_create_stac.py b/production/step_2_create_stac.py deleted file mode 100644 index a0bd7d41..00000000 --- a/production/step_2_create_stac.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Create STAC Item for HEC-RAS model.""" - -import logging -import traceback -from time import sleep - -from production.db_utils import PGFim -import ripple1d -from ripple1d.ras_to_gpkg import new_stac_item_s3 -from ripple1d.ripple1d_logger import configure_logging - - -def main(mip_group: str, processing_table_name: str, bucket: str): - """Read from database a list of geopackages to create stac items.""" - db = PGFim() - optional_condition = "AND gpkg_complete=true AND stac_complete IS NULL" - data = db.read_cases(processing_table_name, ["case_id", "s3_key"], mip_group, optional_condition) - while data: - for i, (mip_case, s3_ras_project_key) in enumerate(data): - gpkg_key = s3_ras_project_key.replace(".prj", ".gpkg").replace("dev2", "gpkg_tx_ble_1") - thumbnail_png_s3_key = ( - s3_ras_project_key.replace("mip", "stac").replace("dev2", "tx_ble_1").replace(".prj", ".png") - ) - new_stac_item_s3_key = ( - s3_ras_project_key.replace("mip", "stac").replace("dev2", "tx_ble_1").replace(".prj", ".json") - ) - - logging.info( - f"Progress: ({i+1}/{len(data)} | {round(100*(i+1)/len(data),1)}% | working on key: {s3_ras_project_key}" - ) - - try: - new_stac_item_s3( - gpkg_key, - new_stac_item_s3_key, - thumbnail_png_s3_key, - s3_ras_project_key, - bucket, - mip_case, - ) - - db.update_case_status( - processing_table_name, mip_group, mip_case, s3_ras_project_key, True, None, None, "stac" - ) - logging.debug(f"Successfully finished stac item for {s3_ras_project_key}") - except Exception as e: - exc = str(e) - tb = str(traceback.format_exc()) - logging.error(exc) - logging.error(tb) - db.update_case_status( - processing_table_name, mip_group, mip_case, s3_ras_project_key, False, exc, tb, "stac" - ) - sleep(1) - data = db.read_cases(processing_table_name, ["case_id", "s3_key"], mip_group, optional_condition) - - -if __name__ == "__main__": - configure_logging(level=logging.INFO, logfile="create_stac.log") - - mip_group = "tx_ble_1" - processing_table_name = "processing_tx_ble_1" - bucket = "fim" - - main(mip_group, processing_table_name, bucket) diff --git a/production/step_3_5_metrics.py b/production/step_3_5_metrics.py deleted file mode 100644 index 832cd7f8..00000000 --- a/production/step_3_5_metrics.py +++ /dev/null @@ -1,10 +0,0 @@ -from ripple1d.ops.metrics import compute_conflation_metrics -from ripple1d.ripple1d_logger import configure_logging - -if __name__ == "__main__": - TEST_DIR = os.path.dirname(__file__).replace("production", "tests") - - src_gpkg_path = os.path.join(TEST_DIR, "ras-data\\Baxter\\Baxter.gpkg") - conflation_json = os.path.join(TEST_DIR, "ras-data\\Baxter\\Baxter.conflation.json") - nwm_pq_path = os.path.join(TEST_DIR, "nwm-data\\flows.parquet") - conflation_parameters = compute_conflation_metrics(src_gpkg_path, nwm_pq_path, conflation_json) diff --git a/production/step_3_conflate_model.py b/production/step_3_conflate_model.py deleted file mode 100644 index 7b4b36bb..00000000 --- a/production/step_3_conflate_model.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Conflate HEC-RAS model with NWM hydrofabric.""" - -import logging -import traceback -from time import sleep - -import pystac - -from production.db_utils import PGFim -from ripple1d.conflate.rasfim import RasFimConflater -from ripple1d.ops.ras_conflate import conflate_s3_model, href_to_vsis -from ripple1d.ripple1d_logger import configure_logging -from ripple1d.utils.s3_utils import init_s3_resources - - -def main(processing_table_name: str, mip_group: str, bucket: str, nwm_pq_path: str): - """Read from database a list of geopackages to create stac items.""" - db = PGFim() - - _, client, _ = init_s3_resources() - rfc = None - optional_condition = "AND stac_complete=true AND conflation_complete IS NULL" - data = db.read_cases( - processing_table_name, - [ - "case_id", - "s3_key", - ], - mip_group, - optional_condition, - ) - while data: - for i, (mip_case, s3_ras_project_key) in enumerate(data): - try: - logging.info( - f"Progress: ({i+1}/{len(data)} | {round(100*(i+1)/len(data),1)}% | working on key: {s3_ras_project_key}" - ) - - stac_item_s3_key = s3_ras_project_key.replace("mip", "stac").replace(".prj", ".json") - stac_item_href = f"https://{bucket}.s3.amazonaws.com/{stac_item_s3_key}".replace(" ", "%20") - - item = pystac.Item.from_file(stac_item_href) - - for asset in item.get_assets(role="ras-geometry-gpkg"): - ras_gpkg = href_to_vsis(item.assets[asset].href, bucket=bucket) - - if not rfc: - rfc = RasFimConflater(nwm_pq_path, ras_gpkg) - else: - rfc.set_ras_gpkg(ras_gpkg) - - for river_reach_name in rfc.ras_river_reach_names: - logging.debug(f"item_id {item.id}, river_reach {river_reach_name}") - - conflate_s3_model( - item, - client, - bucket, - stac_item_s3_key, - rfc, - ) - logging.debug(f"{item.id}: Successfully processed") - db.update_case_status( - processing_table_name, mip_group, mip_case, s3_ras_project_key, True, None, None, "conflation" - ) - - except Exception as e: - exc = str(e) - tb = str(traceback.format_exc()) - logging.error(exc) - logging.error(tb) - db.update_case_status( - processing_table_name, mip_group, mip_case, s3_ras_project_key, False, exc, tb, "conflation" - ) - sleep(360) - - data = db.read_cases( - processing_table_name, - [ - "case_id", - "s3_key", - ], - mip_group, - optional_condition, - ) - - -if __name__ == "__main__": - configure_logging(level=logging.INFO, logfile="create_stac.log") - mip_group = "b" - processing_table_name = "processing_v2" - bucket = "fim" - nwm_pq_path = r"C:\Users\mdeshotel\Downloads\nwm_flows_v3.parquet" - - main(processing_table_name, mip_group, bucket, nwm_pq_path) diff --git a/production/step_4_subset_gpkg.py b/production/step_4_subset_gpkg.py deleted file mode 100644 index 568937b0..00000000 --- a/production/step_4_subset_gpkg.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Create geopackage for cross sections that conflated to NWM hydrofabric.""" - -import json -import logging -import os - -from ripple1d.ops.subset_gpkg import extract_submodel -from ripple1d.ripple1d_logger import configure_logging - -if __name__ == "__main__": - configure_logging(level=logging.INFO) - - SOURCE_MODEL_DIR = os.path.dirname(__file__).replace("production", "tests\\ras-data\\Baxter") - nwm_id = "2823932" - submodel_dir = f"{SOURCE_MODEL_DIR}\\submodels\\{nwm_id}" - - extract_submodel( - SOURCE_MODEL_DIR, - submodel_dir, - nwm_id, - ) diff --git a/production/step_5_create_terrain.py b/production/step_5_create_terrain.py deleted file mode 100644 index 845841ab..00000000 --- a/production/step_5_create_terrain.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Create HEC-RAS terrain.""" - -import json -import logging -import os - -from ripple1d.ops.ras_terrain import create_ras_terrain -from ripple1d.ripple1d_logger import configure_logging - -if __name__ == "__main__": - configure_logging(level=logging.INFO) - SOURCE_MODEL_DIR = os.path.dirname(__file__).replace("production", "tests\\ras-data\\Baxter") - SUBMODEL_DIR = f"{SOURCE_MODEL_DIR}\\submodels\\2823932" - create_ras_terrain(SUBMODEL_DIR) diff --git a/production/step_6_initial_normal_depth.py b/production/step_6_initial_normal_depth.py deleted file mode 100644 index b632aaa6..00000000 --- a/production/step_6_initial_normal_depth.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Create HEC-RAS flow/plan files for an initial normal depth run and run the plan.""" - -import json -import logging -import os - -from ripple1d.ops.ras_run import create_model_run_normal_depth -from ripple1d.ripple1d_logger import configure_logging - -if __name__ == "__main__": - configure_logging(level=logging.INFO) - SAMPLE_DATA = os.path.dirname(__file__).replace("production", "tests\\ras-data\\Baxter") - SAMPLE_DATA = f"{SAMPLE_DATA}\\submodels\\2823932" - - r = create_model_run_normal_depth( - SAMPLE_DATA, f"ind", num_of_discharges_for_initial_normal_depth_runs=2, show_ras=False - ) diff --git a/production/step_7_incremental_normal_depth.py b/production/step_7_incremental_normal_depth.py deleted file mode 100644 index be47003a..00000000 --- a/production/step_7_incremental_normal_depth.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Create HEC-RAS flow/plan files for an incremental normal depth run and run the plan.""" - -import json -import logging -import os - -from ripple1d.ops.ras_run import run_incremental_normal_depth -from ripple1d.ripple1d_logger import configure_logging - -if __name__ == "__main__": - configure_logging(level=logging.INFO) - SAMPLE_DATA = os.path.dirname(__file__).replace("production", "tests\\ras-data\\Baxter") - SAMPLE_DATA = f"{SAMPLE_DATA}\\submodels\\2823932" - - run_incremental_normal_depth(SAMPLE_DATA, f"nd", depth_increment=2, show_ras=True) diff --git a/production/step_8_known_water_surface_elevation.py b/production/step_8_known_water_surface_elevation.py deleted file mode 100644 index 2a69aeac..00000000 --- a/production/step_8_known_water_surface_elevation.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Create HEC-RAS flow/plan files for an known water surface elevation run and run the plan.""" - -import json -import logging -import os - -import numpy as np - -from ripple1d.ops.ras_run import ( - establish_order_of_nwm_ids, - get_kwse_from_ds_model, - run_known_wse, -) -from ripple1d.ripple1d_logger import configure_logging - -if __name__ == "__main__": - configure_logging(level=logging.INFO) - - SAMPLE_DATA = os.path.dirname(__file__).replace("production", "tests\\ras-data\\Baxter") - SAMPLE_DATA = f"{SAMPLE_DATA}\\submodels\\2823932" - - r = run_known_wse( - SAMPLE_DATA, - "kwse", - min_elevation=60.0, - max_elevation=62.0, - depth_increment=1.0, - ras_version="631", - show_ras=True, - ) diff --git a/production/step_9_create_fim_lib.py b/production/step_9_create_fim_lib.py deleted file mode 100644 index b017f98f..00000000 --- a/production/step_9_create_fim_lib.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Create FIM library.""" - -import json -import logging -import os - -from ripple1d.ops.fim_lib import create_fim_lib -from ripple1d.ripple1d_logger import configure_logging - -if __name__ == "__main__": - configure_logging(level=logging.INFO) - SAMPLE_DATA = os.path.dirname(__file__).replace("production", "tests\\ras-data\\Baxter") - SAMPLE_DATA = f"{SAMPLE_DATA}\\submodels\\2823932" - - create_fim_lib( - SAMPLE_DATA, - plans=["nd", "kwse"], - ) From 7634b36156d607e72a9fc7264ff0e8ab90cc800b Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Fri, 20 Sep 2024 05:50:49 -0400 Subject: [PATCH 24/27] bump version --- pyproject.toml | 2 +- ripple1d/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e32ec628..9b7256ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ripple1d" -version = "0.6.0" +version = "0.6.1" description = "HEC-RAS model automation" readme = "README.md" maintainers = [ diff --git a/ripple1d/version.py b/ripple1d/version.py index f5fbb780..ea95a7ef 100644 --- a/ripple1d/version.py +++ b/ripple1d/version.py @@ -1,3 +1,3 @@ """ripple1d version.""" -__version__ = "0.6.0" +__version__ = "0.6.1" From 4a9692a114417c268b77cc47cf6a62cfd59b67bd Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Fri, 20 Sep 2024 05:53:44 -0400 Subject: [PATCH 25/27] remove prod ref from readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index e53fcb8f..aae5f1f5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ Utilities for repurposing HEC-RAS models for use in the production of Flood Inun ## Contents - [api](api/) : Source code for the [Flask](https://flask.palletsprojects.com/en/3.0.x/) API and [Huey](https://huey.readthedocs.io/en/latest/) queueing system for managing parallel compute. - - [production](production/) (*Deprecation Warning*) : This directory contains scripts used by the development team for testing ripple1d outside of the API. The contents are not included in the PyPi package and may not be stable or up to date. - [ripple1d](ripple1d/): Source code for the ripple1d library. - [tests](tests/): Unit tests.up From cf6b5fcb57eb42ec17dd0e3ae27ccba27d205f1a Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Fri, 20 Sep 2024 06:17:33 -0400 Subject: [PATCH 26/27] update users change_log and api ref docs --- docs/api_reference.rst | 2 +- docs/users_change_log.rst | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/api_reference.rst b/docs/api_reference.rst index 3b8752f9..145103a5 100644 --- a/docs/api_reference.rst +++ b/docs/api_reference.rst @@ -3,7 +3,7 @@ API Reference / Postman Collection For reference and documentation of the API, please open the postman collection for the version of ripple1d -`v0.6.0: `_ This beta version contains new args for the create_fim_lib endpoint: +`v0.6.0-v0.6.1: `_ This beta version contains new args for the create_fim_lib endpoint: - `library_directory`: Specifies the output directory for the FIM grids and database. - `cleanup`: Boolean indicating if the ras HEC-RAS output grids should be deleted or not. diff --git a/docs/users_change_log.rst b/docs/users_change_log.rst index 0f772058..decaabc4 100644 --- a/docs/users_change_log.rst +++ b/docs/users_change_log.rst @@ -3,6 +3,28 @@ Change Log for Users Go to the `Releases `_ page for a list of all releases. +Feature Release 0.6.1 +~~~~~~~~~~~~~~~~~~~~~ + +Users Changelog +---------------- +This release of `ripple1d` fixes several bugs identified during testing. + +Features Added +---------------- +- A minor change was added to the logging behavior to improve error tracking. + +Bug Fixes +---------- +- A bug causing increasing processing time when calling `creat_ras_terrain` in parallel mode. +- A bug in the `extract_submodel` endpoint which failed when trying to grab the upstream cross section. A check was added for the eclipsed parameter, where if true no geopackage will be created. +- Several bugs associated with the `create_fim_lib endpoint`: + + 1. The library_directory arg was not being implemented correctly in the function. + 2. If a fim.db already exists append fuctionality has been implemented. + 3. If the directory containing the raw RAS depth grids is empty the clean up function will not be invoked. +- Resolves issues introduced when a concave hull from a source model where cross section existed in the wrong direction (resulting in a multipart polygon). A check was added to correct direction and reverses the cross section if it was drawn incorrectly. This is limited to the development of the concave hull and does not modify the cross section direction for use in the modeling. + Feature Release 0.6.0 ~~~~~~~~~~~~~~~~~~~~~ Users Changelog From df97c61dcb287e1c033ff043aebeacc21cd2ac26 Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Fri, 20 Sep 2024 06:28:21 -0400 Subject: [PATCH 27/27] fix double quote in f string --- ripple1d/utils/ripple_utils.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/ripple1d/utils/ripple_utils.py b/ripple1d/utils/ripple_utils.py index 8504328e..f95e3c18 100644 --- a/ripple1d/utils/ripple_utils.py +++ b/ripple1d/utils/ripple_utils.py @@ -73,16 +73,25 @@ def get_path(expected_path: str, client: boto3.client = None, bucket: str = None if path.endswith(Path(expected_path).suffix.upper()): return path -def fix_reversed_xs(xs:gpd.GeoDataFrame,river:gpd.GeoDataFrame)->gpd.GeoDataFrame: + +def fix_reversed_xs(xs: gpd.GeoDataFrame, river: gpd.GeoDataFrame) -> gpd.GeoDataFrame: """Check if cross sections are drawn right to left looking downstream. If not reverse them.""" - subsets=[] - for _,reach in river.iterrows(): - subset_xs=xs.loc[xs["river_reach"]== reach["river_reach"]] - not_reversed_xs=check_xs_direction(subset_xs,reach.geometry) - subset_xs["geometry"]=subset_xs.apply(lambda row: row.geometry if row["river_reach_rs"] in list(not_reversed_xs["river_reach_rs"]) else reverse(row.geometry),axis=1) + subsets = [] + for _, reach in river.iterrows(): + subset_xs = xs.loc[xs["river_reach"] == reach["river_reach"]] + not_reversed_xs = check_xs_direction(subset_xs, reach.geometry) + subset_xs["geometry"] = subset_xs.apply( + lambda row: ( + row.geometry + if row["river_reach_rs"] in list(not_reversed_xs["river_reach_rs"]) + else reverse(row.geometry) + ), + axis=1, + ) subsets.append(subset_xs) return pd.concat(subsets) + def validate_point(geom): """Validate that point is of type Point. If Multipoint or Linestring create point from first coordinate pair.""" if isinstance(geom, Point): @@ -94,6 +103,7 @@ def validate_point(geom): else: raise TypeError(f"expected point at xs-river intersection got: {type(geom)}") + def check_xs_direction(cross_sections: gpd.GeoDataFrame, reach: LineString): """Return only cross sections that are drawn right to left looking downstream.""" river_reach_rs = [] @@ -112,10 +122,13 @@ def check_xs_direction(cross_sections: gpd.GeoDataFrame, reach: LineString): river_reach_rs.append(xs["river_reach_rs"]) except TypeError as e: - logging.warning(f"could not validate xs-river intersection for: {xs["river"]} {xs['reach']} {xs['river_station']}") + logging.warning( + f"could not validate xs-river intersection for: {xs['river']} {xs['reach']} {xs['river_station']}" + ) continue return cross_sections.loc[cross_sections["river_reach_rs"].isin(river_reach_rs)] + def xs_concave_hull(xs: gpd.GeoDataFrame, junction: gpd.GeoDataFrame = None) -> gpd.GeoDataFrame: """Compute and return the concave hull (polygon) for a set of cross sections (lines all facing the same direction).""" polygons = []