Skip to content

Commit

Permalink
CLN: Construct Spec using pydantic
Browse files Browse the repository at this point in the history
  • Loading branch information
JB Lovland committed Feb 9, 2024
1 parent f4908a8 commit e027058
Show file tree
Hide file tree
Showing 2 changed files with 169 additions and 123 deletions.
279 changes: 157 additions & 122 deletions src/fmu/dataio/_objectdata_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@
from dataclasses import dataclass, field
from datetime import datetime as dt
from pathlib import Path
from typing import Any, Final, Optional
from typing import TYPE_CHECKING, Any, Dict, Final, NamedTuple, Optional
from warnings import warn

import numpy as np
Expand All @@ -98,15 +98,23 @@
from ._definitions import ALLOWED_CONTENTS, STANDARD_TABLE_INDEX_COLUMNS, _ValidFormats
from ._logging import null_logger
from ._utils import generate_description, parse_timedata
from .datastructure.meta import meta
from .datastructure.meta import meta, specification

logger: Final = null_logger(__name__)

if TYPE_CHECKING:
import pandas as pd


class ConfigurationError(ValueError):
pass


class SpecificationAndBoundingBox(NamedTuple):
spec: Dict[str, Any]
bbox: Dict[str, Any]


@dataclass
class _ObjectDataProvider:
"""Class for providing metadata for data objects in fmu-dataio, e.g. a surface.
Expand Down Expand Up @@ -346,12 +354,25 @@ def _derive_objectdata(self) -> dict:

return result

def _derive_spec_bbox_regularsurface(self) -> tuple[dict, dict]:
def _derive_spec_bbox_regularsurface(self) -> SpecificationAndBoundingBox:
"""Process/collect the data.spec and data.bbox for RegularSurface"""
logger.info("Derive bbox and specs for RegularSurface")
regsurf = self.obj
regsurf: xtgeo.RegularSurface = self.obj

specs = {}
""""
REQUIRED = {
"ncol": 1,
"nrow": 1,
"xori": 0.0,
"yori": 0.0,
"xinc": 1.0,
"yinc": 1.0,
"yflip": 1,
"rotation": 0.0,
"undef": xtgeo.UNDEF,
}
"""

xtgeo_specs = regsurf.metadata.required
for spec, val in xtgeo_specs.items():
Expand All @@ -360,70 +381,78 @@ def _derive_spec_bbox_regularsurface(self) -> tuple[dict, dict]:
specs[spec] = val
specs["undef"] = 1.0e30 # irap binary undef

return specs, meta.content.BoundingBox(
xmin=float(regsurf.xmin),
xmax=float(regsurf.xmax),
ymin=float(regsurf.ymin),
ymax=float(regsurf.ymax),
zmin=float(regsurf.values.min()),
zmax=float(regsurf.values.max()),
).model_dump(
mode="json",
exclude_none=True,
return SpecificationAndBoundingBox(
spec=specs,
bbox=meta.content.BoundingBox(
xmin=float(regsurf.xmin),
xmax=float(regsurf.xmax),
ymin=float(regsurf.ymin),
ymax=float(regsurf.ymax),
zmin=float(regsurf.values.min()),
zmax=float(regsurf.values.max()),
).model_dump(
mode="json",
exclude_none=True,
),
)

def _derive_spec_bbox_polygons(self) -> tuple[dict, dict]:
def _derive_spec_bbox_polygons(self) -> SpecificationAndBoundingBox:
"""Process/collect the data.spec and data.bbox for Polygons"""
logger.info("Derive bbox and specs for Polygons")
poly = self.obj

specs = {}
# number of polygons:
specs["npolys"] = np.unique(
poly.get_dataframe(copy=False)[poly.pname].values
).size
poly: xtgeo.Polygons = self.obj
xmin, xmax, ymin, ymax, zmin, zmax = poly.get_boundary()

return specs, meta.content.BoundingBox(
xmin=float(xmin),
xmax=float(xmax),
ymin=float(ymin),
ymax=float(ymax),
zmin=float(zmin),
zmax=float(zmax),
).model_dump(
mode="json",
exclude_none=True,
return SpecificationAndBoundingBox(
spec=specification.PolygonsSpecification(
npolys=np.unique(poly.get_dataframe(copy=False)[poly.pname].values).size
).model_dump(
mode="json",
exclude_none=True,
),
bbox=meta.content.BoundingBox(
xmin=float(xmin),
xmax=float(xmax),
ymin=float(ymin),
ymax=float(ymax),
zmin=float(zmin),
zmax=float(zmax),
).model_dump(
mode="json",
exclude_none=True,
),
)

def _derive_spec_bbox_points(self) -> tuple[dict[str, Any], dict[str, Any]]:
def _derive_spec_bbox_points(self) -> SpecificationAndBoundingBox:
"""Process/collect the data.spec and data.bbox for Points"""
logger.info("Derive bbox and specs for Points")
pnts = self.obj

specs: dict[str, Any] = {}

if len(pnts.get_dataframe(copy=False).columns) > 3:
attrnames = pnts.get_dataframe(copy=False).columns[3:]
specs["attributes"] = list(attrnames)
specs["size"] = int(pnts.get_dataframe(copy=False).size)

return specs, meta.content.BoundingBox(
xmin=float(pnts.get_dataframe(copy=False)[pnts.xname].min()),
xmax=float(pnts.get_dataframe(copy=False)[pnts.xname].max()),
ymax=float(pnts.get_dataframe(copy=False)[pnts.yname].min()),
ymin=float(pnts.get_dataframe(copy=False)[pnts.yname].max()),
zmin=float(pnts.get_dataframe(copy=False)[pnts.zname].min()),
zmax=float(pnts.get_dataframe(copy=False)[pnts.zname].max()),
).model_dump(
mode="json",
exclude_none=True,
pnts: xtgeo.Points = self.obj
df: pd.DataFrame = pnts.get_dataframe(copy=False)

return SpecificationAndBoundingBox(
spec=specification.PointsSpecification(
attributes=list(df.columns[3:]) if len(df.columns) > 3 else None,
size=int(df.size),
).model_dump(
mode="json",
exclude_none=True,
),
bbox=meta.content.BoundingBox(
xmin=float(df[pnts.xname].min()),
xmax=float(df[pnts.xname].max()),
ymax=float(df[pnts.yname].min()),
ymin=float(df[pnts.yname].max()),
zmin=float(df[pnts.zname].min()),
zmax=float(df[pnts.zname].max()),
).model_dump(
mode="json",
exclude_none=True,
),
)

def _derive_spec_bbox_cube(self) -> tuple[dict, dict]:
def _derive_spec_bbox_cube(self) -> SpecificationAndBoundingBox:
"""Process/collect the data.spec and data.bbox Cube"""
logger.info("Derive bbox and specs for Cube")
cube = self.obj
cube: xtgeo.Cube = self.obj

specs = {}

Expand All @@ -435,34 +464,35 @@ def _derive_spec_bbox_cube(self) -> tuple[dict, dict]:

# current xtgeo is missing xmin, xmax etc attributes for cube, so need
# to compute (simplify when xtgeo has this):
xmin = 1.0e23
ymin = xmin
xmax = -1 * xmin
ymax = -1 * ymin
xmin, ymin = 1.0e23, 1.0e23
xmax, ymax = -xmin, -ymin

for corner in ((1, 1), (1, cube.nrow), (cube.ncol, 1), (cube.ncol, cube.nrow)):
xco, yco = cube.get_xy_value_from_ij(*corner)
xmin = xco if xco < xmin else xmin
xmax = xco if xco > xmax else xmax
ymin = yco if yco < ymin else ymin
ymax = yco if yco > ymax else ymax

return specs, meta.content.BoundingBox(
xmin=float(xmin),
xmax=float(xmax),
ymin=float(ymin),
ymax=float(ymax),
zmin=float(cube.zori),
zmax=float(cube.zori + cube.zinc * (cube.nlay - 1)),
).model_dump(
mode="json",
exclude_none=True,
xmin = min(xmin, xco)
xmax = max(xmax, xco)
ymin = min(ymin, yco)
ymax = max(ymax, yco)

return SpecificationAndBoundingBox(
spec=specs,
bbox=meta.content.BoundingBox(
xmin=float(xmin),
xmax=float(xmax),
ymin=float(ymin),
ymax=float(ymax),
zmin=float(cube.zori),
zmax=float(cube.zori + cube.zinc * (cube.nlay - 1)),
).model_dump(
mode="json",
exclude_none=True,
),
)

def _derive_spec_bbox_cpgrid(self) -> tuple[dict, dict]:
def _derive_spec_bbox_cpgrid(self) -> SpecificationAndBoundingBox:
"""Process/collect the data.spec and data.bbox CornerPoint Grid geometry"""
logger.info("Derive bbox and specs for Gride (geometry)")
grid = self.obj
grid: xtgeo.Grid = self.obj

specs = {}

Expand All @@ -474,71 +504,76 @@ def _derive_spec_bbox_cpgrid(self) -> tuple[dict, dict]:

geox = grid.get_geometrics(cellcenter=False, allcells=True, return_dict=True)

return specs, meta.content.BoundingBox(
xmin=round(float(geox["xmin"]), 4),
xmax=round(float(geox["xmax"]), 4),
ymin=round(float(geox["ymin"]), 4),
ymax=round(float(geox["ymax"]), 4),
zmin=round(float(geox["zmin"]), 4),
zmax=round(float(geox["zmax"]), 4),
).model_dump(
mode="json",
exclude_none=True,
return SpecificationAndBoundingBox(
spec=specs,
bbox=meta.content.BoundingBox(
xmin=round(float(geox["xmin"]), 4),
xmax=round(float(geox["xmax"]), 4),
ymin=round(float(geox["ymin"]), 4),
ymax=round(float(geox["ymax"]), 4),
zmin=round(float(geox["zmin"]), 4),
zmax=round(float(geox["zmax"]), 4),
).model_dump(
mode="json",
exclude_none=True,
),
)

def _derive_spec_bbox_cpgridproperty(self) -> tuple[dict, dict]:
def _derive_spec_bbox_cpgridproperty(self) -> SpecificationAndBoundingBox:
"""Process/collect the data.spec and data.bbox GridProperty"""
logger.info("Derive bbox and specs for GridProperty")
gridprop = self.obj

specs: dict[str, Any] = {}
bbox: dict[str, Any] = {}

specs["ncol"] = gridprop.ncol
specs["nrow"] = gridprop.nrow
specs["nlay"] = gridprop.nlay
return specs, bbox
gridprop: xtgeo.GridProperty = self.obj

return SpecificationAndBoundingBox(
spec=specification.CPGridPropertySpecification(
nrow=gridprop.nrow,
ncol=gridprop.ncol,
nlay=gridprop.nlay,
).model_dump(
mode="json",
exclude_none=True,
),
bbox={},
)

def _derive_spec_bbox_dataframe(
self,
) -> tuple[
dict[str, Any],
dict[str, Any],
]:
) -> SpecificationAndBoundingBox:
"""Process/collect the data items for DataFrame."""
logger.info("Process data metadata for DataFrame (tables)")
dfr = self.obj

specs: dict[str, Any] = {}
bbox: dict[str, Any] = {}

specs["columns"] = list(dfr.columns)
specs["size"] = int(dfr.size)

return specs, bbox
df: pd.DataFrame = self.obj
return SpecificationAndBoundingBox(
spec=specification.TableSpecification(
columns=list(df.columns),
size=int(df.size),
).model_dump(
mode="json",
exclude_none=True,
),
bbox={},
)

def _derive_spec_bbox_arrowtable(
self,
) -> tuple[
dict[str, Any],
dict[str, Any],
]:
) -> SpecificationAndBoundingBox:
"""Process/collect the data items for Arrow table."""
logger.info("Process data metadata for arrow (tables)")
table = self.obj
return SpecificationAndBoundingBox(
spec=specification.TableSpecification(
columns=list(table.column_names),
size=table.num_columns * table.num_rows,
).model_dump(
mode="json",
exclude_none=True,
),
bbox={},
)

specs: dict[str, Any] = {}
bbox: dict[str, Any] = {}

specs["columns"] = list(table.column_names)
specs["size"] = table.num_columns * table.num_rows

return specs, bbox

def _derive_spec_bbox_dict(self) -> tuple[dict[str, Any], dict[str, Any]]:
def _derive_spec_bbox_dict(self) -> SpecificationAndBoundingBox:
"""Process/collect the data items for dictionary."""
logger.info("Process data metadata for dictionary")
return {}, {}
return SpecificationAndBoundingBox({}, {})

def _get_columns(self) -> list[str]:
"""Get the columns from table"""
Expand Down
Loading

0 comments on commit e027058

Please sign in to comment.