From 55f163aa6d1343c6c7e7ebf12a41214e1d161a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20C=2E=20Riven=C3=A6s?= Date: Wed, 20 Oct 2021 13:23:12 +0200 Subject: [PATCH] Improve timedata spesification, cf. #101 --- src/fmu/dataio/_export_item.py | 52 +++++++++++++------ src/fmu/dataio/dataio.py | 2 +- tests/test_export_item.py | 22 ++++----- tests/test_fmu_dataio_surface.py | 85 ++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 28 deletions(-) diff --git a/src/fmu/dataio/_export_item.py b/src/fmu/dataio/_export_item.py index 183e9c3b5..1cae8c747 100644 --- a/src/fmu/dataio/_export_item.py +++ b/src/fmu/dataio/_export_item.py @@ -499,16 +499,29 @@ def _data_process_timedata(self): # first detect if timedata is given, the process it logger.info("Evaluate data:name attribute") meta = self.dataio.metadata4data + + datelimits = (18140517, 33000101) + if self.timedata is None: return self.times = [] # e.g. ["20211102", "20231101"] or ["20211102", None] for xtime in self.timedata: + if isinstance(xtime[0], int): + if xtime[0] < datelimits[0] or xtime[0] > datelimits[1]: + raise ValidationError( + "Integer date input seems to be outside reasonable " + f"limits: {datelimits}" + ) tdate = str(xtime[0]) tlabel = None if len(xtime) > 1: tlabel = xtime[1] tdate = tdate.replace("-", "") # 2021-04-23 --> 20210403 + if tdate and int(tdate) < datelimits[0] or int(tdate) > datelimits[1]: + raise ValidationError( + f"Date input outside reasonable limits: {datelimits}" + ) tdate = datetime.strptime(tdate, "%Y%m%d") self.times.append(tdate) tdate = tdate.strftime("%Y-%m-%dT%H:%M:%S") @@ -854,7 +867,7 @@ def _construct_filename_fmustandard1(self): surface: namehorizon--tagname namehorizon--tagname--t1 - namehorizon--tagname--t2_t1 + namehorizon--tagname--t1_t2 # t1 is monitor time while t2 is base time e.g. topvolantis--ds_gf_extracted @@ -866,20 +879,21 @@ def _construct_filename_fmustandard1(self): gridproperty gridname--proptagname gridname--tagname--t1 - gridname--tagname--t2_t1 + gridname--tagname--t1_t2 e.g. geogrid_valysar--phit Destinations accoring to datatype. - Removing dots from filename: - Currently, when multiple dots in a filename stem, - XTgeo, using pathlib, will interpret the part after the - last dot as the file suffix, and remove it. This causes - errors in the output filenames. While this is being - taken care of in XTgeo, we temporarily sanitize dots from - the outgoing filename only to avoid this. + For timedata with two dates, the standard is some--monitortime_basetime. Hence + t1 is newer than t2. + + Removing dots from filename: Currently, when multiple dots in a filename stem, + XTgeo, using pathlib, will interpret the part after the last dot as the file + suffix, and remove it. This causes errors in the output filenames. While this is + being taken care of in XTgeo, we temporarily sanitize dots from the outgoing + filename only to avoid this. Space will also be replaced in file names. @@ -898,13 +912,19 @@ def _construct_filename_fmustandard1(self): stem = self.parent.lower() + "--" + stem if self.times: - time1 = self.times[0] - time2 = self.times[1] - if time1 and not time2: - stem += "--" + str(time1) - - elif time1 and time2: - stem += "--" + str(time2) + "_" + str(time1) + time0 = self.times[0] + time1 = self.times[1] + if time0 and not time1: + stem += "--" + (str(time0)[0:10]).replace("-", "") + + elif time0 and time1: + monitor = (str(time0)[0:10]).replace("-", "") + base = (str(time1)[0:10]).replace("-", "") + if monitor == base: + warnings.warn( + "The monitor date and base date are equal", UserWarning + ) # TODO: consider add clocktimes in such cases? + stem += "--" + monitor + "_" + base stem = stem.replace(".", "_").replace(" ", "_") diff --git a/src/fmu/dataio/dataio.py b/src/fmu/dataio/dataio.py index 5a88c9daa..9f8e4aeee 100644 --- a/src/fmu/dataio/dataio.py +++ b/src/fmu/dataio/dataio.py @@ -120,7 +120,7 @@ class ExportData: vertical_domain: This is dictionary with a key and a reference e.g. {"depth": "msl"} which is default (if None is input) timedata: If given, a list of lists with dates, .e.g. - [[20200101, "firsttime"], [20180101, "secondtime"]] or just [[20210101]] + [[20200101, "monitor"], [20180101, "base"]] or just [[20210101]] is_prediction: True (default) of model prediction data is_observation: Default is False. workflow: Short tag desciption of workflow (as description) diff --git a/tests/test_export_item.py b/tests/test_export_item.py index d5ac16f68..80be446f7 100644 --- a/tests/test_export_item.py +++ b/tests/test_export_item.py @@ -151,7 +151,7 @@ def test_data_process_timedata(): name="Valysar", config=CFG2, content="depth", - timedata=[["20210101", "first"], [20210902, "second"]], + timedata=[["20230101", "monitor"], [20210902, "base"]], tagname="WhatEver", ) obj = xtgeo.RegularSurface( @@ -160,8 +160,8 @@ def test_data_process_timedata(): exportitem = ei._ExportItem(dataio, obj, verbosity="INFO") exportitem._data_process_timedata() print(json.dumps(dataio.metadata4data["time"], indent=2, default=str)) - assert dataio.metadata4data["time"][0]["value"] == "2021-01-01T00:00:00" - assert dataio.metadata4data["time"][0]["label"] == "first" + assert dataio.metadata4data["time"][0]["value"] == "2023-01-01T00:00:00" + assert dataio.metadata4data["time"][0]["label"] == "monitor" def test_data_process_content(): @@ -171,7 +171,7 @@ def test_data_process_content(): name="Valysar", config=CFG2, content="depth", - timedata=[["20210101", "first"], [20210902, "second"]], + timedata=[["20230101", "monitor"], [20210902, "base"]], tagname="WhatEver", ) obj = xtgeo.RegularSurface( @@ -186,7 +186,7 @@ def test_data_process_content(): name="Valysar", config=CFG2, content={"seismic": {"attribute": "attribute_timeshifted_somehow"}}, - timedata=[["20210101", "first"], [20210902, "second"]], + timedata=[["20230101", "monitor"], [20210902, "base"]], tagname="WhatEver", ) obj = xtgeo.RegularSurface( @@ -207,7 +207,7 @@ def test_data_process_content_shall_fail(): name="Valysar", config=CFG2, content="something_invalid", - timedata=[["20210101", "first"], [20210902, "second"]], + timedata=[["20230101", "monitor"], [20210902, "base"]], tagname="WhatEver", ) obj = xtgeo.RegularSurface( @@ -223,7 +223,7 @@ def test_data_process_content_shall_fail(): name="Valysar", config=CFG2, content={"seismic": {"attribute": 100}}, - timedata=[["20210101", "first"], [20210902, "second"]], + timedata=[["20230101", "monitor"], [20210902, "base"]], tagname="WhatEver", ) obj = xtgeo.RegularSurface( @@ -239,7 +239,7 @@ def test_data_process_content_shall_fail(): name="Valysar", config=CFG2, content={"seismic": {"invalid_attribute": "some"}}, - timedata=[["20210101", "first"], [20210902, "second"]], + timedata=[["20230101", "monitor"], [20210902, "base"]], tagname="WhatEver", ) obj = xtgeo.RegularSurface( @@ -286,7 +286,7 @@ def test_data_process_content_fluid_contact(): name="Valysar", config=CFG2, content={"fluid_contact": {"contact": "owc"}}, - timedata=[["20210101", "first"], [20210902, "second"]], + timedata=[["20230101", "monitor"], [20210902, "base"]], tagname="WhatEver", ) obj = xtgeo.Polygons() @@ -298,7 +298,7 @@ def test_data_process_content_fluid_contact(): name="Valysar", config=CFG2, content={"fluid_contact": {"wrong": "owc"}}, - timedata=[["20210101", "first"], [20210902, "second"]], + timedata=[["20210101", "monitor"], [20210902, "base"]], tagname="WhatEver", ) obj = xtgeo.Polygons() @@ -312,7 +312,7 @@ def test_data_process_content_fluid_contact(): name="Valysar", config=CFG2, content={"field_outline": {"wrong": "owc"}}, - timedata=[["20210101", "first"], [20210902, "second"]], + timedata=[["20230101", "monitor"], [20210902, "base"]], tagname="WhatEver", ) obj = xtgeo.Polygons() diff --git a/tests/test_fmu_dataio_surface.py b/tests/test_fmu_dataio_surface.py index 09eabbe19..d3b3ccf63 100644 --- a/tests/test_fmu_dataio_surface.py +++ b/tests/test_fmu_dataio_surface.py @@ -3,6 +3,7 @@ import logging import shutil from collections import OrderedDict +from pathlib import Path import numpy as np import pytest @@ -51,6 +52,90 @@ def test_surface_io(tmp_path): assert (tmp_path / FMUP1 / "maps" / ".test.gri.yml").is_file() is True +@pytest.mark.parametrize( + "dates, expected", + [ + ( + [[20440101, "monitor"], [20230101, "base"]], + "test--20440101_20230101", + ), + ( + [["20440101", "monitor"], [20230101, "base"]], + "test--20440101_20230101", + ), + ( + [["20440101", "monitor"], ["20230101", "base"]], + "test--20440101_20230101", + ), + ( + [["2044-01-01", "monitor"], ["20230101", "base"]], + "test--20440101_20230101", + ), + ], +) +def test_surface_io_with_timedata(tmp_path, dates, expected): + """Minimal test surface io with timedata, uses tmp_path.""" + + # make a fake RegularSurface + srf = xtgeo.RegularSurface( + ncol=20, nrow=30, xinc=20, yinc=20, values=np.ma.ones((20, 30)), name="test" + ) + fmu.dataio.ExportData.surface_fformat = "irap_binary" + + exp = fmu.dataio.ExportData( + content="depth", + timedata=dates, + runpath=tmp_path, + ) + out = Path(exp.export(srf)).stem + + assert expected == out + + +@pytest.mark.parametrize( + "dates, errmessage", + [ + ( + [[40440101, "monitor"], [20230101, "base"]], + "Integer date input seems to be outside reasonable limits", + ), + ( + [[20210101, "monitor"], [17220101, "base"]], + "Integer date input seems to be outside reasonable limits", + ), + ( + [["20210101", "monitor"], ["17220101", "base"]], + "Date input outside reasonable limits", + ), + ( + [["2021-01-01", "monitor"], ["1722-01-01", "base"]], + "Date input outside reasonable limits", + ), + ( + [["666", "monitor"], ["1722-01-01", "base"]], + "Date input outside reasonable limits", + ), + ], +) +def test_surface_io_with_timedata_shall_fail(tmp_path, dates, errmessage): + """Minimal test surface io with timedata, uses tmp_path, with invalid input.""" + + # make a fake RegularSurface + srf = xtgeo.RegularSurface( + ncol=20, nrow=30, xinc=20, yinc=20, values=np.ma.ones((20, 30)), name="test" + ) + fmu.dataio.ExportData.surface_fformat = "irap_binary" + + exp = fmu.dataio.ExportData( + content="depth", + timedata=dates, + runpath=tmp_path, + ) + with pytest.raises(ValueError) as err: + exp.export(srf) + assert errmessage in str(err.value) + + def test_surface_io_export_subfolder(tmp_path): """Minimal test surface io with export_subfolder set, uses tmp_path."""