DEP: Deprecate fmu_context='preprocessed'

src/fmu/dataio/_definitions.py

@@ -83,16 +83,15 @@ class FmuContext(str, Enum):
     Use a Enum class for fmu_context entries.
 
     The different entries will impact where data is exported:
-    REALIZATION = "To realization-N/iter_M/share"
-    CASE = "To casename/share, but will also work on project disk"
-    PREPROCESSED = "To share/preprocessed; from interactive runs but re-used later"
-    NON_FMU = "Not ran in a FMU setting, e.g. interactive RMS"
-
+    REALIZATION: To realization-N/iter_M/share
+    CASE: To casename/share, but will also work on project disk
+    ITERATION: To casename/itername/share, only applicable for aggregated data
+    NON_FMU: Not ran in a FMU setting, e.g. interactive RMS.
     """
 
     REALIZATION = "realization"
     CASE = "case"
-    PREPROCESSED = "preprocessed"
+    ITERATION = "iteration"
     NON_FMU = "non-fmu"
 
     @classmethod

src/fmu/dataio/_metadata.py

@@ -15,7 +15,7 @@
 from pydantic import AnyHttpUrl, TypeAdapter
 
 from . import types
-from ._definitions import SCHEMA, SOURCE, VERSION, FmuContext
+from ._definitions import SCHEMA, SOURCE, VERSION
 from ._logging import null_logger
 from .datastructure._internal import internal
 from .datastructure.meta import meta
@@ -151,5 +151,5 @@ def generate_export_metadata(
         file=_get_meta_filedata(dataio, obj, objdata, fmudata, compute_md5),
         tracklog=generate_meta_tracklog(),
         display=_get_meta_display(dataio, objdata),
-        preprocessed=dataio.fmu_context == FmuContext.PREPROCESSED,
+        preprocessed=dataio.preprocessed,
     )

src/fmu/dataio/aggregation.py

@@ -9,6 +9,7 @@
 from pydantic import ValidationError
 
 from . import _utils, dataio, types
+from ._definitions import FmuContext
 from ._logging import null_logger
 from ._metadata import generate_meta_tracklog
 from .providers.objectdata._provider import objectdata_provider_factory
@@ -229,7 +230,7 @@ def _generate_aggrd_metadata(
         template["fmu"]["aggregation"]["id"] = self.aggregation_id
 
         # fmu.context.stage should be 'iteration'
-        template["fmu"]["context"]["stage"] = "iteration"
+        template["fmu"]["context"]["stage"] = FmuContext.ITERATION.value
 
         # next, the new object will trigger update of: 'file', 'data' (some fields) and
         # 'tracklog'.

src/fmu/dataio/dataio.py

@@ -186,10 +186,11 @@ class ExportData:
             ``{"access_level": "restricted", "rep_include": False}``
             Deprecated and replaced by 'classification' and 'rep_include' arguments.
 
-        casepath: To override the automatic and actual ``rootpath``. Absolute path to
-            the case root. If not provided, the rootpath will be attempted parsed from
-            the file structure or by other means. See also fmu_context, where "case"
-            may need an explicit casepath!
+        casepath: Optional path to a case directory that contains valid case metadata
+            "fmu_case.yml" in folder "<casepath>/share/metadata/".
+            Note for the fmu_context ``case`` the ``casepath`` is required, while
+            for fmu_context ``realization`` it will be attempted inferred from
+            an ERT environment variable.
 
         classification: Optional. Security classification level of the data object.
             If present it will override the default found in the config.
@@ -213,16 +214,14 @@ class ExportData:
             Example is "depth" or {"fluid_contact": {"xxx": "yyy", "zzz": "uuu"}}.
             Content is checked agains a white-list for validation!
 
-        fmu_context: In normal forward models, the fmu_context is ``realization`` which
-            is default and will put data per realization. Other contexts may be ``case``
-            which will put data relative to the case root (see also casepath). Another
-            important context is "preprocessed" which will output to a dedicated
-            "preprocessed" folder instead, and metadata will be partially re-used in
-            an ERT model run. If a non-FMU run is detected (e.g. you run from project),
-            fmu-dataio will detect that and set actual context to None as fall-back
-            (unless preprocessed is specified). If this key is not explicitly given it
-            will be inferred to be either "case"/"realization"/"non-fmu" based on the
-            presence of ERT environment variables.
+        fmu_context: Optional string with value ``realization`` or ``case``. If not
+            explicitly given it will be inferred based on the presence of ERT
+            environment variables.
+            The fmu_context ``realization`` will export data per realization, and should
+            be used in normal ERT forward models, while the fmu_context ``case``
+            will export data relative to the case directory. Note that for the
+            fmu_context ``case`` the case directory needs to be provided through the
+            argument ``casepath``.
 
         description: A multiline description of the data either as a string or a list
             of strings.
@@ -233,7 +232,7 @@ class ExportData:
             possible to output to a non-standard folder relative to casepath/rootpath,
             as dependent on the both fmu_context and the is_observations boolean value.
             A typical use-case is forcefolder="seismic" which will replace the "cubes"
-            standard folder for Cube output with "seismics". Use with care and avoid if
+            standard folder for Cube output with "seismic". Use with care and avoid if
             possible!
 
         geometry: Optional, and for grid properties only, which may need a
@@ -253,9 +252,9 @@ class ExportData:
         is_prediction: True (default) if model prediction data
 
         is_observation: Default is False. If True, then disk storage will be on the
-            "share/observations" folder, otherwise on share/result. An exception arise
-            if fmu_context is "preprocessed", then the folder will be set to
-            "share/processed" irrespective the value of is_observation.
+            "share/observations" folder, otherwise on "share/result". An exception arise
+            if ``preprocessed=True``, then the folder will be set to
+            "share/preprocessed" irrespective the value of ``is_observation``.
 
         name: Optional but recommended. The name of the object. If not set it is tried
             to be inferred from the xtgeo/pandas/... object. The name is then checked
@@ -271,6 +270,10 @@ class ExportData:
             `geometry` key instead. If both `parent` and `geometry` is given, the grid
             name derived from the `geometry` object will have predence.
 
+        preprocessed: Default is False. If True, the data exported are output to a
+            dedicated "share/preprocessed" folder, and metadata can be partially re-used
+            in an ERT model run using the ``ExportPreprocessedData`` class.
+
         rep_include: Optional. If True then the data object will be available in REP.
             Default is False.
 
@@ -372,6 +375,7 @@ class ExportData:
     name: str = ""
     undef_is_zero: bool = False
     parent: str = ""
+    preprocessed: bool = False
     realization: Optional[int] = None  # deprecated
     rep_include: Optional[bool] = None
     reuse_metadata_rule: Optional[str] = None  # deprecated
@@ -630,6 +634,16 @@ def _validate_and_establish_fmucontext(self) -> None:
                 UserWarning,
             )
 
+        if self.fmu_context == "preprocessed":
+            warnings.warn(
+                "Using the 'fmu_context' argument with value 'preprocessed' is "
+                "deprecated and will be removed in the future. Use the more explicit "
+                "'preprocessed' argument instead: ExportData(preprocessed=True)",
+                FutureWarning,
+            )
+            self.preprocessed = True
+            self.fmu_context = None
+
         env_fmu_context = get_fmu_context_from_environment()
         logger.debug("fmu context from input is %s", self.fmu_context)
         logger.debug("fmu context from environment is %s", env_fmu_context)
@@ -641,19 +655,24 @@ def _validate_and_establish_fmucontext(self) -> None:
                 env_fmu_context,
             )
             self.fmu_context = env_fmu_context
-        else:
-            self.fmu_context = FmuContext(self.fmu_context.lower())
-        logger.info("FMU context is %s", self.fmu_context)
 
-        if not self._fmurun and self.fmu_context != FmuContext.PREPROCESSED:
+        elif not self._fmurun:
             logger.warning(
                 "Requested fmu_context is <%s> but since this is detected as a non "
-                "FMU run, the actual context is force set to <%s>",
+                "FMU run, the actual context is force set to non-fmu",
                 self.fmu_context,
-                FmuContext.NON_FMU,
             )
             self.fmu_context = FmuContext.NON_FMU
 
+        else:
+            self.fmu_context = FmuContext(self.fmu_context.lower())
+            logger.info("FMU context is %s", self.fmu_context)
+
+        if self.preprocessed and self.fmu_context == FmuContext.REALIZATION:
+            raise ValueError(
+                "Can't export preprocessed data in a fmu_context='realization'."
+            )
+
         if self.fmu_context != FmuContext.CASE and env_fmu_context == FmuContext.CASE:
             warn(
                 "fmu_context is set to 'realization', but unable to detect "

src/fmu/dataio/providers/_filedata.py

@@ -94,7 +94,7 @@ def get_metadata(self) -> meta.File:
 
     def _get_share_folders(self) -> Path:
         """Get the export share folders."""
-        if self.dataio.fmu_context == FmuContext.PREPROCESSED:
+        if self.dataio.preprocessed:
             sharefolder = Path(ShareFolder.PREPROCESSED.value)
         elif self.dataio.is_observation:
             sharefolder = Path(ShareFolder.OBSERVATIONS.value)

tests/test_units/test_dataio.py

@@ -645,11 +645,88 @@ def test_fmu_context_outside_fmu_no_input_overwrite(rmsglobalconfig):
     For non-fmu run fmu_context should not be overwritten when input
     is "preprocessed"
     """
+    edata = ExportData(config=rmsglobalconfig, content="depth", preprocessed=True)
+    assert edata._fmurun is False
+    assert edata.preprocessed is True
+    assert edata.fmu_context == FmuContext.NON_FMU
+
+
+def test_fmu_context_preprocessed_deprecation_outside_fmu(rmsglobalconfig, regsurf):
+    """
+    Test the deprecated fmu_context="preprocessed" outside fmu.
+    This should set the preprocessed flag to True and overwrite the
+    fmu_context (if any) to "non-fmu".
+    """
+    with pytest.warns(FutureWarning, match="is deprecated"):
+        edata = ExportData(
+            config=rmsglobalconfig, content="depth", fmu_context="preprocessed"
+        )
+    assert edata.preprocessed is True
+    assert edata.fmu_context == FmuContext.NON_FMU
+
+    meta = edata.generate_metadata(regsurf)
+    assert meta["file"]["relative_path"] == "share/preprocessed/maps/unknown.gri"
+
+
+def test_fmu_context_preprocessed_deprecation_inside_fmu(
+    fmurun_prehook, rmsglobalconfig, regsurf
+):
+    """
+    Test the deprecated fmu_context="preprocessed" inside fmu.
+    This should set the preprocessed flag to True and overwrite the
+    fmu_context (if any) to "case".
+    """
+    with pytest.warns(FutureWarning, match="is deprecated"):
+        edata = ExportData(
+            config=rmsglobalconfig,
+            content="depth",
+            fmu_context="preprocessed",
+            casepath=fmurun_prehook,
+        )
+    assert edata.preprocessed is True
+    assert edata.fmu_context == FmuContext.CASE
+
+    meta = edata.generate_metadata(regsurf)
+    assert meta["file"]["relative_path"] == "share/preprocessed/maps/unknown.gri"
+
+
+def test_preprocessed_outside_fmu(rmsglobalconfig, regsurf):
+    """Test the preprocessed argument outside FMU context"""
+
+    edata = ExportData(config=rmsglobalconfig, content="depth", preprocessed=True)
+    assert edata.preprocessed is True
+    assert edata.fmu_context == FmuContext.NON_FMU
+
+    meta = edata.generate_metadata(regsurf)
+    # check that the relative file is at case level and has a preprocessed folder
+    assert meta["file"]["relative_path"] == "share/preprocessed/maps/unknown.gri"
+
+
+def test_preprocessed_inside_fmu(fmurun_w_casemetadata, rmsglobalconfig, regsurf):
+    """Test the preprocessed argument inside FMU context"""
+    # should raise error if preprocessed=True and fmu_context="realization"
+    with pytest.raises(ValueError, match="Can't export preprocessed"):
+        edata = ExportData(
+            config=rmsglobalconfig,
+            content="depth",
+            fmu_context="realization",
+            preprocessed=True,
+        )
+
+    # test that no error is raised if preprocessed=True and fmu_context="case"
     edata = ExportData(
-        config=rmsglobalconfig, content="depth", fmu_context="preprocessed"
+        config=rmsglobalconfig,
+        content="depth",
+        fmu_context="case",
+        preprocessed=True,
     )
-    assert edata._fmurun is False
-    assert edata.fmu_context == FmuContext.PREPROCESSED
+    assert edata._fmurun is True
+    assert edata.preprocessed is True
+    assert edata.fmu_context == FmuContext.CASE
+
+    meta = edata.generate_metadata(regsurf)
+    # check that the relative file is at case level and has a preprocessed folder
+    assert meta["file"]["relative_path"] == "share/preprocessed/maps/unknown.gri"
 
 
 def test_norwegian_letters_globalconfig(

tests/test_units/test_enum_classes.py

@@ -12,9 +12,9 @@ def test_fmu_context_validation() -> None:
     with pytest.raises(ValueError, match="Invalid FmuContext value='invalid_context'"):
         FmuContext("invalid_context")
 
-    assert FmuContext.list_valid_values() == [
+    assert set(FmuContext.list_valid_values()) == {
         "realization",
+        "iteration",
         "case",
-        "preprocessed",
         "non-fmu",
-    ]
+    }

tests/test_units/test_objectdataprovider_class.py

@@ -113,7 +113,7 @@ def _export_data_from_rms(rmssetup, rmsglobalconfig, regsurf):
         os.chdir(rmssetup)
         edata = dataio.ExportData(
             config=rmsglobalconfig,  # read from global config
-            fmu_context="preprocessed",
+            preprocessed=True,
             name="TopVolantis",
             content="depth",
             is_observation=True,

tests/test_units/test_prerealization_surfaces.py

@@ -4,7 +4,7 @@
 folder and classification, but there are various ways to to this:
 
 1) Have files in a folder without any metadata; cf fmu_context="case"
-2) Have files with pregenerated matadata in a folder; cf fmu_context="preprocessed"
+2) Have files with pregenerated matadata in a folder; cf preprocessed=True
 
 These objects are normally made as hook workflows before ERT has ran any forward jobs
 and are typically used to compare results.
@@ -67,7 +67,7 @@ def _export_data_from_rms(rmssetup, rmsglobalconfig, regsurf):
         os.chdir(rmssetup)
         edata = dataio.ExportData(
             config=rmsglobalconfig,  # read from global config
-            fmu_context="preprocessed",
+            preprocessed=True,
             name="TopVolantis",
             content="depth",
             is_observation=True,
@@ -183,7 +183,7 @@ def _export_data_from_rms(
         os.chdir(rmssetup)
         edata = dataio.ExportData(
             config=rmsglobalconfig,  # read from global config
-            fmu_context="preprocessed",
+            preprocessed=True,
             content="depth",
             parent=parent,
             timedata=[[20240802, "moni"], [20200909, "base"]],
@@ -245,7 +245,7 @@ def _export_data_from_rms(rmssetup, rmsglobalconfig, regsurf):
         os.chdir(rmssetup)
         edata = dataio.ExportData(
             config=rmsglobalconfig,  # read from global config
-            fmu_context="preprocessed",
+            preprocessed=True,
             name="preprocessedmap",
             content="depth",
             is_observation=True,
@@ -298,7 +298,7 @@ def test_preprocessed_with_abs_forcefolder_shall_fail(
     os.chdir(rmssetup)
     edata = dataio.ExportData(
         config=rmsglobalconfig,  # read from global config
-        fmu_context="preprocessed",
+        preprocessed=True,
         name="some",
         content="depth",
         is_observation=True,
@@ -318,7 +318,7 @@ def test_preprocessed_with_rel_forcefolder_ok(rmssetup, rmsglobalconfig, regsurf
     os.chdir(rmssetup)
     edata = dataio.ExportData(
         config=rmsglobalconfig,  # read from global config
-        fmu_context="preprocessed",
+        preprocessed=True,
         name="some",
         content="depth",
         is_observation=True,
@@ -351,7 +351,7 @@ def _export_data_from_rms(rmssetup, rmsglobalconfig, regsurf):
         os.chdir(rmssetup)
         edata = dataio.ExportData(
             config=rmsglobalconfig,
-            fmu_context="preprocessed",
+            preprocessed=True,
             name="preprocessedmap",
             content="depth",
             access_ssdl={"access_level": "restricted"},  # access != config