Add libres test data to semeio

This copies test data from libres into semeio, this should not
be a permanent solution, and should be deleted as soon as there is
an api to set scaling factors on observations. For the time being
that logic is contained in semeio, and as such the integration
tests are quite involved.

.pre-commit-config.yaml

@@ -24,3 +24,4 @@ repos:
       language: system
       types: [python]
       entry: "python -m pylint"
+exclude: tests/legacy_test_data/

.travis.yml

@@ -40,7 +40,6 @@ install:
     - pushd libres
     - pip install -r requirements.txt --prefix=$INSTALL_DIR
     - source .libecl_version
-    - export LIBRES_TEST_DATA_DIR="$(pwd)/test-data"
     - popd
     - git clone --branch $LIBECL_VERSION --depth 1 https://github.com/equinor/libecl
     - bash ert/.build_install.sh libecl

tests/conftest.py

@@ -1,5 +1,11 @@
 import os
 import pytest
+from tests import legacy_test_data
+
+
+@pytest.fixture()
+def test_data_root():
+    yield legacy_test_data.__path__[0]
 
 
 @pytest.fixture()

tests/jobs/correlated_observations_scaling/conftest.py

@@ -17,22 +17,14 @@ def pytest_runtest_setup(item):
 
 
 @pytest.fixture()
-def setup_ert(tmpdir):
+def setup_ert(tmpdir, test_data_root):
     cwd = os.getcwd()
     tmpdir.chdir()
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
 
     res_config = ResConfig("snake_oil.ert")
     yield res_config
 
     os.chdir(cwd)
-
-
-@pytest.fixture()
-def setup_tmpdir(tmpdir):
-    cwd = os.getcwd()
-    tmpdir.chdir()
-    yield
-    os.chdir(cwd)

tests/jobs/correlated_observations_scaling/test_integration.py

@@ -25,7 +25,6 @@ def assert_obs_vector(vector, val_1, index_list=None, val_2=None):
                 assert node.getStdScaling(index) == val_1
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_ert")
 def test_old_enkf_scaling_job(setup_ert):
     res_config = setup_ert
@@ -42,7 +41,6 @@ def test_old_enkf_scaling_job(setup_ert):
     assert_obs_vector(obs_vector, np.sqrt(4.0 / 2.0))
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_ert")
 def test_installed_python_version_of_enkf_scaling_job(setup_ert, monkeypatch):
 
@@ -89,12 +87,11 @@ def test_installed_python_version_of_enkf_scaling_job(setup_ert, monkeypatch):
     )
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_compare_different_jobs():
+def test_compare_different_jobs(test_data_root):
     cos_config = {"CALCULATE_KEYS": {"keys": [{"key": "WPR_DIFF_1"}]}}
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -119,15 +116,14 @@ def test_compare_different_jobs():
     assert_obs_vector(obs_vector, np.sqrt(4 / 2))
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_main_entry_point_gen_data():
+def test_main_entry_point_gen_data(test_data_root):
     cos_config = {
         "CALCULATE_KEYS": {"keys": [{"key": "WPR_DIFF_1"}]},
         "UPDATE_KEYS": {"keys": [{"key": "WPR_DIFF_1", "index": [400, 800]}]},
     }
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -178,14 +174,13 @@ def test_main_entry_point_gen_data():
         assert reported_scalefactor == pytest.approx(1.224744871391589, 0.1)
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_main_entry_point_summary_data_calc():
+def test_main_entry_point_summary_data_calc(test_data_root):
     cos_config = {
         "CALCULATE_KEYS": {"keys": [{"key": "WOPR_OP1_108"}, {"key": "WOPR_OP1_144"}]}
     }
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -206,7 +201,7 @@ def test_main_entry_point_summary_data_calc():
         assert node.getStdScaling(index) == np.sqrt(1.0)
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
+@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no equinor libres test-data")
 @pytest.mark.equinor_test
 @pytest.mark.usefixtures("setup_tmpdir")
 def test_main_entry_point_summary_data_update():
@@ -241,7 +236,7 @@ def test_main_entry_point_summary_data_update():
         assert node.getStdScaling(index) == 1.0
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
+@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no equinor libres test-data")
 @pytest.mark.equinor_test
 @pytest.mark.usefixtures("setup_tmpdir")
 def test_main_entry_point_block_data_calc():
@@ -267,7 +262,7 @@ def test_main_entry_point_block_data_calc():
         assert node.getStdScaling(index) == 2.0
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
+@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no equinor libres test-data")
 @pytest.mark.equinor_test
 @pytest.mark.usefixtures("setup_tmpdir")
 def test_main_entry_point_block_and_summary_data_calc():

tests/jobs/correlated_observations_scaling/unit/test_obs_util.py

@@ -13,7 +13,6 @@
     find_and_expand_wildcards,
     keys_with_data,
 )
-from tests.jobs.correlated_observations_scaling.conftest import TEST_DATA_DIR
 
 
 @pytest.mark.parametrize(
@@ -129,7 +128,6 @@ def test_failed_wildcard_expansion(config_dict, obs_list, expected_fails, err_ms
             pytest.fail("unexpectedly raised with config: {}".format(config_dict))
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_ert")
 def test_create_observation_vectors(setup_ert):
 
@@ -155,16 +153,15 @@ def test_create_observation_vectors(setup_ert):
     assert "SNAKE_OIL_WPR_DIFF" not in keys
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_add_observation_vectors():
+def test_add_observation_vectors(test_data_root):
 
     valid_config_data = {"UPDATE_KEYS": {"keys": [{"key": "WOPR_OP1_108"}]}}
 
     schema = job_config._CORRELATED_OBSERVATIONS_SCHEMA
     config = configsuite.ConfigSuite(valid_config_data, schema, deduce_required=True)
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil_field")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -183,13 +180,12 @@ def test_add_observation_vectors():
     assert "WOPR_OP1_144" not in keys
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_validate_failed_realizations():
+def test_validate_failed_realizations(test_data_root):
     """
     Config has several failed realisations
     """
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "custom_kw")
+    test_data_dir = os.path.join(test_data_root, "failed_runs_in_storage")
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
 
@@ -206,13 +202,12 @@ def test_validate_failed_realizations():
     assert result == ["GEN_PERLIN_1"]
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_validate_no_realizations():
+def test_validate_no_realizations(test_data_root):
     """
     Ensamble has not run
     """
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "poly_normal")
+    test_data_dir = os.path.join(test_data_root, "poly_normal")
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
 

tests/jobs/misfit_preprocessor/test_integration.py

@@ -10,14 +10,12 @@
 from semeio.workflows.correlated_observations_scaling.exceptions import (
     EmptyDatasetException,
 )
-from tests.jobs.correlated_observations_scaling.conftest import TEST_DATA_DIR
 
 from unittest.mock import Mock
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_misfit_preprocessor_main_entry_point_gen_data(monkeypatch):
+def test_misfit_preprocessor_main_entry_point_gen_data(monkeypatch, test_data_root):
     run_mock = Mock()
     scal_job = Mock(return_value=Mock(run=run_mock))
     monkeypatch.setattr(
@@ -26,7 +24,7 @@ def test_misfit_preprocessor_main_entry_point_gen_data(monkeypatch):
         scal_job,
     )
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -49,9 +47,8 @@ def test_misfit_preprocessor_main_entry_point_gen_data(monkeypatch):
     assert len(list(run_mock.call_args)[0][0]) == 47, "wrong number of clusters"
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_misfit_preprocessor_passing_scaling_parameters(monkeypatch):
+def test_misfit_preprocessor_passing_scaling_parameters(monkeypatch, test_data_root):
     run_mock = Mock()
     scal_job = Mock(return_value=Mock(run=run_mock))
     monkeypatch.setattr(
@@ -60,7 +57,7 @@ def test_misfit_preprocessor_passing_scaling_parameters(monkeypatch):
         scal_job,
     )
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -79,9 +76,8 @@ def test_misfit_preprocessor_passing_scaling_parameters(monkeypatch):
         assert 0.5 == scaling_config["CALCULATE_KEYS"]["threshold"]
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_misfit_preprocessor_main_entry_point_no_config(monkeypatch):
+def test_misfit_preprocessor_main_entry_point_no_config(monkeypatch, test_data_root):
     run_mock = Mock()
     scal_job = Mock(return_value=Mock(run=run_mock))
     monkeypatch.setattr(
@@ -90,7 +86,7 @@ def test_misfit_preprocessor_main_entry_point_no_config(monkeypatch):
         scal_job,
     )
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -103,10 +99,9 @@ def test_misfit_preprocessor_main_entry_point_no_config(monkeypatch):
     assert len(run_mock.call_args[0][0]) > 1  # pylint: disable=unsubscriptable-object
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_misfit_preprocessor_with_scaling():
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+def test_misfit_preprocessor_with_scaling(test_data_root):
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -127,9 +122,10 @@ def test_misfit_preprocessor_with_scaling():
         assert obs.getNode(index).getStdScaling() == 2.8284271247461903
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_misfit_preprocessor_skip_clusters_yielding_empty_data_matrixes(monkeypatch):
+def test_misfit_preprocessor_skip_clusters_yielding_empty_data_matrixes(
+    monkeypatch, test_data_root
+):
     def raising_scaling_job(data):
         if data == {"CALCULATE_KEYS": {"keys": [{"index": [88, 89], "key": "FOPR"}]}}:
             raise EmptyDatasetException("foo")
@@ -139,7 +135,7 @@ def raising_scaling_job(data):
         misfit_preprocessor, "CorrelatedObservationsScalingJob", scaling_mock
     )
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -160,10 +156,9 @@ def raising_scaling_job(data):
         pytest.fail("EmptyDatasetException was not handled by misfit preprocessor")
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_misfit_preprocessor_invalid_config():
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+def test_misfit_preprocessor_invalid_config(test_data_root):
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))

tests/jobs/spearman_correlation_job/test_integration.py

@@ -9,19 +9,17 @@
 from semeio.workflows.correlated_observations_scaling.exceptions import (
     EmptyDatasetException,
 )
-from tests.jobs.correlated_observations_scaling.conftest import TEST_DATA_DIR
 
 from unittest.mock import Mock
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_main_entry_point_gen_data(monkeypatch):
+def test_main_entry_point_gen_data(monkeypatch, test_data_root):
     run_mock = Mock()
     scal_job = Mock(return_value=Mock(run=run_mock))
     monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scal_job)
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -50,10 +48,9 @@ def test_main_entry_point_gen_data(monkeypatch):
         assert len(clusters.keys()) == 47
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_scaling():
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+def test_scaling(test_data_root):
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))
@@ -69,17 +66,16 @@ def test_scaling():
         assert obs.getNode(index).getStdScaling() == 2.8284271247461903
 
 
-@pytest.mark.skipif(TEST_DATA_DIR is None, reason="no libres test-data")
 @pytest.mark.usefixtures("setup_tmpdir")
-def test_skip_clusters_yielding_empty_data_matrixes(monkeypatch):
+def test_skip_clusters_yielding_empty_data_matrixes(monkeypatch, test_data_root):
     def raising_scaling_job(data):
         if data == {"CALCULATE_KEYS": {"keys": [{"index": [88, 89], "key": "FOPR"}]}}:
             raise EmptyDatasetException("foo")
 
     scaling_mock = Mock(return_value=Mock(**{"run.side_effect": raising_scaling_job}))
     monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scaling_mock)
 
-    test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
+    test_data_dir = os.path.join(test_data_root, "snake_oil")
 
     shutil.copytree(test_data_dir, "test_data")
     os.chdir(os.path.join("test_data"))

tests/legacy_test_data/failed_runs_in_storage/Observations/observations

@@ -0,0 +1,17 @@
+GENERAL_OBSERVATION GEN_PERLIN_1 {
+    DATA = PERLIN;
+    RESTART = 1;
+    OBS_FILE =  perlin_obs_1.txt;
+};
+
+GENERAL_OBSERVATION GEN_PERLIN_2 {
+    DATA = PERLIN;
+    RESTART = 2;
+    OBS_FILE =  perlin_obs_2.txt;
+};
+
+GENERAL_OBSERVATION GEN_PERLIN_3 {
+    DATA = PERLIN;
+    RESTART = 3;
+    OBS_FILE =  perlin_obs_3.txt;
+};