Update tests to match new datasets order

scripts/example_merge_mask_default.sh

@@ -1,6 +1,6 @@
 # For lauching Mask Hydro merged
 python -m lidro.main_merge_mask \
-io.input_dir=./data/mask_hydro/ \
+io.input_dir=./data/mask_hydro_merge/ \
 io.output_dir=./tmp/merge_mask_hydro/ \
 mask_generation.vector.min_water_area=150 \
 mask_generation.vector.buffer_positive=1 \

test/create_mask_hydro/pointcloud/test_io.py

@@ -9,7 +9,7 @@
 
 TMP_PATH = Path("./tmp/create_mask_hydro/pointcloud/io")
 
-LAS_FILE = "./data/pointcloud/LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST.las"
+LAS_FILE = "./data/other/pointcloud/LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST.las"
 
 
 def setup_module(module):

test/create_mask_hydro/pointcloud/test_read_las.py

@@ -9,7 +9,7 @@
 
 TMP_PATH = Path("./tmp/create_mask_hydro/pointcloud/io")
 
-LAS_FILE = "./data/pointcloud/LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST.las"
+LAS_FILE = "./data/other/pointcloud/LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST.las"
 
 
 def setup_module(module):

test/create_mask_hydro/rasters/test_create_mask_raster.py

@@ -14,7 +14,7 @@
 
 TMP_PATH = Path("./tmp/create_mask_hydro/rasters/create_mask_raster")
 
-LAS_FILE = "./data/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz"
+LAS_FILE = "./data/tile_0830_6291/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz"
 
 
 tile_size = 1000

test/create_mask_hydro/vectors/test_convert_to_vector.py

@@ -10,7 +10,7 @@
 
 TMP_PATH = Path("./tmp/create_mask_hydro/vectors/convert_to_vector")
 
-las_file = "./data/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz"
+las_file = "./data/tile_0830_6291/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz"
 output = "./tmp/create_mask_hydro/vectors/convert_to_vector/MaskHydro_Semis_2021_0830_6291_LA93_IGN69.GeoJSON"
 
 

test/create_virtual_point/pointcloud/test_add_virtual_points_to_pointcloud.py

@@ -11,8 +11,8 @@
 
 # Path to temporary directory for output files
 TMP_PATH = Path("./tmp/virtual_points/pointcloud/add_virtual_points_by_tiles")
-input_file = str("./data/point_virtual/virtual_points.laz")
-input_las = str("./data/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz")
+input_file = str("./data/tile_0830_6291/virtual_points/virtual_points.laz")
+input_las = str("./data/tile_0830_6291/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz")
 output_laz_file = str(TMP_PATH / "output_0830_6291_with_virtual_points.laz")
 geom = Polygon(
     [

test/create_virtual_point/pointcloud/test_auto_tiling_from_las.py

@@ -24,22 +24,22 @@ def setup_module():
 def mock_laz_files():
     # LAZ files
     return [
-        "./data/pointcloud/LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST.las",
-        "./data/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz",
+        "./data/other/pointcloud/LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST.las",
+        "./data/tile_0830_6291/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz",
     ]
 
 
 def test_extract_bounds_from_laz_file_not_found():
     """Test that the function raises a FileNotFoundError when the file does not exist."""
-    non_existing_file = "./data/pointcloud/non_existing_file.laz"
+    non_existing_file = "./data/other/pointcloud/non_existing_file.laz"
 
     with pytest.raises(FileNotFoundError, match=f"The file '{non_existing_file}' does not exist."):
         extract_bounds_from_laz(non_existing_file)
 
 
 def test_extract_bounds_from_real_laz_file():
     """Test if extract_bounds_from_laz returns a bounding box of 1km x 1km for the real file."""
-    laz_file = "./data/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz"
+    laz_file = "./data/tile_0830_6291/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz"
     bounds = extract_bounds_from_laz(laz_file)
 
     assert len(bounds) == 4, "Bounding box should return 4 values."

test/create_virtual_point/pointcloud/test_crop_las.py

@@ -9,7 +9,7 @@
 )
 
 TMP_PATH = Path("./tmp/create_virtual_point/pointcloud/crop_las")
-LAS_FILE = "./data/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz"
+LAS_FILE = "./data/tile_0830_6291/pointcloud/Semis_2021_0830_6291_LA93_IGN69.laz"
 
 
 def setup_module(module):

test/create_virtual_point/vectors/test_create_mask_hydro_buffer.py

@@ -12,7 +12,7 @@
 
 TMP_PATH = Path("./tmp/create_virtual_point/vectors/create_mask_hydro_buffer/")
 
-file_mask = "./data/merge_mask_hydro/dataset_1/MaskHydro_merge_with_multibranch_skeleton.geojson"
+file_mask = "./data/tile_0830_6291/mask_hydro_merge/MaskHydro_merge.geojson"
 output = Path("./tmp/create_virtual_point/vectors/create_mask_hydro_buffer/MaskHydro_merge_buffer.geojson")
 
 

test/create_virtual_point/vectors/test_extract_points_around_skeleton.py

@@ -11,9 +11,9 @@
 )
 
 TMP_PATH = Path("./tmp/create_virtual_point/vectors/extract_points_around_skeleton")
-INPUT_DIR = Path("./data/")
-MASK_HYDRO = "./data/merge_mask_hydro/dataset_1/MaskHydro_merge_with_multibranch_skeleton.geojson"
-POINTS_SKELETON = "./data/skeleton_hydro/dataset_1/points_along_skeleton.geojson"
+INPUT_DIR = Path("./data/tile_0830_6291")
+MASK_HYDRO = "./data/tile_0830_6291/mask_hydro_merge/MaskHydro_merge.geojson"
+POINTS_SKELETON = "./data/tile_0830_6291/skeleton/points_along_skeleton.geojson"
 OUTPUT_GEOJSON = TMP_PATH / "Semis_2021_0830_6291_LA93_IGN69_points_skeleton.geojson"
 
 
@@ -30,6 +30,7 @@ def test_extract_points_around_skeleton_default():
     k = 3  # Example k value
 
     # Mask Hydro with buffer
+
     mask_hydro_buffered = "MULTIPOLYGON (((829969.0856167737 6292032.553442742,\
         830452.9506643447 6292032.553442742, \
         830498.1716968281 6291675.307286125, \

test/create_virtual_point/vectors/test_merge_skeleton_by_mask.py

@@ -85,18 +85,18 @@ def test_combine_skeletons_raise(skeletons, hydro_masks):
     "skeleton, mask",
     [
         (  # simple mono_branch skeleton (real case)
-            Path("./data/skeleton_hydro/dataset_2/skeleton_hydro.geojson"),
-            Path("./data/merge_mask_hydro/dataset_2/MaskHydro_merge.geojson"),
+            Path("./data/OM_subset/skeleton/skeleton_hydro.geojson"),
+            Path("./data/OM_subset/mask_hydro_merge/MaskHydro_merge.geojson"),
         ),
         (  # simple mono_branch skeleton going over an island
-            Path("./data/skeleton_hydro/dataset_2/skeleton_hydro_over_island.geojson"),
-            Path("./data/merge_mask_hydro/dataset_2/MaskHydro_merge.geojson"),
+            Path("./data/OM_subset/skeleton//skeleton_hydro_over_island.geojson"),
+            Path("./data/OM_subset/mask_hydro_merge/MaskHydro_merge.geojson"),
         ),
-        (  # simple mono_branch skeleton
+        (  # mono_branch skeleton with mutliple LineString
             # original skeleton code generated skeleton_hydro_fail_with_small_gap.geojson which failed
             # newer output should pass
-            Path("./data/skeleton_hydro/dataset_1/skeleton_hydro_single_branch.geojson"),
-            Path("./data/merge_mask_hydro/dataset_1/MaskHydro_merge_with_multibranch_skeleton.geojson"),
+            Path("./data/tile_0830_6291/skeleton/skeleton_hydro.geojson"),
+            Path("./data/tile_0830_6291/mask_hydro_merge/MaskHydro_merge.geojson"),
         ),
     ],
 )
@@ -118,12 +118,12 @@ def test_merge_skeleton_by_mask_default(skeleton, mask):
     "skeleton, mask",
     [
         (  # simple mono_branch skeleton, which in fact has 2 parts that are disjoint of around 1 mm
-            Path("./data/skeleton_hydro/dataset_1/skeleton_hydro_failt_with_small_gap.geojson"),
-            Path("./data/merge_mask_hydro/dataset_1/MaskHydro_merge_with_multibranch_skeleton.geojson"),
+            Path("./data/tile_0830_6291/skeleton/skeleton_hydro_fail_with_small_gap.geojson"),
+            Path("./data/tile_0830_6291/mask_hydro_merge/MaskHydro_merge.geojson"),
         ),
         (  # multi_branch skeleton (should not happen)
-            Path("./data/skeleton_hydro/dataset_1/Skeleton_Hydro_multibranch.geojson"),
-            Path("./data/merge_mask_hydro/dataset_1/MaskHydro_merge_with_multibranch_skeleton.geojson"),
+            Path("./data/tile_0830_6291/skeleton/skeleton_hydro_fail_with_multibranch.geojson"),
+            Path("./data/tile_0830_6291/mask_hydro_merge/MaskHydro_merge.geojson"),
         ),
     ],
 )

test/create_virtual_point/vectors/test_points_along_skeleton.py

@@ -13,7 +13,7 @@
 
 TMP_PATH = Path("./tmp/create_virtual_point/vectors/points_along_skeleton")
 
-file = "./data/skeleton_hydro/dataset_2/skeleton_hydro.geojson"
+file = "./data/tile_0830_6291/skeleton/skeleton_hydro.geojson"
 output = Path("./tmp/create_virtual_point/vectors/points_along_skeleton/points.geojson")
 
 

test/merge_mask_hydro/vectors/test_check_rectify_geometry.py

@@ -8,7 +8,7 @@
 
 
 def test_apply_buffers_to_geometry_default():
-    input = "./data/merge_mask_hydro/dataset_1/MaskHydro_merge_with_multibranch_skeleton.geojson"
+    input = "./data/tile_0830_6291/mask_hydro_merge/MaskHydro_merge.geojson"
     # Load each GeoJSON file as GeoDataFrame
     geojson = gpd.read_file(input)
     geometry = geojson["geometry"].unary_union
@@ -17,7 +17,7 @@ def test_apply_buffers_to_geometry_default():
 
 
 def test_fix_topology_default():
-    input = "./data/merge_mask_hydro/dataset_1/MaskHydro_merge_with_multibranch_skeleton.geojson"
+    input = "./data/tile_0830_6291/mask_hydro_merge/MaskHydro_merge.geojson"
     # Load each GeoJSON file as GeoDataFrame
     geojson = gpd.read_file(input)
     check_geom = fix_topology(geojson)
@@ -34,7 +34,7 @@ def test_fix_topology_default():
 
 
 def test_fix_topology_error():
-    input_error = "./data/merge_mask_hydro/dataset_1/MaskHydro_merge_NoValid.geojson"
+    input_error = "./data/tile_0830_6291/mask_hydro_merge/MaskHydro_merge_NoValid.geojson"
     # Load each GeoJSON file as GeoDataFrame
     geojson = gpd.read_file(input_error)
     check_geom = fix_topology(geojson)

test/merge_mask_hydro/vectors/test_merge_vector.py

@@ -10,7 +10,8 @@
 
 TMP_PATH = Path("./tmp/merge_mask_hydro/vectors/merge_mask_hydro")
 
-input_folder = "./data/mask_hydro/"
+
+input_folder = "./data/tile_0830_6291/mask_hydro"
 output = Path("./tmp/merge_mask_hydro/vectors/merge_mask_hydro/MaskHydro_merge.geojson")
 
 

test/skeleton/test_branch.py

@@ -17,7 +17,7 @@
 
 sys.path.append("lidro/skeleton")
 
-BRANCH_TEST_1_PATH = "data/skeleton_hydro/test_files/90.geojson"
+BRANCH_TEST_1_PATH = "data/other/skeleton/90.geojson"
 CRS_FOR_TEST = 2154
 WATER_MIN_SIZE_TEST = 20
 CLIPPING_LENGTH = 50

test/skeleton/test_main_skeleton.py

@@ -1,26 +1,30 @@
 import os
+from test.skeleton.test_branch import read_branch
 
-from hydra import compose, initialize
-import pandas as pd
 import geopandas as gpd
-from dotenv import load_dotenv
+import pandas as pd
 import pytest
+from dotenv import load_dotenv
+from hydra import compose, initialize
 
-from lidro.skeleton.create_skeleton_lines import create_branches_list, create_branches_pair
-from lidro.skeleton.create_skeleton_lines import select_candidates, query_db_for_bridge_across_gap
 from lidro.skeleton.branch import Candidate
-from test.skeleton.test_branch import read_branch
+from lidro.skeleton.create_skeleton_lines import (
+    create_branches_list,
+    create_branches_pair,
+    query_db_for_bridge_across_gap,
+    select_candidates,
+)
 
 load_dotenv()
 
-IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true"   # check if tests runs under github_action
+IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true"  # check if tests runs under github_action
 
-DB_UNI_USER = os.getenv('DB_UNI_USER')
-DB_UNI_PASSWORD = os.getenv('DB_UNI_PASSWORD')
+DB_UNI_USER = os.getenv("DB_UNI_USER")
+DB_UNI_PASSWORD = os.getenv("DB_UNI_PASSWORD")
 CRS = 2154
-MAIN_SKELETON_TEST_1_1_PATH = "data/skeleton_hydro/test_files/40.geojson"
-MAIN_SKELETON_TEST_1_2_PATH = "data/skeleton_hydro/test_files/43.geojson"
-MAIN_SKELETON_TEST_1_3_PATH = "data/skeleton_hydro/test_files/44.geojson"
+MAIN_SKELETON_TEST_1_1_PATH = "data/other/skeleton/40.geojson"
+MAIN_SKELETON_TEST_1_2_PATH = "data/other/skeleton/43.geojson"
+MAIN_SKELETON_TEST_1_3_PATH = "data/other/skeleton/44.geojson"
 
 
 def test_main_skeleton_1():
@@ -44,14 +48,14 @@ def test_main_skeleton_1():
         candidate_1 = validated_candidates[0]
         candidate_2 = validated_candidates[1]
 
-        assert (candidate_1.squared_distance < 30)
-        assert (candidate_2.squared_distance < 75)
+        assert candidate_1.squared_distance < 30
+        assert candidate_2.squared_distance < 75
 
 
 # do that test only if we are not on github action, since github can't connect to BD UNI
 @pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="BD UNI not reachable from github action")
 def test_query_db_for_bridge_across_gap():
-    """Test : query_db_for_bridge_across_gap """
+    """Test : query_db_for_bridge_across_gap"""
     with initialize(version_base="1.2", config_path="../../configs"):
         config = compose(
             config_name="configs_lidro.yaml",

test/test_main_clip_virtual_point_by_tile.py

@@ -6,40 +6,39 @@
 
 from lidro.main_clip_virtual_point_by_tile import main
 
-INPUT_DIR = Path("data/pointcloud")
-INPUT_DIR_POINT_VIRTUAL = Path("data/point_virtual")
-OUTPUT_DIR = Path("tmp") / "clip_virtual_point_by_tile/main"
+DATA_DIR = Path("data/tile_0830_6291")
+INPUT_DIR = DATA_DIR / "pointcloud"
+INPUT_DIR_POINT_VIRTUAL = DATA_DIR / "virtual_points"
+OUTPUT_DIR = Path("tmp") / "main_clip_virtual_point_by_tile"
 
 
 def setup_module(module):
-    os.makedirs("tmp/clip_virtual_point_by_tile/main", exist_ok=True)
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
 
 
 def test_main_run_okay():
-    repo_dir = Path.cwd().parent
+    output_dir = OUTPUT_DIR / "main_run_okay"
     cmd = f"""python -m lidro.main_clip_virtual_point_by_tile \
-        io.input_dir="{repo_dir}/lidro/data/pointcloud/"\
-        io.input_dir_point_virtual="{repo_dir}/lidro/data/point_virtual/"\
-        io.output_dir="{repo_dir}/lidro/tmp/clip_virtual_point_by_tile/main/"
+        io.input_dir={INPUT_DIR}\
+        io.input_dir_point_virtual={INPUT_DIR_POINT_VIRTUAL}\
+        io.output_dir={output_dir}
         """
     sp.run(cmd, shell=True, check=True)
 
 
-def test_main_clip_virtual_point_by_tile():
-    input_dir = INPUT_DIR
-    input_dir_point_virtual = INPUT_DIR_POINT_VIRTUAL
-    output_dir = OUTPUT_DIR / "main_clip_virtual_point_by_tile"
+def test_main_creates_expected_output():
     input_filename = "Semis_2021_0830_6291_LA93_IGN69.laz"
     srid = 2154
+    output_dir = OUTPUT_DIR / "main_creates_expected_output"
 
     with initialize(version_base="1.2", config_path="../configs"):
         # config is relative to a module
         cfg = compose(
             config_name="configs_lidro",
             overrides=[
                 f"io.input_filename={input_filename}",
-                f"io.input_dir={input_dir}",
-                f"io.input_dir_point_virtual={input_dir_point_virtual}",
+                f"io.input_dir={INPUT_DIR}",
+                f"io.input_dir_point_virtual={INPUT_DIR_POINT_VIRTUAL}",
                 f"io.output_dir={output_dir}",
                 f"io.srid={srid}",
             ],

test/test_main_create_mask.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import subprocess as sp
 from pathlib import Path
 
@@ -7,19 +8,21 @@
 
 from lidro.main_create_mask import main
 
-INPUT_DIR = Path("data") / "pointcloud"
-OUTPUT_DIR = Path("tmp") / "create_mask_hydro/main"
+DATA_DIR = Path("data/tile_0830_6291")
+INPUT_DIR = DATA_DIR / "pointcloud"
+OUTPUT_DIR = Path("tmp") / "main_create_mask"
 
 
 def setup_module(module):
-    os.makedirs("tmp/create_mask_hydro/main", exist_ok=True)
+    if OUTPUT_DIR.is_dir():
+        shutil.rmtree(OUTPUT_DIR)
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
 
 
 def test_main_run_okay():
-    repo_dir = Path.cwd().parent
     cmd = f"""python -m lidro.main_create_mask \
-        io.input_dir="{repo_dir}/lidro/data/pointcloud/"\
-        io.output_dir="{repo_dir}/lidro/tmp/create_mask_hydro/main/"
+        io.input_dir={INPUT_DIR}\
+        io.output_dir={OUTPUT_DIR}
         """
     sp.run(cmd, shell=True, check=True)
 
@@ -43,11 +46,11 @@ def test_main_lidro_default():
             ],
         )
     main(cfg)
-    assert (Path(output_dir) / "MaskHydro_LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST.GeoJSON").is_file()
+    assert (Path(output_dir) / "MaskHydro_Semis_2021_0830_6291_LA93_IGN69.GeoJSON").is_file()
 
 
 def test_main_lidro_input_file():
-    input_dir = INPUT_DIR
+    input_dir = Path("data/other/pointcloud")
     output_dir = OUTPUT_DIR / "main_lidro_input_file"
     input_filename = "LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST.las"
     pixel_size = 1