From 6fa7045ec9092ffa801ecd85d3276aebbbe02e43 Mon Sep 17 00:00:00 2001 From: Yifu Tao Date: Thu, 31 Oct 2024 14:20:46 +0000 Subject: [PATCH 1/6] fix: support if input is str not Path in eval_recon --- scripts/reconstruction_benchmark/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/reconstruction_benchmark/main.py b/scripts/reconstruction_benchmark/main.py index dfb6258..6519ded 100644 --- a/scripts/reconstruction_benchmark/main.py +++ b/scripts/reconstruction_benchmark/main.py @@ -217,9 +217,9 @@ def run_nerfstudio( self.evaluate_reconstruction(metric_cloud_gt_frame) def evaluate_reconstruction(self, input_cloud_path, results_dir=None): - assert input_cloud_path.exists(), f"Input cloud not found at {input_cloud_path}" + assert Path(input_cloud_path).exists(), f"Input cloud not found at {input_cloud_path}" assert Path(input_cloud_path).suffix == ".pcd", "Input cloud must be a pcd file" - assert self.gt_octree_path.exists(), f"Ground truth octree not found at {self.gt_octree_path}" + assert Path(self.gt_octree_path).exists(), f"Ground truth octree not found at {self.gt_octree_path}" recon_thresholds = [0.03, 0.05, 0.1, 0.2] results_dir = self.recon_benchmark_dir if results_dir is None else Path(results_dir) input_cloud = o3d.io.read_point_cloud(str(input_cloud_path)) From 0233336cae6066918ccf637a3837bbbede67e457 Mon Sep 17 00:00:00 2001 From: Yifu Tao Date: Fri, 8 Nov 2024 17:10:56 +0000 Subject: [PATCH 2/6] refactor: do not clip max dist for cloud eval - as done in IJRR submission --- oxford_spires_utils/eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oxford_spires_utils/eval.py b/oxford_spires_utils/eval.py index 86bdab2..5a91137 100644 --- a/oxford_spires_utils/eval.py +++ b/oxford_spires_utils/eval.py @@ -46,7 +46,7 @@ def get_recon_metrics( def get_recon_metrics_multi_thresholds( - input_cloud: np.ndarray, gt_cloud: np.ndarray, thresholds: list = [0.02, 0.05, 0.1], max_distance=2.0 + input_cloud: np.ndarray, gt_cloud: np.ndarray, thresholds: list = [0.02, 0.05, 0.1], max_distance=9999.0 ): assert isinstance(input_cloud, np.ndarray) and isinstance(gt_cloud, np.ndarray) assert input_cloud.shape[1] == 3 and gt_cloud.shape[1] == 3 From db4d4dbd65b92e977f2b2eb90ffb2bd3a0de72a5 Mon Sep 17 00:00:00 2001 From: Yifu Tao Date: Fri, 8 Nov 2024 17:32:07 +0000 Subject: [PATCH 3/6] update the README with loc and recon benchmarks --- README.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 6a8777b..c71b67b 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,26 @@ # Oxford Spires Dataset -This repository contains scripts that are used to evaluate Lidar/Visual SLAM on the Oxford Spires Dataset. +This repository contains scripts that are used to evaluate localisation, 3D reconstruction and radiance field methods on the [Oxford Spires Dataset](https://dynamic.robots.ox.ac.uk/datasets/oxford-spires/). -## Installation -### oxford_spires_utils (Python) -```bash -pip install -e . -``` +This is a pre-release of the software. The codebase will be refactored in the near future. + +### Localisation Benchmark +The localisation benchmark runs LiDAR SLAM methods ([Fast-LIO-SLAM](https://github.com/gisbi-kim/FAST_LIO_SLAM), [SC-LIO-SAM](https://github.com/gisbi-kim/SC-LIO-SAM)) and LiDAR Bundle Adjustment method ([HBA](https://github.com/hku-mars/HBA)). The resultant trajectory are evaluated against the ground truth trajectory using [evo](https://github.com/MichaelGrupp/evo). -### spires_cpp (C++ Pybind) -Install [octomap](https://github.com/OctoMap/octomap) and [PCL](https://github.com/PointCloudLibrary/pcl) to your system, then + +### Reconstruction Benchmark +The reconstruction benchmark runs Structure-from-Motion ([COLMAP](https://colmap.github.io/)), Multi-view Stereo ([OpenMVS](https://github.com/cdcseacave/openMVS)), radiance field methods ([Nerfstudio](https://github.com/nerfstudio-project/nerfstudio/tree/main/nerfstudio)'s Nerfacto and Splatfacto), and generates 3D point cloud reconstruction, which is evaluated against the TLS-captured ground truth 3D point cloud. + +Build the docker container and run ```bash -cd spires_cpp -pip install -e . +cd oxford_spires_dataset +docker compose -f .docker/docker-compose.yml run --build spires + +# inside the docker +python scripts/reconstruction_benchmark/main.py --config-file config/recon_benchmark.yaml ``` +### Novel-view Synthesis Benchmark +Currently, the NVS benchmakr is included in the reconstruction benchmark script, since it builds upon output from COLMAP. ## Contributing Please refer to Angular's guide for contributing(https://github.com/angular/angular/blob/22b96b96902e1a42ee8c5e807720424abad3082a/CONTRIBUTING.md). From 71f5b76046a74941965ff9d003e65c77d5d3756e Mon Sep 17 00:00:00 2001 From: Yifu Tao Date: Fri, 8 Nov 2024 17:33:05 +0000 Subject: [PATCH 4/6] update the README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c71b67b..14bb21d 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # Oxford Spires Dataset -This repository contains scripts that are used to evaluate localisation, 3D reconstruction and radiance field methods on the [Oxford Spires Dataset](https://dynamic.robots.ox.ac.uk/datasets/oxford-spires/). +This repository contains scripts that are used to evaluate localisation, 3D reconstruction and radiance field methods using the [Oxford Spires Dataset](https://dynamic.robots.ox.ac.uk/datasets/oxford-spires/). This is a pre-release of the software. The codebase will be refactored in the near future. -### Localisation Benchmark +## Localisation Benchmark The localisation benchmark runs LiDAR SLAM methods ([Fast-LIO-SLAM](https://github.com/gisbi-kim/FAST_LIO_SLAM), [SC-LIO-SAM](https://github.com/gisbi-kim/SC-LIO-SAM)) and LiDAR Bundle Adjustment method ([HBA](https://github.com/hku-mars/HBA)). The resultant trajectory are evaluated against the ground truth trajectory using [evo](https://github.com/MichaelGrupp/evo). -### Reconstruction Benchmark +## Reconstruction Benchmark The reconstruction benchmark runs Structure-from-Motion ([COLMAP](https://colmap.github.io/)), Multi-view Stereo ([OpenMVS](https://github.com/cdcseacave/openMVS)), radiance field methods ([Nerfstudio](https://github.com/nerfstudio-project/nerfstudio/tree/main/nerfstudio)'s Nerfacto and Splatfacto), and generates 3D point cloud reconstruction, which is evaluated against the TLS-captured ground truth 3D point cloud. Build the docker container and run @@ -19,7 +19,7 @@ docker compose -f .docker/docker-compose.yml run --build spires python scripts/reconstruction_benchmark/main.py --config-file config/recon_benchmark.yaml ``` -### Novel-view Synthesis Benchmark +## Novel-view Synthesis Benchmark Currently, the NVS benchmakr is included in the reconstruction benchmark script, since it builds upon output from COLMAP. ## Contributing From f39b4ab405b18fba66e043db3e95d6b5d6fac72f Mon Sep 17 00:00:00 2001 From: Yifu Tao Date: Fri, 8 Nov 2024 17:38:03 +0000 Subject: [PATCH 5/6] update README with loc methods --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 14bb21d..02570e3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,14 @@ This repository contains scripts that are used to evaluate localisation, 3D reco This is a pre-release of the software. The codebase will be refactored in the near future. ## Localisation Benchmark -The localisation benchmark runs LiDAR SLAM methods ([Fast-LIO-SLAM](https://github.com/gisbi-kim/FAST_LIO_SLAM), [SC-LIO-SAM](https://github.com/gisbi-kim/SC-LIO-SAM)) and LiDAR Bundle Adjustment method ([HBA](https://github.com/hku-mars/HBA)). The resultant trajectory are evaluated against the ground truth trajectory using [evo](https://github.com/MichaelGrupp/evo). +The localisation benchmark runs LiDAR SLAM methods ([Fast-LIO-SLAM](https://github.com/gisbi-kim/FAST_LIO_SLAM), [SC-LIO-SAM](https://github.com/gisbi-kim/SC-LIO-SAM), [ImMesh](https://github.com/ori-drs/ImMesh_hesai)) and LiDAR Bundle Adjustment method ([HBA](https://github.com/hku-mars/HBA)). The resultant trajectory are evaluated against the ground truth trajectory using [evo](https://github.com/MichaelGrupp/evo). + +```bash +python scripts/localisation_benchmark/colmap.py +python scripts/localisation_benchmark/fast_lio_slam.py +python scripts/localisation_benchmark/immesh.py +python scripts/localisation_benchmark/vilens_hba.py +``` ## Reconstruction Benchmark From 98489efebf99244c5ba0899f35af32bee21efc97 Mon Sep 17 00:00:00 2001 From: Yifu Tao Date: Fri, 8 Nov 2024 17:40:58 +0000 Subject: [PATCH 6/6] update README with localisation docker instructions --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 02570e3..a8f304a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,12 @@ This is a pre-release of the software. The codebase will be refactored in the ne ## Localisation Benchmark The localisation benchmark runs LiDAR SLAM methods ([Fast-LIO-SLAM](https://github.com/gisbi-kim/FAST_LIO_SLAM), [SC-LIO-SAM](https://github.com/gisbi-kim/SC-LIO-SAM), [ImMesh](https://github.com/ori-drs/ImMesh_hesai)) and LiDAR Bundle Adjustment method ([HBA](https://github.com/hku-mars/HBA)). The resultant trajectory are evaluated against the ground truth trajectory using [evo](https://github.com/MichaelGrupp/evo). +Build the docker container and run the methods: ```bash +cd oxford_spires_dataset +docker compose -f .docker_loc/docker-compose.yml run --build spires + +# in the docker python scripts/localisation_benchmark/colmap.py python scripts/localisation_benchmark/fast_lio_slam.py python scripts/localisation_benchmark/immesh.py @@ -17,7 +22,7 @@ python scripts/localisation_benchmark/vilens_hba.py ## Reconstruction Benchmark The reconstruction benchmark runs Structure-from-Motion ([COLMAP](https://colmap.github.io/)), Multi-view Stereo ([OpenMVS](https://github.com/cdcseacave/openMVS)), radiance field methods ([Nerfstudio](https://github.com/nerfstudio-project/nerfstudio/tree/main/nerfstudio)'s Nerfacto and Splatfacto), and generates 3D point cloud reconstruction, which is evaluated against the TLS-captured ground truth 3D point cloud. -Build the docker container and run +Build the docker container and run the methods: ```bash cd oxford_spires_dataset docker compose -f .docker/docker-compose.yml run --build spires