Improve Dxm preprocessing scripts

.clang-format

@@ -0,0 +1,6 @@
+---
+Language:        Cpp
+BasedOnStyle:  LLVM
+ColumnLimit:     100
+...
+

.flake8

@@ -2,7 +2,7 @@
 max_line_length = 99
 show_source = True
 format = pylint
-extend-ignore = E203,E501
+extend-ignore = E501
 exclude =
     .git
     __pycache__

.github/workflows/cicd_docker.yml

@@ -0,0 +1,27 @@
+name: cicd_docker
+
+on:
+  # Run tests for non-draft pull request on main
+  pull_request:
+    branches:
+      - main
+
+env:
+  DOCKER_IMAGE_NAME: pdal_ign_plugin
+
+jobs:
+  build_docker_image_and_run_tests:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout branch
+      uses: actions/checkout@v3
+
+    - name: Build docker image
+      run: docker build -t ${{ env.DOCKER_IMAGE_NAME }}:test .
+
+    - name: Run tests in docker image
+      run: docker run ${{ env.DOCKER_IMAGE_NAME }}:test python -m pytest
+
+

.github/workflows/cicd_test.yml

@@ -1,7 +1,7 @@
 name: cicd_test
 
 on:
-  # Run each time some code are push on any branch
+  # Run each time some code is pushed on any branch
   push:
     branches:
       - '**'
@@ -25,8 +25,8 @@ jobs:
         activate-environment: pdal_ign_plugin
         environment-file: ./environment.yml
         auto-activate-base: true
-    
-    - name: compil_plugins
+
+    - name: compile_plugins
       run: source ./ci/build.sh
 
     - name: test

.gitignore

@@ -1,5 +1,7 @@
 xcode
+.vscode
+build
 install
-__pycache__
+*/__pycache__
 test/__pycache_
 test/.idea

Dockerfile

@@ -1,27 +1,33 @@
 FROM mambaorg/micromamba:bullseye-slim as build
-
+
 COPY environment_docker.yml /environment_docker.yml
- 
-USER root
-RUN micromamba env create -f /environment_docker.yml
+
+USER root
+RUN micromamba env create -f /environment_docker.yml
 SHELL ["micromamba", "run", "-n", "pdal_ign_plugin", "/bin/bash", "-c"]
 RUN apt-get update && apt-get install --no-install-recommends -y cmake make build-essential g++ && rm -rf /var/lib/apt/lists/*
- 
-COPY src src
+
+COPY src src
 COPY CMakeLists.txt CMakeLists.txt
-COPY macro macro
-
-RUN cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release
+RUN cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release
 RUN make -j4 install
-  
+
 FROM debian:bullseye-slim
- 
+
 COPY --from=build /opt/conda/envs/pdal_ign_plugin /opt/conda/envs/pdal_ign_plugin
-RUN mkdir -p /pdal_ign_plugin
+RUN mkdir -p /pdal_ign_plugin
 COPY --from=build /tmp/install/lib /pdal_ign_plugin/install/lib
-COPY --from=build /tmp/macro /macro 
 
-ENV PATH=$PATH:/opt/conda/envs/pdal_ign_plugin/bin/
-ENV PROJ_LIB=/opt/conda/envs/pdal_ign_plugin/share/proj/
+ENV PATH=$PATH:/opt/conda/envs/pdal_ign_plugin/bin/
+ENV PROJ_LIB=/opt/conda/envs/pdal_ign_plugin/share/proj/
 ENV PDAL_DRIVER_PATH=/pdal_ign_plugin/install/lib
 
+# Install python macro module
+COPY macro /pdal_ign_plugin/macro
+COPY pyproject.toml /pdal_ign_plugin/pyproject.toml
+WORKDIR /pdal_ign_plugin
+RUN pip install .
+
+# Add example scripts + test data (to be able to test inside the docker image)
+COPY scripts /pdal_ign_plugin/scripts
+COPY test /pdal_ign_plugin/test

README.md

@@ -2,68 +2,79 @@
 
 ## Compile
 
-You need to have conda ! 
+You need to have conda!
+
+Create the ign_pdal_tools conda environment using the `environment.yml` file
+to be able to run the compilation in this environment.
 
 ### linux/mac
 
 run ci/build.sh
 
-### Windows 
+### Windows
 
 one day, maybe...
 
 ## Architecture of the code
 
-The code is structured as :
+The code is structured as:
 
 ```
 ├── src
-│   ├── plugins forlder
-│   │   ├── plufinFilter.cpp
-│   │   ├── plufinFilter.h
+│   ├── plugin folder
+│   │   ├── pluginFilter.cpp
+│   │   ├── pluginFilter.h
 │   │   ├── CMakeLists.txt
 ├── doc
 │   ├── pluginFilter.md
 ├── ci
+├── macro  # Python module with ready-to-use filters combinations
+│   ├── __init__.py
+│   ├── macro.py
+│   └── version.py
+├── scripts
+│   ├── *.py  # Example scripts to use the plugin filters + the filters combinations contained in `macro`
 ├── test
 ├── CMakeLists.txt
 ├── environment*.yml
-├── Dockerfile 
-├── .github 
+├── Dockerfile
+├── pyproject.toml  # Setup file to install the `macro` python module with pip
+├── .github
 └── .gitignore
 ```
 
 ## Run the tests
 
-Each plugin should have his own test. To run test :
+Each plugin should have his own test. To run all tests:
 
 ```
 python -m pytest -s
 ```
 
 ## List of Filters
 
-[grid decimation](./doc/grid_decimation.md)
+[grid decimation](./doc/grid_decimation.md) [Deprecated: use the gridDecimation filter from the pdal repository]
 
 [radius assign](./doc/radius_assign.md)
 
 ## Adding a filter
 
-In order to add a filter, you have to add a new folder in the src directory : 
+In order to add a filter, you have to add a new folder in the src directory :
 
 ```
 ├── src
 │   ├── filter_my_new_PI
 │   │   ├── my_new_PI_Filter.cpp
 │   │   ├── my_new_PI_Filter.h
-│   │   ├── CMakeLisits.txt
+│   │   ├── CMakeLists.txt
 ```
 
 The name of the folder informs of the plugIN nature (reader, writer, filter).
 
-The code should respect the documentation purpose by pdal : [build a pdal plugin](https://pdal.io/en/2.6.0/development/plugins.html). Be careful to change if the plugIn is a reader, a writer or a filter. 
+The code should respect the documentation proposed by pdal: [build a pdal plugin](https://pdal.io/en/2.6.0/development/plugins.html).
+Be careful to change if the plugIn is a reader, a writer or a filter.
 
-The CMakeList should contains : 
+The CMakeList should contain:
 
 ```
 file( GLOB_RECURSE GD_SRCS ${CMAKE_SOURCE_DIR} *)
@@ -78,15 +89,44 @@ PDAL_CREATE_PLUGIN(
 install(TARGETS pdal_plugin_filter_my_new_PI)
 ```
 
-You should complet the principal CMakeList by adding the new plugIN :
-
+You should complete the main CMakeList by adding the new plugIN:
 ```
 add_subdirectory(src/filter_my_new_PI)
 ```
 
-Each plugIN has his own md file in the doc directory, structured as the [model](./doc/_doc_model_plugIN.md). 
+Each plugIN has his own md file in the doc directory, structured as the [model](./doc/_doc_model_plugIN.md).
+
+Don't forget to update [the list](#list-of-filters) with a link to the documentation.
+
+## `macro` python module usage
+
+The `macro` python module is installed in the project docker image so that it can be imported from anywhere in the
+docker image.
+
+
+### Syntax to use it in a python script
+
+```python
+from macro import macro
+
+marco.my_macro(...)
+```
+
+See the `scripts` folder for example usages of this module.
 
-D'ont forget to update [the list](#list-of-filters) with a link with the documentation.
+### Usage from outside the docker image:
 
+If you have a python script on your computer, you can mount its containing folder as a volume in order to
+run it in the docker image.
+
+Example:
+
+```bash
+docker run \
+    -v /my/data/folder:/data \
+    -v  /my/output/folder:/output \
+    -v /my/script/folder:/scripts \
+    pdal_ign_plugin \
+    python /scripts/my_script.py --input /data/my_data_file.las -o /output/my_output.las
+```
 
-

ci/build.sh

@@ -1,7 +1,9 @@
 #!/bin/sh
 
+set -e
+
 FILE=~/anaconda3/etc/profile.d/conda.sh
-if [ -e ~/anaconda3/etc/profile.d/conda.sh ] 
+if [ -e ~/anaconda3/etc/profile.d/conda.sh ]
 then
     source ~/anaconda3/etc/profile.d/conda.sh
 elif [ -e ~/miniconda3/etc/profile.d/conda.sh ]
@@ -10,8 +12,8 @@ then
 elif [ -e /usr/share/miniconda/etc/profile.d/conda.sh ]
 then
     source /usr/share/miniconda/etc/profile.d/conda.sh
-elif [ -e ~/miniforge3/etc/profile.d/conda.sh ] 
-then    
+elif [ -e ~/miniforge3/etc/profile.d/conda.sh ]
+then
      source ~/miniforge3/etc/profile.d/conda.sh
 elif [[ -z "${CONDASH}" ]]; then
     echo ERROR: Failed to load conda.sh : ~/anaconda3/etc/profile.d/conda.sh or ~/miniforge3/etc/profile.d/conda.sh or env CONDASH
@@ -28,10 +30,10 @@ echo conda is $CONDA_PREFIX
 
 mkdir build
 cd build
-cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release ../ 
+cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release ../
 make install
 
 conda deactivate
 
 cd ..
-rm -rf build 
+rm -rf build

environment.yml

@@ -13,7 +13,7 @@ dependencies:
   - isort # import sorting
   - flake8 # code analysis
   - pytest
-# --------- pip & pip librairies --------- #
+# --------- pip & pip libraries --------- #
   - pip
   - pip:
     - ign-pdal-tools

environment_docker.yml

@@ -6,8 +6,10 @@ dependencies:
   - pdal
   - python-pdal
   - gdal
-# --------- pip & pip librairies --------- #
+  - pytest
+
+  # --------- pip & pip libraries --------- #
   - pip
   - pip:
     - ign-pdal-tools
-
+