refactor: update Django to 4.2 LTS

.github/workflows/build-and-publish.yml

@@ -16,26 +16,26 @@ jobs:
       with:
         submodules: 'recursive'
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v3
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v3
     - name: Login to DockerHub
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v3 
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
     - name: Docker meta
       id: docker_meta
-      uses: crazy-max/ghaction-docker-meta@v1
+      uses: docker/metadata-action@v5
       with:
         images: opendronemap/webodm_webapp
-        tag-semver: |
-          {{version}}
+        tags: |
+          type=semver,pattern={{version}}
     - name: Build and push Docker image
       id: docker_build
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v6
       with:
-        file: ./Dockerfile
+        context: .
         platforms: linux/amd64,linux/arm64
         push: true
         no-cache: true

.gitignore

@@ -95,6 +95,7 @@ node_modules/
 webpack-stats.json
 pip-selfcheck.json
 .idea/
+.vscode/
 package-lock.json
 .cronenv
 .initialized

Dockerfile

@@ -1,43 +1,67 @@
-FROM ubuntu:21.04
+FROM ubuntu:22.04
 MAINTAINER Piero Toffanin <[email protected]>
 
 ARG TEST_BUILD
 ARG DEBIAN_FRONTEND=noninteractive
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONPATH $PYTHONPATH:/webodm
-ENV PROJ_LIB=/usr/share/proj
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=$PYTHONPATH:/webodm
+ENV NODE_MAJOR=20
+ENV PYTHON_MAJOR=3.10
+ENV GDAL_VERSION=3.8.5
+ENV LD_LIBRARY_PATH=/usr/local/lib
 
 # Prepare directory
 ADD . /webodm/
 WORKDIR /webodm
 
-# Use old-releases for 21.04
-RUN printf "deb http://old-releases.ubuntu.com/ubuntu/ hirsute main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-backports main restricted universe multiverse" > /etc/apt/sources.list
+RUN apt-get -o Acquire::Retries=3 -qq update > /dev/null && \
+    apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends wget curl git g++ clang make cmake postgresql-client > /dev/null && \
 
-# Install Node.js using new Node install method
-RUN apt-get -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends wget curl && \
-    apt-get -o Acquire::Retries=3 install -y ca-certificates gnupg && \
+    # Install PDAL, letsencrypt, psql, cron
+    apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends binutils pdal certbot gettext tzdata libproj-dev libpq-dev > /dev/null && \
+
+    # Install Python in target version
+    apt-get -qq autoremove -y python3 > /dev/null && \
+    apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends python$PYTHON_MAJOR-dev python$PYTHON_MAJOR-full > /dev/null && \
+    ln -s /usr/bin/python$PYTHON_MAJOR /usr/bin/python && \
+    curl https://bootstrap.pypa.io/get-pip.py | python && \
+
+    echo $(pip -V) && \
+    echo $(python -V) && \
+
+    # Build GDAL from source
+    wget --no-check-certificate -q https://github.com/OSGeo/gdal/releases/download/v$GDAL_VERSION/gdal-$GDAL_VERSION.tar.gz && \
+    tar -xzf gdal-$GDAL_VERSION.tar.gz && \
+    cd gdal-$GDAL_VERSION && mkdir build && cd build && \
+    cmake .. > /dev/null && cmake --build . -j$(nproc) --target install > /dev/null && \
+    cd / && rm -rf gdal-$GDAL_VERSION gdal-$GDAL_VERSION.tar.gz && \
+
+    # Install pip reqs
+    cd /webodm && \
+    pip install --quiet -U pip && \
+    pip install -r requirements.txt "boto3==1.34.145" > /dev/null && \
+
+    # Install Node.js using new Node install method
+    apt-get -o Acquire::Retries=3 -qq install -y ca-certificates gnupg > /dev/null && \
     mkdir -p /etc/apt/keyrings && \
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
-    NODE_MAJOR=20 && \
     echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-    apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y nodejs && \
-    # Install Python3, GDAL, PDAL, nginx, letsencrypt, psql
-    apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot gettext-base cron postgresql-client-13 gettext tzdata && \
-    update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2 && \
-    # Install pip reqs
-    pip install pip==24.0 && pip install -r requirements.txt "boto3==1.14.14" && \
+    apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y nodejs > /dev/null && \
+
     # Setup cron
+    apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends nginx cron > /dev/null && \
     ln -s /webodm/nginx/crontab /var/spool/cron/crontabs/root && chmod 0644 /webodm/nginx/crontab && service cron start && chmod +x /webodm/nginx/letsencrypt-autogen.sh && \
     /webodm/nodeodm/setup.sh && /webodm/nodeodm/cleanup.sh && cd /webodm && \
-    npm install --quiet -g [email protected] && npm install --quiet -g [email protected] && npm install --quiet && webpack --mode production && \
+    npm install --quiet -g [email protected] > /dev/null && npm install --quiet -g [email protected] > /dev/null && npm install --quiet > /dev/null && webpack --mode production > /dev/null && \
     echo "UTC" > /etc/timezone && \
     python manage.py collectstatic --noinput && \
     python manage.py rebuildplugins && \
     python manage.py translate build --safe && \
+
     # Cleanup
-    apt-get remove -y g++ python3-dev libpq-dev && apt-get autoremove -y && \
+    apt-get remove -y g++ python2 && apt-get autoremove -y && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    rm /webodm/webodm/secret_key.py
+    rm /webodm/webodm/secret_key.py && \
+    mkdir -p /webodm/app/media/tmp
 
 VOLUME /webodm/app/media

app/admin.py

@@ -3,11 +3,10 @@
 import zipfile
 import shutil
 
-from django.conf.urls import url
 from django.contrib import admin
 from django.contrib import messages
 from django.http import HttpResponseRedirect
-from django.urls import reverse
+from django.urls import reverse, path, re_path
 from django.utils.html import format_html
 from guardian.admin import GuardedModelAdmin
 from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
@@ -129,23 +128,23 @@ def author(self, obj):
     def get_urls(self):
         urls = super().get_urls()
         custom_urls = [
-            url(
-                r'^(?P<plugin_name>.+)/enable/$',
+            re_path(
+                '(?P<plugin_name>.+)/enable/',
                 self.admin_site.admin_view(self.plugin_enable),
                 name='plugin-enable',
             ),
-            url(
-                r'^(?P<plugin_name>.+)/disable/$',
+            re_path(
+                '(?P<plugin_name>.+)/disable/',
                 self.admin_site.admin_view(self.plugin_disable),
                 name='plugin-disable',
             ),
-            url(
-                r'^(?P<plugin_name>.+)/delete/$',
+            re_path(
+                '(?P<plugin_name>.+)/delete/',
                 self.admin_site.admin_view(self.plugin_delete),
                 name='plugin-delete',
             ),
-            url(
-                r'^actions/upload/$',
+            re_path(
+                'actions/upload/',
                 self.admin_site.admin_view(self.plugin_upload),
                 name='plugin-upload',
             ),

app/api/authentication.py

@@ -1,6 +1,8 @@
-from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication
-
-
-class JSONWebTokenAuthenticationQS(BaseJSONWebTokenAuthentication):
-    def get_jwt_value(self, request):
-         return request.query_params.get('jwt')
+# from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication
+#
+# jwt_decode_handler = api_settings.JWT_DECODE_HANDLER
+# jwt_get_username_from_payload_handler = api_settings.JWT_PAYLOAD_GET_USERNAME_HANDLER
+#
+# class JSONWebTokenAuthenticationQS(BaseJSONWebTokenAuthentication):
+#     def get_jwt_value(self, request):
+#          return request.query_params.get('jwt')

app/api/formulas.py

@@ -120,6 +120,59 @@
         'help': _('Temperature in Centikelvin degrees.')
     },
 
+
+    # new vegetation index
+
+    'SIPI': {
+        'expr': '(N - B) / (N - R)',
+        'help': _('Structure-Insensitive Pigment Index.')
+    },
+    'mNDVI': {
+        'expr': '(N - Re) / (N + Re - 2 * B)',
+        'help': _('Modified Normalized Difference Vegetation Index.')
+    },
+    'RENDVI': {
+        'expr': '(N - Re) / (N + Re)',
+        'help': _('Ratio Enhanced Normalized Difference Vegetation Index.')
+    },
+    'MTVI_1': {
+        'expr': '1.2 * (1.2 * (N - G) - 2.5 * (R - G))',
+        'help': _('Modified Transformed Vegetation Index 1.')
+    },
+    'MTVI_2': {
+        'expr': '1.5 * (1.2 * (N - G) - 2.5(R - G)) / (((2 * N + 1) ** 2 - (6 * N - 5 * (R0 ** 0.5)) - 0.5) ** 0.5)',
+        'help': _('Modified Transformed Vegetation Index 2.')
+    },
+    'GI': {
+        'expr': 'G / R',
+        'help': _('Greenness Index.'),
+    },
+    'TVI': {
+        'expr': '0.5 * (120 * (N - G) - 200 * (R - G))',
+        'help': _('Transparency Vegetation Index.')
+    },
+    'MCARI_1': {
+        'expr': '1.2 * (2.5 * (N - R) - 1.3 * (N - G))',
+        'help': _('Modified Chlorophyll Absorption Reflectance Index 1.')
+    },
+    'CI': {
+        'expr': 'N / G - 1',
+        'help': _('Chlorophyll Index.')
+    },
+    'SR': {
+        'expr': 'N / R',
+        'help': _('Simple Ratio.')
+    },
+    'mSR_2': {
+        'expr': '(N - Re) / (N + Re)',
+        'help': _('Modified Simple Ratio.')
+    },
+    'IPVI': {
+        'expr': 'N / (N + R)',
+        'help': _('Improved Perpendicular Vegetation Index.')
+    },
+
+
     # more?
 
     '_TESTRB': {

app/api/tiler.py

@@ -11,12 +11,11 @@
 from rio_tiler.errors import TileOutsideBounds
 from rio_tiler.utils import has_alpha_band, \
     non_alpha_indexes, render, create_cutline
-from rio_tiler.utils import _stats as raster_stats
-from rio_tiler.models import ImageStatistics, ImageData
-from rio_tiler.models import Metadata as RioMetadata
+from rio_tiler.utils import get_array_statistics
+from rio_tiler.models import BandStatistics
 from rio_tiler.profiles import img_profiles
 from rio_tiler.colormap import cmap as colormap, apply_cmap
-from rio_tiler.io import COGReader
+from rio_tiler.io import Reader
 from rio_tiler.errors import InvalidColorMapName, AlphaBandWarning
 import numpy as np
 from .custom_colormaps_helper import custom_colormaps
@@ -97,7 +96,7 @@ def get(self, request, pk=None, project_pk=None, tile_type=""):
         if not os.path.isfile(raster_path):
             raise exceptions.NotFound()
 
-        with COGReader(raster_path) as src:
+        with Reader(raster_path) as src:
             minzoom, maxzoom = get_zoom_safe(src)
 
         return Response({
@@ -164,7 +163,7 @@ def get(self, request, pk=None, project_pk=None, tile_type=""):
         if not os.path.isfile(raster_path):
             raise exceptions.NotFound()
         try:
-            with COGReader(raster_path) as src:
+            with Reader(raster_path) as src:
                 band_count = src.dataset.meta['count']
                 if boundaries_feature is not None:
                     boundaries_cutline = create_cutline(src.dataset, boundaries_feature, CRS.from_string('EPSG:4326'))
@@ -187,18 +186,20 @@ def get(self, request, pk=None, project_pk=None, tile_type=""):
                     data = np.ma.array(data)
                     data.mask = mask == 0
                     stats = {
-                        str(b + 1): raster_stats(data[b], percentiles=(pmin, pmax), bins=255, range=hrange)
+                        str(b + 1): get_array_statistics(data[b], percentiles=(pmin, pmax), bins=255, range=hrange)
                         for b in range(data.shape[0])
                     }
-                    stats = {b: ImageStatistics(**s) for b, s in stats.items()}
-                    metadata = RioMetadata(statistics=stats, **src.info().dict())
+                    stats = {b: BandStatistics(**s[0]) for b, s in stats.items()}
                 else:
                     if (boundaries_cutline is not None) and (boundaries_bbox is not None):
-                        metadata = src.metadata(pmin=pmin, pmax=pmax, hist_options=histogram_options, nodata=nodata
-                                                , bounds=boundaries_bbox, vrt_options={'cutline': boundaries_cutline})
+                        stats = src.statistics(percentiles=(pmin, pmax), hist_options=histogram_options, nodata=nodata,
+                                                  vrt_options={'cutline': boundaries_cutline})
                     else:
-                        metadata = src.metadata(pmin=pmin, pmax=pmax, hist_options=histogram_options, nodata=nodata)
-                info = json.loads(metadata.json())
+                        stats = src.statistics(percentiles=(pmin, pmax), hist_options=histogram_options, nodata=nodata)
+                info = src.info().model_dump()  
+                info['statistics']= {}
+                for k,v in stats.items():
+                    info['statistics'][k] = v.model_dump()
         except IndexError as e:
             # Caught when trying to get an invalid raster metadata
             raise exceptions.ValidationError("Cannot retrieve raster metadata: %s" % str(e))
@@ -207,8 +208,9 @@ def get(self, request, pk=None, project_pk=None, tile_type=""):
             for b in info['statistics']:
                 info['statistics'][b]['min'] = hrange[0]
                 info['statistics'][b]['max'] = hrange[1]
-                info['statistics'][b]['percentiles'][0] = max(hrange[0], info['statistics'][b]['percentiles'][0])
-                info['statistics'][b]['percentiles'][1] = min(hrange[1], info['statistics'][b]['percentiles'][1])
+                info['statistics'][b]['percentiles'] = [None] * 2
+                info['statistics'][b]['percentiles'][0] = max(hrange[0], info['statistics'][b]['percentile_' + str(int(pmax))])
+                info['statistics'][b]['percentiles'][1] = min(hrange[1], info['statistics'][b]['percentile_' + str(int(pmin))])
 
         cmap_labels = {
             "viridis": "Viridis",
@@ -351,7 +353,7 @@ def get(self, request, pk=None, project_pk=None, tile_type="", z="", x="", y="",
         if not os.path.isfile(url):
             raise exceptions.NotFound()
 
-        with COGReader(url) as src:
+        with Reader(url) as src:
             if not src.tile_exists(z, x, y):
                 raise exceptions.NotFound(_("Outside of bounds"))