From 6e591c02fe0890afb0d5b1c30aafe4fcf781b00a Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Mon, 13 Nov 2023 09:13:00 +0800 Subject: [PATCH 01/16] add: new vegetation index by @NtskwK , @QinShengLiangAn --- app/api/formulas.py | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/app/api/formulas.py b/app/api/formulas.py index 320878bf0..f3b6c109c 100644 --- a/app/api/formulas.py +++ b/app/api/formulas.py @@ -120,6 +120,59 @@ 'help': _('Temperature in Centikelvin degrees.') }, + + # new vegetation index + + 'SIPI': { + 'expr': '(N - B) / (N - R)', + 'help': _('Structure-Insensitive Pigment Index.') + }, + 'mNDVI': { + 'expr': '(N - Re) / (N + Re - 2 * B)', + 'help': _('Modified Normalized Difference Vegetation Index.') + }, + 'RENDVI': { + 'expr': '(N - Re) / (N + Re)', + 'help': _('Ratio Enhanced Normalized Difference Vegetation Index.') + }, + 'MTVI_1': { + 'expr': '1.2 * (1.2 * (N - G) - 2.5 * (R - G))', + 'help': _('Modified Transformed Vegetation Index 1.') + }, + 'MTVI_2': { + 'expr': '1.5 * (1.2 * (N - G) - 2.5(R - G)) / (((2 * N + 1) ** 2 - (6 * N - 5 * (R0 ** 0.5)) - 0.5) ** 0.5)', + 'help': _('Modified Transformed Vegetation Index 2.') + }, + 'GI': { + 'expr': 'G / R', + 'help': _('Greenness Index.'), + }, + 'TVI': { + 'expr': '0.5 * (120 * (N - G) - 200 * (R - G))', + 'help': _('Transparency Vegetation Index.') + }, + 'MCARI_1': { + 'expr': '1.2 * (2.5 * (N - R) - 1.3 * (N - G))', + 'help': _('Modified Chlorophyll Absorption Reflectance Index 1.') + }, + 'CI': { + 'expr': 'N / G - 1', + 'help': _('Chlorophyll Index.') + }, + 'SR': { + 'expr': 'N / R', + 'help': _('Simple Ratio.') + }, + 'mSR_2': { + 'expr': '(N - Re) / (N + Re)', + 'help': _('Modified Simple Ratio.') + }, + 'IPVI': { + 'expr': 'N / (N + R)', + 'help': _('Improved Perpendicular Vegetation Index.') + }, + + # more? '_TESTRB': { From e2e4fb36a8aca962d32e9419a9a1e040ea2e419f Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 20 Aug 2024 03:06:00 +0800 Subject: [PATCH 02/16] style: Update scss style to fit webpack in new version --- app/static/app/js/css/GCPPopup.scss | 6 +++--- app/static/app/js/css/Map.scss | 17 ++++++++--------- app/static/app/js/css/MapView.scss | 5 +++-- app/static/app/js/css/ModelView.scss | 8 +++----- app/static/app/js/css/Paginator.scss | 2 +- app/static/app/js/css/ProjectListItem.scss | 7 +++---- app/static/app/js/css/SharePopup.scss | 10 +++++----- app/static/app/js/css/TagsField.scss | 8 ++++---- app/static/app/js/css/TaskListItem.scss | 4 ++-- 9 files changed, 32 insertions(+), 35 deletions(-) diff --git a/app/static/app/js/css/GCPPopup.scss b/app/static/app/js/css/GCPPopup.scss index 757f50730..5380cf1df 100644 --- a/app/static/app/js/css/GCPPopup.scss +++ b/app/static/app/js/css/GCPPopup.scss @@ -34,13 +34,13 @@ opacity: 0.2; } &.fullscreen{ - &.loading{ - opacity: 1; - } display: flex; align-items: center; justify-content: center; color: white; + &.loading{ + opacity: 1; + } i{ font-size: 200%; position: absolute; diff --git a/app/static/app/js/css/Map.scss b/app/static/app/js/css/Map.scss index 76492ca58..f73ad46c2 100644 --- a/app/static/app/js/css/Map.scss +++ b/app/static/app/js/css/Map.scss @@ -33,23 +33,22 @@ } .asset-links{ + margin-top: 8px; + padding-left: 16px; + + columns: 2; + -webkit-columns: 2; + -moz-columns: 2; li:first-child{ display: none; } &.loading{ - li{ display: none } + padding-left: 0; + list-style-type: none;li{ display: none } li:first-child{ display: block; } - padding-left: 0; - list-style-type: none; } - margin-top: 8px; - padding-left: 16px; - - columns: 2; - -webkit-columns: 2; - -moz-columns: 2; } .switchModeButton{ bottom: 12px; diff --git a/app/static/app/js/css/MapView.scss b/app/static/app/js/css/MapView.scss index 5f709f601..478563c2c 100644 --- a/app/static/app/js/css/MapView.scss +++ b/app/static/app/js/css/MapView.scss @@ -4,13 +4,14 @@ } .map-view{ + height: calc(100% - 20px); + position: relative; + .map-title{ text-overflow: ellipsis; overflow: hidden; white-space: nowrap; } - height: calc(100% - 20px); - position: relative; input[type="range"]{ margin-left: 4px; diff --git a/app/static/app/js/css/ModelView.scss b/app/static/app/js/css/ModelView.scss index 6cf23f8b7..f24a3a881 100644 --- a/app/static/app/js/css/ModelView.scss +++ b/app/static/app/js/css/ModelView.scss @@ -153,7 +153,9 @@ #sidebar_root{ width: 300px; - + position: absolute; + min-height: 100%; + height: 100%; .pv-menu-list{ padding-right: 12px; @@ -170,10 +172,6 @@ cursor: pointer; } - position: absolute; - min-height: 100%; - height: 100%; - .potree_sidebar_brand{ display: flex; flex-direction: row; diff --git a/app/static/app/js/css/Paginator.scss b/app/static/app/js/css/Paginator.scss index d0f78a19d..f467740b9 100644 --- a/app/static/app/js/css/Paginator.scss +++ b/app/static/app/js/css/Paginator.scss @@ -4,10 +4,10 @@ margin-bottom: 8px; .toolbar{ + margin-right: 8px; i{ opacity: 0.8; } - margin-right: 8px; &.no-margin{ margin-right: 0; } diff --git a/app/static/app/js/css/ProjectListItem.scss b/app/static/app/js/css/ProjectListItem.scss index 4572e4edd..4371b8acb 100644 --- a/app/static/app/js/css/ProjectListItem.scss +++ b/app/static/app/js/css/ProjectListItem.scss @@ -1,5 +1,7 @@ .project-list-item{ min-height: 60px; + -webkit-transition: background-color 1s ease; + transition: background-color 1s ease; .project-name{ font-weight: bold; @@ -40,14 +42,11 @@ background-color: #eee; } - -webkit-transition: background-color 1s ease; - transition: background-color 1s ease; - &.dz-drag-hover{ + background-color: #f7f7f7; .drag-drop-icon{ display: block; } - background-color: #f7f7f7; } .project-links{ diff --git a/app/static/app/js/css/SharePopup.scss b/app/static/app/js/css/SharePopup.scss index e19a9327e..5c3ba25e6 100644 --- a/app/static/app/js/css/SharePopup.scss +++ b/app/static/app/js/css/SharePopup.scss @@ -1,4 +1,6 @@ .sharePopup{ + right: 0; + pointer-events: none; position: absolute; &.top{ bottom: 0; @@ -6,8 +8,6 @@ &.bottom{ top: 0; } - right: 0; - pointer-events: none; .sharePopupContainer{ pointer-events: auto; @@ -52,15 +52,15 @@ } .share-links{ + max-height: 0; + overflow: hidden; + transition: max-height 1s ease-in-out; & > div{ margin-top: 8px; &:first-child{ margin-top: 4px; } } - max-height: 0; - overflow: hidden; - transition: max-height 1s ease-in-out; &.show{ max-height: 800px; diff --git a/app/static/app/js/css/TagsField.scss b/app/static/app/js/css/TagsField.scss index 3c7a07820..dbd55d59b 100644 --- a/app/static/app/js/css/TagsField.scss +++ b/app/static/app/js/css/TagsField.scss @@ -7,9 +7,6 @@ cursor: text; } .tag-badge{ - &:hover{ - cursor: grab; - } display: inline-block; width: auto; padding-left: 6px; @@ -19,7 +16,10 @@ margin-right: 4px; margin-bottom: 8px; border-radius: 6px; - + + &:hover{ + cursor: grab; + } a{ margin-top: 2px; font-weight: bold; diff --git a/app/static/app/js/css/TaskListItem.scss b/app/static/app/js/css/TaskListItem.scss index 2644fbfcb..8a7b474e2 100644 --- a/app/static/app/js/css/TaskListItem.scss +++ b/app/static/app/js/css/TaskListItem.scss @@ -52,10 +52,10 @@ @media screen and (max-width: 576px){ .status-label { + text-align: center; & > span{ display: none; } - text-align: center; } } @@ -70,6 +70,7 @@ padding-right: 16px; .info-table { + margin-bottom: 12px; tr td:first-child { width: 33%; max-width: 150px; @@ -77,7 +78,6 @@ td{ padding: 0; } - margin-bottom: 12px; @media screen and (min-width: 1200px) { tr td:first-child { width: 20%; From edb3b994b11c2285b61332b2325e244c42467ee1 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 20 Aug 2024 04:25:35 +0800 Subject: [PATCH 03/16] feat: add .vscode to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d69cbb8b0..168adfca7 100644 --- a/.gitignore +++ b/.gitignore @@ -95,6 +95,7 @@ node_modules/ webpack-stats.json pip-selfcheck.json .idea/ +.vscode/ package-lock.json .cronenv .initialized From 844297b9e4747ff336db79b2cea70fb7b8e6e7fd Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 20 Aug 2024 04:26:28 +0800 Subject: [PATCH 04/16] feat: remove version tag from docker-compose files --- docker-compose.build.yml | 1 - docker-compose.dev.yml | 1 - docker-compose.nodemicmac.yml | 2 -- docker-compose.nodeodm.gpu.intel.yml | 2 -- docker-compose.nodeodm.gpu.nvidia.yml | 2 -- docker-compose.nodeodm.yml | 2 -- docker-compose.settings.yml | 1 - docker-compose.ssl-manual.yml | 1 - docker-compose.ssl.yml | 1 - docker-compose.worker-cpu.yml | 1 - docker-compose.worker-memory.yml | 1 - docker-compose.yml | 1 - 12 files changed, 16 deletions(-) diff --git a/docker-compose.build.yml b/docker-compose.build.yml index 302c8c517..80dd43e09 100644 --- a/docker-compose.build.yml +++ b/docker-compose.build.yml @@ -1,6 +1,5 @@ # This configuration builds from Dockerfiles # instead of pulling from opendronemap hub -version: '2.1' services: db: build: ./db diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 393fec3ec..b28b90459 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,4 +1,3 @@ -version: '2.1' services: webapp: entrypoint: /bin/bash -c "chmod +x /webodm/*.sh && /bin/bash -c \"/webodm/wait-for-postgres.sh db /webodm/wait-for-it.sh -t 0 broker:6379 -- /webodm/start.sh --create-default-pnode --setup-devenv\"" diff --git a/docker-compose.nodemicmac.yml b/docker-compose.nodemicmac.yml index 3e71f2474..26f1ec911 100644 --- a/docker-compose.nodemicmac.yml +++ b/docker-compose.nodemicmac.yml @@ -1,8 +1,6 @@ # Chaining this file to the main docker-compose file adds # a default processing node instance. This is best for users # who are just getting started with WebODM. - -version: '2.1' services: webapp: depends_on: diff --git a/docker-compose.nodeodm.gpu.intel.yml b/docker-compose.nodeodm.gpu.intel.yml index 4e3f72c24..8759ad512 100644 --- a/docker-compose.nodeodm.gpu.intel.yml +++ b/docker-compose.nodeodm.gpu.intel.yml @@ -1,8 +1,6 @@ # Chaining this file to the main docker-compose file adds # a default processing node instance. This is best for users # who are just getting started with WebODM. - -version: '2.1' services: webapp: depends_on: diff --git a/docker-compose.nodeodm.gpu.nvidia.yml b/docker-compose.nodeodm.gpu.nvidia.yml index b16d92011..be4c046e9 100644 --- a/docker-compose.nodeodm.gpu.nvidia.yml +++ b/docker-compose.nodeodm.gpu.nvidia.yml @@ -1,8 +1,6 @@ # Chaining this file to the main docker-compose file adds # a default processing node instance. This is best for users # who are just getting started with WebODM. - -version: '2.1' services: webapp: depends_on: diff --git a/docker-compose.nodeodm.yml b/docker-compose.nodeodm.yml index 08454eff8..29771ad46 100644 --- a/docker-compose.nodeodm.yml +++ b/docker-compose.nodeodm.yml @@ -1,8 +1,6 @@ # Chaining this file to the main docker-compose file adds # a default processing node instance. This is best for users # who are just getting started with WebODM. - -version: '2.1' services: webapp: depends_on: diff --git a/docker-compose.settings.yml b/docker-compose.settings.yml index 16182fb1c..df759d370 100644 --- a/docker-compose.settings.yml +++ b/docker-compose.settings.yml @@ -1,4 +1,3 @@ -version: '2.1' services: webapp: volumes: diff --git a/docker-compose.ssl-manual.yml b/docker-compose.ssl-manual.yml index 6c809a91e..6a57b3980 100644 --- a/docker-compose.ssl-manual.yml +++ b/docker-compose.ssl-manual.yml @@ -1,5 +1,4 @@ # This configuration adds the volumes necessary for SSL manual setup -version: '2.1' services: webapp: volumes: diff --git a/docker-compose.ssl.yml b/docker-compose.ssl.yml index bfa68bd28..67c30000f 100644 --- a/docker-compose.ssl.yml +++ b/docker-compose.ssl.yml @@ -1,5 +1,4 @@ # This configuration adds support for SSL -version: '2.1' volumes: letsencrypt: driver: local diff --git a/docker-compose.worker-cpu.yml b/docker-compose.worker-cpu.yml index b002e0e1d..9afffb5c5 100644 --- a/docker-compose.worker-cpu.yml +++ b/docker-compose.worker-cpu.yml @@ -1,4 +1,3 @@ -version: '2.2' services: worker: cpus: ${WO_WORKER_CPUS} \ No newline at end of file diff --git a/docker-compose.worker-memory.yml b/docker-compose.worker-memory.yml index b2f7b98a0..cbaef4bd0 100644 --- a/docker-compose.worker-memory.yml +++ b/docker-compose.worker-memory.yml @@ -1,4 +1,3 @@ -version: '2.1' services: worker: mem_limit: ${WO_WORKER_MEMORY} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 9c460d99b..00eed19b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,5 @@ # This configuration does not include a processing node # Which makes for faster setup times -version: '2.1' volumes: dbdata: appmedia: From 197e72ccecb0ba8b530d7382333c57fe53151a49 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 20 Aug 2024 04:36:48 +0800 Subject: [PATCH 05/16] feat: update postgresql to 16.1 --- db/Dockerfile | 52 ++++++++++++++++++----------------------- db/docker-entrypoint.sh | 2 ++ 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/db/Dockerfile b/db/Dockerfile index 13f2b12d8..d5bd85b94 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -1,17 +1,16 @@ FROM ubuntu:20.04 MAINTAINER Piero Toffanin -ENV POSTGRES_PASSWORD postgres -ENV POSTGRES_HOST_AUTH_METHOD trust -ENV POSTGRES_ALLOW_HOST all -ENV GOSU_VERSION 1.12 -ENV PG_MAJOR 9.5 -ENV PG_VERSION 9.5.25 -ENV POSTGIS_VERSION 2.3.2 -ENV PATH $PATH:/usr/local/pgsql/bin +ENV POSTGRES_PASSWORD=postgres +ENV POSTGRES_HOST_AUTH_METHOD=trust +ENV POSTGRES_ALLOW_HOST=all +ENV GOSU_VERSION=1.12 +ENV PG_MAJOR=16.1 +ENV POSTGIS_VERSION=3.3.6 +ENV PATH=$PATH:/usr/local/pgsql/bin ENV DEBIAN_FRONTEND=noninteractive -ENV LANG en_US.utf8 -ENV PGDATA /var/lib/postgresql/data +ENV LANG=en_US.utf8 +ENV PGDATA=/var/lib/postgresql/data RUN mkdir /docker-entrypoint-initdb.d COPY init.sql /docker-entrypoint-initdb.d/init-db.sql @@ -29,30 +28,26 @@ RUN apt-get update; \ chown -R postgres:postgres /var/lib/postgresql; \ # grab gosu for easy step-down from root # https://github.com/tianon/gosu/releases - apt-get update; \ - apt-get install -y --no-install-recommends wget; \ - rm -rf /var/lib/apt/lists/*; \ + apt-get install -y --no-install-recommends wget gcc libreadline-dev > /dev/null; \ dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ - wget --no-check-certificate -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget --no-check-certificate -q -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ chmod +x /usr/local/bin/gosu; \ gosu --version; \ gosu nobody true; \ # make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default - apt-get update; \ - apt-get install -y --no-install-recommends locales; \ + apt-get install -y --no-install-recommends locales > /dev/null; \ localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8; \ # Build Postgres from source mkdir /staging; \ - apt-get update; \ - apt-get install -y --no-install-recommends wget gcc build-essential libproj-dev libgeos-dev libxml2-dev zlib1g-dev libreadline-dev; \ + apt-get install -y --no-install-recommends build-essential libproj-dev libgeos-dev libxml2-dev zlib1g-dev > /dev/null ; \ cd /staging; \ - wget --no-check-certificate -q https://github.com/OpenDroneMap/WebODM/releases/download/v1.9.2/postgresql-$PG_VERSION.tar.gz; \ + wget --no-check-certificate -q https://ftp.postgresql.org/pub/source/v$PG_MAJOR/postgresql-$PG_MAJOR.tar.gz; \ cd /staging; \ - tar -zxf postgresql-$PG_VERSION.tar.gz; \ - cd postgresql-$PG_VERSION; \ - ./configure; \ - make -j$(nproc); \ - make install; \ + tar -zxf postgresql-$PG_MAJOR.tar.gz; \ + cd postgresql-$PG_MAJOR; \ + ./configure --without-icu > /dev/null; \ + make -j$(nproc) > /dev/null; \ + make install > /dev/null; \ postgres --version; \ sed -ri "s/#autovacuum_max_workers = 3/autovacuum_max_workers = 6/" /usr/local/pgsql/share/postgresql.conf.sample; \ sed -ri "s/#autovacuum_naptime = 1min/autovacuum_naptime = 15s/" /usr/local/pgsql/share/postgresql.conf.sample; \ @@ -69,15 +64,14 @@ RUN apt-get update; \ # this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA"; \ # Build PostGIS from source - apt-get update; \ - apt-get install -y --no-install-recommends libgdal-dev libjson-c-dev; \ + apt-get install -o Acquire::Retries=3 -y --no-install-recommends libgdal-dev libjson-c-dev protobuf-c-compiler libprotobuf-c-dev; \ cd /staging; \ - wget --no-check-certificate -q https://github.com/OpenDroneMap/WebODM/releases/download/v1.9.2/postgis-$POSTGIS_VERSION.tar.gz; \ - wget --no-check-certificate -q -O /usr/include/json-c/json_object_private.h https://raw.githubusercontent.com/json-c/json-c/json-c-0.13/json_object_private.h; \ + wget --no-check-certificate -q https://download.osgeo.org/postgis/source/postgis-$POSTGIS_VERSION.tar.gz; \ + wget --no-check-certificate -q -O /usr/include/json-c/json_object_private.h https://raw.githubusercontent.com/json-c/json-c/json-c-0.16/json_object_private.h; \ tar -zxf postgis-$POSTGIS_VERSION.tar.gz; \ sed -i 's/#error.*/#define ACCEPT_USE_OF_DEPRECATED_PROJ_API_H 1/' /usr/include/proj_api.h; \ cd /staging/postgis-$POSTGIS_VERSION; \ - ./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config; \ + ./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config --with-gdalconfig=/usr/bin/gdal-config > /dev/null; \ make; \ make install; \ sed -i '1d' /usr/local/pgsql/share/extension/postgis--$POSTGIS_VERSION.sql; \ diff --git a/db/docker-entrypoint.sh b/db/docker-entrypoint.sh index 7adf35c92..eb6d88f86 100755 --- a/db/docker-entrypoint.sh +++ b/db/docker-entrypoint.sh @@ -321,6 +321,8 @@ _main() { fi exec "$@" + docker_process_sql --dbname="$POSTGRES_DB" <<<'CREATE extension postgis_raster CASCADE;' + } if ! _is_sourced; then From e1656cdb12749ab267cd57ff8832d48302f25e41 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 20 Aug 2024 04:45:24 +0800 Subject: [PATCH 06/16] feat: update webpack config for hash filename --- webpack.config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webpack.config.js b/webpack.config.js index 5ca2465d1..9070496b4 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -22,7 +22,7 @@ module.exports = { output: { path: path.join(__dirname, './app/static/app/bundles/'), - filename: "[name]-[hash].js", + filename: "[name]-[fullhash].js", publicPath: '/static/app/bundles/' }, @@ -33,7 +33,7 @@ module.exports = { path: path.join(__dirname, './'), }), new MiniCssExtractPlugin({ - filename: "./css/[name]-[hash].css", + filename: "./css/[name]-[fullhash].css", chunkFilename: "[id].css" }), new webpack.ProvidePlugin({ From 83ab17095646d28a0b6440b65032369edcd0c8bc Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 20 Aug 2024 04:46:35 +0800 Subject: [PATCH 07/16] refactor: update celery loglevel in worker.sh for new performance --- worker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker.sh b/worker.sh index d471b6231..1b48f09f4 100755 --- a/worker.sh +++ b/worker.sh @@ -52,7 +52,7 @@ start(){ action=$1 echo "Starting worker using broker at $WO_BROKER" - celery -A worker worker --autoscale $(grep -c '^processor' /proc/cpuinfo),2 --max-tasks-per-child 1000 --loglevel=warn > /dev/null + celery -A worker worker --autoscale $(grep -c '^processor' /proc/cpuinfo),2 --max-tasks-per-child 1000 --loglevel=WARNING > /dev/null } start_scheduler(){ From d694ed84a25b6c98b4f823cbc96639af1717b0c4 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 20 Aug 2024 04:47:30 +0800 Subject: [PATCH 08/16] refactor: ingore authentication.py for rest_framework_jwt.authentication --- app/api/authentication.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/api/authentication.py b/app/api/authentication.py index c95c14ad4..2dd11b098 100644 --- a/app/api/authentication.py +++ b/app/api/authentication.py @@ -1,6 +1,8 @@ -from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication - - -class JSONWebTokenAuthenticationQS(BaseJSONWebTokenAuthentication): - def get_jwt_value(self, request): - return request.query_params.get('jwt') \ No newline at end of file +# from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication +# +# jwt_decode_handler = api_settings.JWT_DECODE_HANDLER +# jwt_get_username_from_payload_handler = api_settings.JWT_PAYLOAD_GET_USERNAME_HANDLER +# +# class JSONWebTokenAuthenticationQS(BaseJSONWebTokenAuthentication): +# def get_jwt_value(self, request): +# return request.query_params.get('jwt') \ No newline at end of file From b103c57fb497fcc61d0b8dcacb5035bd4c08af83 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 20 Aug 2024 04:51:39 +0800 Subject: [PATCH 09/16] feat: add GDAL version check and makemigrations in start.sh --- start.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/start.sh b/start.sh index 4c8763d2d..c47eb8b1d 100755 --- a/start.sh +++ b/start.sh @@ -28,6 +28,7 @@ if [ $? -ne 0 ]; then fi # Check GDAL version +echo $(gdalinfo --version) python -c "import sys;import re;import subprocess;version = subprocess.Popen([\"gdalinfo\", \"--version\"], stdout=subprocess.PIPE).communicate()[0].decode().rstrip();ret = 0 if re.compile('^GDAL [2-9]\.[0-9]+').match(version) else 1; print('Checking GDAL version... ' + ('{}, excellent!'.format(version) if ret == 0 else version));sys.exit(ret);" if [ $? -ne 0 ]; then almost_there @@ -55,6 +56,9 @@ if [ "$1" = "--setup-devenv" ] || [ "$2" = "--setup-devenv" ]; then echo Build translations... python manage.py translate build --safe + echo Running makemigrations + python manage.py makemigrations app + echo Setup webpack watch... webpack --watch & fi From 9e1c5b8179320520d4e038d1b2114733488a23b5 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Sun, 25 Aug 2024 23:55:33 +0800 Subject: [PATCH 10/16] refactor: change Docker base image to Ubuntu 20.04 --- Dockerfile | 54 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2f63c0cda..57b608771 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,55 @@ -FROM ubuntu:21.04 +FROM ubuntu:20.04 MAINTAINER Piero Toffanin ARG TEST_BUILD ARG DEBIAN_FRONTEND=noninteractive -ENV PYTHONUNBUFFERED 1 -ENV PYTHONPATH $PYTHONPATH:/webodm -ENV PROJ_LIB=/usr/share/proj +ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=$PYTHONPATH:/webodm +ENV NODE_MAJOR=20 +ENV PYTHON_MAJOR=3.9 +ENV GDAL_VERSION=3.8.5 +ENV LD_LIBRARY_PATH=/usr/local/lib # Prepare directory ADD . /webodm/ WORKDIR /webodm -# Use old-releases for 21.04 -RUN printf "deb http://old-releases.ubuntu.com/ubuntu/ hirsute main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-backports main restricted universe multiverse" > /etc/apt/sources.list +RUN apt-get -o Acquire::Retries=3 -qq update > /dev/null && \ + apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends wget curl git g++ clang make cmake postgresql-client > /dev/null && \ -# Install Node.js using new Node install method -RUN apt-get -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends wget curl && \ - apt-get -o Acquire::Retries=3 install -y ca-certificates gnupg && \ + # Install PDAL, letsencrypt, psql, cron + apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends binutils pdal certbot gettext tzdata libproj-dev libpq-dev > /dev/null && \ + + # Install Python in target version + apt-get -qq autoremove -y python3 && \ + apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends python$PYTHON_MAJOR-dev python$PYTHON_MAJOR-full && \ + ln -s /usr/bin/python$PYTHON_MAJOR /usr/bin/python && \ + curl https://bootstrap.pypa.io/get-pip.py | python && \ + + echo $(pip -V) && \ + echo $(python -V) && \ + + # Build GDAL from source + wget --no-check-certificate -q https://github.com/OSGeo/gdal/releases/download/v$GDAL_VERSION/gdal-$GDAL_VERSION.tar.gz && \ + tar -xzf gdal-$GDAL_VERSION.tar.gz && \ + cd gdal-$GDAL_VERSION && mkdir build && cd build && \ + cmake .. && cmake --build . -j$(nproc) --target install && \ + cd / && rm -rf gdal-$GDAL_VERSION gdal-$GDAL_VERSION.tar.gz && \ + + # Install pip reqs + cd /webodm && \ + pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \ + pip install --quiet -U pip && \ + pip install --quiet -r requirements.txt "boto3==1.34.145" && \ + + # Install Node.js using new Node install method + apt-get -o Acquire::Retries=3 -qq install -y ca-certificates gnupg && \ mkdir -p /etc/apt/keyrings && \ curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - NODE_MAJOR=20 && \ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y nodejs && \ - # Install Python3, GDAL, PDAL, nginx, letsencrypt, psql - apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot gettext-base cron postgresql-client-13 gettext tzdata && \ - update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2 && \ - # Install pip reqs - pip install pip==24.0 && pip install -r requirements.txt "boto3==1.14.14" && \ # Setup cron + apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends nginx cron && \ ln -s /webodm/nginx/crontab /var/spool/cron/crontabs/root && chmod 0644 /webodm/nginx/crontab && service cron start && chmod +x /webodm/nginx/letsencrypt-autogen.sh && \ /webodm/nodeodm/setup.sh && /webodm/nodeodm/cleanup.sh && cd /webodm && \ npm install --quiet -g webpack@5.89.0 && npm install --quiet -g webpack-cli@5.1.4 && npm install --quiet && webpack --mode production && \ @@ -36,7 +58,7 @@ RUN apt-get -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-inst python manage.py rebuildplugins && \ python manage.py translate build --safe && \ # Cleanup - apt-get remove -y g++ python3-dev libpq-dev && apt-get autoremove -y && \ + apt-get remove -y g++ python2 && apt-get autoremove -y && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ rm /webodm/webodm/secret_key.py From 040392f69f34de9f98b496a32fe0dc5dfca2ea53 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Sun, 25 Aug 2024 23:55:51 +0800 Subject: [PATCH 11/16] chore: update rio_tiler to 6.6.1 --- app/api/tiler.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/app/api/tiler.py b/app/api/tiler.py index a57007721..0e3a38039 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -11,12 +11,11 @@ from rio_tiler.errors import TileOutsideBounds from rio_tiler.utils import has_alpha_band, \ non_alpha_indexes, render, create_cutline -from rio_tiler.utils import _stats as raster_stats -from rio_tiler.models import ImageStatistics, ImageData -from rio_tiler.models import Metadata as RioMetadata +from rio_tiler.utils import get_array_statistics +from rio_tiler.models import BandStatistics from rio_tiler.profiles import img_profiles from rio_tiler.colormap import cmap as colormap, apply_cmap -from rio_tiler.io import COGReader +from rio_tiler.io import Reader from rio_tiler.errors import InvalidColorMapName, AlphaBandWarning import numpy as np from .custom_colormaps_helper import custom_colormaps @@ -97,7 +96,7 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): if not os.path.isfile(raster_path): raise exceptions.NotFound() - with COGReader(raster_path) as src: + with Reader(raster_path) as src: minzoom, maxzoom = get_zoom_safe(src) return Response({ @@ -164,7 +163,7 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): if not os.path.isfile(raster_path): raise exceptions.NotFound() try: - with COGReader(raster_path) as src: + with Reader(raster_path) as src: band_count = src.dataset.meta['count'] if boundaries_feature is not None: boundaries_cutline = create_cutline(src.dataset, boundaries_feature, CRS.from_string('EPSG:4326')) @@ -187,18 +186,20 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): data = np.ma.array(data) data.mask = mask == 0 stats = { - str(b + 1): raster_stats(data[b], percentiles=(pmin, pmax), bins=255, range=hrange) + str(b + 1): get_array_statistics(data[b], percentiles=(pmin, pmax), bins=255, range=hrange) for b in range(data.shape[0]) } - stats = {b: ImageStatistics(**s) for b, s in stats.items()} - metadata = RioMetadata(statistics=stats, **src.info().dict()) + stats = {b: BandStatistics(**s[0]) for b, s in stats.items()} else: if (boundaries_cutline is not None) and (boundaries_bbox is not None): - metadata = src.metadata(pmin=pmin, pmax=pmax, hist_options=histogram_options, nodata=nodata - , bounds=boundaries_bbox, vrt_options={'cutline': boundaries_cutline}) + stats = src.statistics(percentiles=(pmin, pmax), hist_options=histogram_options, nodata=nodata, + vrt_options={'cutline': boundaries_cutline}) else: - metadata = src.metadata(pmin=pmin, pmax=pmax, hist_options=histogram_options, nodata=nodata) - info = json.loads(metadata.json()) + stats = src.statistics(percentiles=(pmin, pmax), hist_options=histogram_options, nodata=nodata) + info = src.info().model_dump() + info['statistics']= {} + for k,v in stats.items(): + info['statistics'][k] = v.model_dump() except IndexError as e: # Caught when trying to get an invalid raster metadata raise exceptions.ValidationError("Cannot retrieve raster metadata: %s" % str(e)) @@ -207,8 +208,9 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): for b in info['statistics']: info['statistics'][b]['min'] = hrange[0] info['statistics'][b]['max'] = hrange[1] - info['statistics'][b]['percentiles'][0] = max(hrange[0], info['statistics'][b]['percentiles'][0]) - info['statistics'][b]['percentiles'][1] = min(hrange[1], info['statistics'][b]['percentiles'][1]) + info['statistics'][b]['percentiles'] = [None] * 2 + info['statistics'][b]['percentiles'][0] = max(hrange[0], info['statistics'][b]['percentile_' + str(int(pmax))]) + info['statistics'][b]['percentiles'][1] = min(hrange[1], info['statistics'][b]['percentile_' + str(int(pmin))]) cmap_labels = { "viridis": "Viridis", @@ -351,7 +353,7 @@ def get(self, request, pk=None, project_pk=None, tile_type="", z="", x="", y="", if not os.path.isfile(url): raise exceptions.NotFound() - with COGReader(url) as src: + with Reader(url) as src: if not src.tile_exists(z, x, y): raise exceptions.NotFound(_("Outside of bounds")) From 5289cd104820b043b8052b469cf5cb0575600060 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Sun, 25 Aug 2024 23:56:25 +0800 Subject: [PATCH 12/16] refactor: update Django to 4.2 LTS --- app/admin.py | 19 ++- app/api/urls.py | 56 ++++----- ...0_alter_plugindatum_bool_value_and_more.py | 118 ++++++++++++++++++ app/models/plugin_datum.py | 5 +- app/models/preset.py | 3 +- app/models/task.py | 6 +- app/plugins/views.py | 10 +- app/urls.py | 38 +++--- app/views/app.py | 2 +- app/views/dev.py | 2 +- app/views/public.py | 2 +- coreplugins/editshortlinks/api.py | 2 +- locale | 2 +- ..._alter_processingnode_available_options.py | 18 +++ nodeodm/models.py | 3 +- requirements.txt | 87 ++++--------- webodm/settings.py | 17 +-- webodm/urls.py | 45 ++++--- 18 files changed, 268 insertions(+), 167 deletions(-) create mode 100644 app/migrations/0040_alter_plugindatum_bool_value_and_more.py create mode 100644 nodeodm/migrations/0010_alter_processingnode_available_options.py diff --git a/app/admin.py b/app/admin.py index 7bb98a5d4..4c7ac679f 100644 --- a/app/admin.py +++ b/app/admin.py @@ -3,11 +3,10 @@ import zipfile import shutil -from django.conf.urls import url from django.contrib import admin from django.contrib import messages from django.http import HttpResponseRedirect -from django.urls import reverse +from django.urls import reverse, path, re_path from django.utils.html import format_html from guardian.admin import GuardedModelAdmin from django.contrib.auth.admin import UserAdmin as BaseUserAdmin @@ -129,23 +128,23 @@ def author(self, obj): def get_urls(self): urls = super().get_urls() custom_urls = [ - url( - r'^(?P.+)/enable/$', + re_path( + '(?P.+)/enable/', self.admin_site.admin_view(self.plugin_enable), name='plugin-enable', ), - url( - r'^(?P.+)/disable/$', + re_path( + '(?P.+)/disable/', self.admin_site.admin_view(self.plugin_disable), name='plugin-disable', ), - url( - r'^(?P.+)/delete/$', + re_path( + '(?P.+)/delete/', self.admin_site.admin_view(self.plugin_delete), name='plugin-delete', ), - url( - r'^actions/upload/$', + re_path( + 'actions/upload/', self.admin_site.admin_view(self.plugin_upload), name='plugin-upload', ), diff --git a/app/api/urls.py b/app/api/urls.py index 7cf862d45..f4e6c11e1 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import url, include +from django.urls import include, re_path, path from app.api.presets import PresetViewSet from app.plugins.views import api_view_handler @@ -8,7 +8,8 @@ from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView from .admin import AdminUserViewSet, AdminGroupViewSet, AdminProfileViewSet from rest_framework_nested import routers -from rest_framework_jwt.views import obtain_jwt_token +from rest_framework_simplejwt.views import TokenObtainPairView, TokenRefreshView + from .tiler import TileJson, Bounds, Metadata, Tiles, Export from .potree import Scene, CameraView from .workers import CheckTask, GetTaskResult @@ -27,44 +28,43 @@ admin_router = routers.DefaultRouter() admin_router.register(r'admin/users', AdminUserViewSet, basename='admin-users') admin_router.register(r'admin/groups', AdminGroupViewSet, basename='admin-groups') -admin_router.register(r'admin/profiles', AdminProfileViewSet, basename='admin-groups') +admin_router.register(r'admin/profiles', AdminProfileViewSet, basename='admin-profiles') urlpatterns = [ - url(r'processingnodes/options/$', ProcessingNodeOptionsView.as_view()), + re_path(r'processingnodes/options/$', ProcessingNodeOptionsView.as_view()), - url(r'^', include(router.urls)), - url(r'^', include(tasks_router.urls)), - url(r'^', include(admin_router.urls)), + re_path(r'^', include(router.urls)), + re_path(r'^', include(tasks_router.urls)), + re_path(r'^', include(admin_router.urls)), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles\.json$', TileJson.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/bounds$', Bounds.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/metadata$', Metadata.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles/(?P[\d]+)/(?P[\d]+)/(?P[\d]+)\.?(?Ppng|jpg|webp)?$', Tiles.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles/(?P[\d]+)/(?P[\d]+)/(?P[\d]+)@(?P[\d]+)x\.?(?Ppng|jpg|webp)?$', Tiles.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm|georeferenced_model)/export$', Export.as_view()), + re_path('projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles\.json', TileJson.as_view()), + re_path('projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/bounds', Bounds.as_view()), + re_path('projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/metadata', Metadata.as_view()), + re_path('projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles/(?P[\d]+)/(?P[\d]+)/(?P[\d]+)\.?(?Ppng|jpg|webp)?', Tiles.as_view()), + re_path('projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles/(?P[\d]+)/(?P[\d]+)/(?P[\d]+)@(?P[\d]+)x\.?(?Ppng|jpg|webp)?', Tiles.as_view()), + re_path('projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm|georeferenced_model)/export', Export.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/download/(?P.+)$', TaskDownloads.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/assets/(?P.+)$', TaskAssets.as_view()), - url(r'projects/(?P[^/.]+)/tasks/import$', TaskAssetsImport.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/backup$', TaskBackup.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/images/thumbnail/(?P.+)$', Thumbnail.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/images/download/(?P.+)$', ImageDownload.as_view()), + re_path(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/download/(?P.+)$', TaskDownloads.as_view()), + re_path(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/assets/(?P.+)$', TaskAssets.as_view()), + re_path(r'projects/(?P[^/.]+)/tasks/import$', TaskAssetsImport.as_view()), + re_path(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/backup$', TaskBackup.as_view()), + re_path(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/images/thumbnail/(?P.+)$', Thumbnail.as_view()), + re_path(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/images/download/(?P.+)$', ImageDownload.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/3d/scene$', Scene.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/3d/cameraview$', CameraView.as_view()), + re_path(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/3d/scene$', Scene.as_view()), + re_path(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/3d/cameraview$', CameraView.as_view()), - url(r'workers/check/(?P.+)', CheckTask.as_view()), - url(r'workers/get/(?P.+)', GetTaskResult.as_view()), + re_path(r'workers/check/(?P.+)', CheckTask.as_view()), + re_path(r'workers/get/(?P.+)', GetTaskResult.as_view()), - url(r'^auth/', include('rest_framework.urls')), - url(r'^token-auth/', obtain_jwt_token), + path(r'auth', include('rest_framework.urls')), - url(r'^plugins/(?P[^/.]+)/(.*)$', api_view_handler), + re_path('plugins/(?P[^/.]+)/(.*)', api_view_handler), ] if settings.ENABLE_USERS_API: - urlpatterns.append(url(r'users', UsersList.as_view())) + urlpatterns.append(path(r'users', UsersList.as_view())) if settings.EXTERNAL_AUTH_ENDPOINT != '': - urlpatterns.append(url(r'^external-token-auth/', ExternalTokenAuth.as_view())) + urlpatterns.append(path(r'external-token-auth', ExternalTokenAuth.as_view())) diff --git a/app/migrations/0040_alter_plugindatum_bool_value_and_more.py b/app/migrations/0040_alter_plugindatum_bool_value_and_more.py new file mode 100644 index 000000000..2e1dbba5c --- /dev/null +++ b/app/migrations/0040_alter_plugindatum_bool_value_and_more.py @@ -0,0 +1,118 @@ +# Generated by Django 4.2.14 on 2024-08-20 05:00 + +import app.models.task +import colorfield.fields +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('app', '0039_task_orthophoto_bands'), + ] + + operations = [ + migrations.AlterField( + model_name='plugindatum', + name='bool_value', + field=models.BooleanField(blank=True, default=None, null=True, verbose_name='Bool value'), + ), + migrations.AlterField( + model_name='plugindatum', + name='json_value', + field=models.JSONField(blank=True, default=None, null=True, verbose_name='JSON value'), + ), + migrations.AlterField( + model_name='plugindatum', + name='user', + field=models.ForeignKey(blank=True, default=None, help_text='The user this setting belongs to. If NULL, the setting is global.', null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User'), + ), + migrations.AlterField( + model_name='preset', + name='options', + field=models.JSONField(blank=True, default=list, help_text="Options that define this preset (same format as in a Task's options).", validators=[app.models.task.validate_task_options], verbose_name='Options'), + ), + migrations.AlterField( + model_name='task', + name='options', + field=models.JSONField(blank=True, default=dict, help_text='Options that are being used to process this task', validators=[app.models.task.validate_task_options], verbose_name='Options'), + ), + migrations.AlterField( + model_name='task', + name='orthophoto_bands', + field=models.JSONField(blank=True, default=list, help_text='List of orthophoto bands', verbose_name='Orthophoto Bands'), + ), + migrations.AlterField( + model_name='task', + name='potree_scene', + field=models.JSONField(blank=True, default=dict, help_text='Serialized potree scene information used to save/load measurements and camera view angle', verbose_name='Potree Scene'), + ), + migrations.AlterField( + model_name='theme', + name='border', + field=colorfield.fields.ColorField(default='#e7e7e7', help_text='The color of most borders.', image_field=None, max_length=25, samples=None, verbose_name='Border'), + ), + migrations.AlterField( + model_name='theme', + name='button_danger', + field=colorfield.fields.ColorField(default='#e74c3c', help_text='Delete button color.', image_field=None, max_length=25, samples=None, verbose_name='Button Danger'), + ), + migrations.AlterField( + model_name='theme', + name='button_default', + field=colorfield.fields.ColorField(default='#95a5a6', help_text='Default button color.', image_field=None, max_length=25, samples=None, verbose_name='Button Default'), + ), + migrations.AlterField( + model_name='theme', + name='button_primary', + field=colorfield.fields.ColorField(default='#2c3e50', help_text='Primary button color.', image_field=None, max_length=25, samples=None, verbose_name='Button Primary'), + ), + migrations.AlterField( + model_name='theme', + name='dialog_warning', + field=colorfield.fields.ColorField(default='#f39c12', help_text='The border color of warning dialogs.', image_field=None, max_length=25, samples=None, verbose_name='Dialog Warning'), + ), + migrations.AlterField( + model_name='theme', + name='failed', + field=colorfield.fields.ColorField(default='#ffcbcb', help_text='The background color of failed notifications.', image_field=None, max_length=25, samples=None, verbose_name='Failed'), + ), + migrations.AlterField( + model_name='theme', + name='header_background', + field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.", image_field=None, max_length=25, samples=None, verbose_name='Header Background'), + ), + migrations.AlterField( + model_name='theme', + name='header_primary', + field=colorfield.fields.ColorField(default='#ffffff', help_text="Text and icons in the site's header.", image_field=None, max_length=25, samples=None, verbose_name='Header Primary'), + ), + migrations.AlterField( + model_name='theme', + name='highlight', + field=colorfield.fields.ColorField(default='#f7f7f7', help_text='The background color of panels and some borders.', image_field=None, max_length=25, samples=None, verbose_name='Highlight'), + ), + migrations.AlterField( + model_name='theme', + name='primary', + field=colorfield.fields.ColorField(default='#2c3e50', help_text='Most text, icons, and borders.', image_field=None, max_length=25, samples=None, verbose_name='Primary'), + ), + migrations.AlterField( + model_name='theme', + name='secondary', + field=colorfield.fields.ColorField(default='#ffffff', help_text='The main background color, and text color of some buttons.', image_field=None, max_length=25, samples=None, verbose_name='Secondary'), + ), + migrations.AlterField( + model_name='theme', + name='success', + field=colorfield.fields.ColorField(default='#cbffcd', help_text='The background color of success notifications.', image_field=None, max_length=25, samples=None, verbose_name='Success'), + ), + migrations.AlterField( + model_name='theme', + name='tertiary', + field=colorfield.fields.ColorField(default='#3498db', help_text='Navigation links.', image_field=None, max_length=25, samples=None, verbose_name='Tertiary'), + ), + ] diff --git a/app/models/plugin_datum.py b/app/models/plugin_datum.py index 2fb67ab09..ce842e79f 100644 --- a/app/models/plugin_datum.py +++ b/app/models/plugin_datum.py @@ -1,5 +1,4 @@ from django.db import models -from django.contrib.postgres import fields from django.conf import settings from django.utils.translation import gettext_lazy as _ @@ -8,9 +7,9 @@ class PluginDatum(models.Model): user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True, default=None, on_delete=models.CASCADE, help_text=_("The user this setting belongs to. If NULL, the setting is global."), verbose_name=_("User")) int_value = models.IntegerField(blank=True, null=True, default=None, verbose_name=_("Integer value")) float_value = models.FloatField(blank=True, null=True, default=None, verbose_name=_("Float value")) - bool_value = models.NullBooleanField(blank=True, null=True, default=None, verbose_name=_("Bool value")) + bool_value = models.BooleanField(blank=True, null=True, default=None, verbose_name=_("Bool value")) string_value = models.TextField(blank=True, null=True, default=None, verbose_name=_("String value")) - json_value = fields.JSONField(default=None, blank=True, null=True, verbose_name=_("JSON value")) + json_value = models.JSONField(default=None, blank=True, null=True, verbose_name=_("JSON value")) def __str__(self): return self.key diff --git a/app/models/preset.py b/app/models/preset.py index 5e6612b13..f3488a898 100644 --- a/app/models/preset.py +++ b/app/models/preset.py @@ -1,5 +1,4 @@ from django.conf import settings -from django.contrib.postgres.fields import JSONField from django.db import models from django.utils import timezone from .task import validate_task_options @@ -8,7 +7,7 @@ class Preset(models.Model): owner = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True, on_delete=models.CASCADE, help_text=_("The person who owns this preset"), verbose_name=_("Owner")) name = models.CharField(max_length=255, blank=False, null=False, help_text=_("A label used to describe the preset"), verbose_name=_("Name")) - options = JSONField(default=list, blank=True, help_text=_("Options that define this preset (same format as in a Task's options)."), verbose_name=_("Options"), + options = models.JSONField(default=list, blank=True, help_text=_("Options that define this preset (same format as in a Task's options)."), verbose_name=_("Options"), validators=[validate_task_options]) created_at = models.DateTimeField(default=timezone.now, help_text=_("Creation date"), verbose_name=_("Created at")) system = models.BooleanField(db_index=True, default=False, help_text=_("Whether this preset is available to every user in the system or just to its owner."), verbose_name=_("System")) diff --git a/app/models/task.py b/app/models/task.py index 44c449eaf..0d7237770 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -248,7 +248,7 @@ class Task(models.Model): auto_processing_node = models.BooleanField(default=True, help_text=_("A flag indicating whether this task should be automatically assigned a processing node"), verbose_name=_("Auto Processing Node")) status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text=_("Current status of the task"), verbose_name=_("Status")) last_error = models.TextField(null=True, blank=True, help_text=_("The last processing error received"), verbose_name=_("Last Error")) - options = fields.JSONField(default=dict, blank=True, help_text=_("Options that are being used to process this task"), validators=[validate_task_options], verbose_name=_("Options")) + options = models.JSONField(default=dict, blank=True, help_text=_("Options that are being used to process this task"), validators=[validate_task_options], verbose_name=_("Options")) available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text=_("List of available assets to download"), verbose_name=_("Available Assets")) orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text=_("Extent of the orthophoto"), verbose_name=_("Orthophoto Extent")) @@ -277,10 +277,10 @@ class Task(models.Model): import_url = models.TextField(null=False, default="", blank=True, help_text=_("URL this task is imported from (only for imported tasks)"), verbose_name=_("Import URL")) images_count = models.IntegerField(null=False, blank=True, default=0, help_text=_("Number of images associated with this task"), verbose_name=_("Images Count")) partial = models.BooleanField(default=False, help_text=_("A flag indicating whether this task is currently waiting for information or files to be uploaded before being considered for processing."), verbose_name=_("Partial")) - potree_scene = fields.JSONField(default=dict, blank=True, help_text=_("Serialized potree scene information used to save/load measurements and camera view angle"), verbose_name=_("Potree Scene")) + potree_scene = models.JSONField(default=dict, blank=True, help_text=_("Serialized potree scene information used to save/load measurements and camera view angle"), verbose_name=_("Potree Scene")) epsg = models.IntegerField(null=True, default=None, blank=True, help_text=_("EPSG code of the dataset (if georeferenced)"), verbose_name="EPSG") tags = models.TextField(db_index=True, default="", blank=True, help_text=_("Task tags"), verbose_name=_("Tags")) - orthophoto_bands = fields.JSONField(default=list, blank=True, help_text=_("List of orthophoto bands"), verbose_name=_("Orthophoto Bands")) + orthophoto_bands = models.JSONField(default=list, blank=True, help_text=_("List of orthophoto bands"), verbose_name=_("Orthophoto Bands")) size = models.FloatField(default=0.0, blank=True, help_text=_("Size of the task on disk in megabytes"), verbose_name=_("Size")) class Meta: diff --git a/app/plugins/views.py b/app/plugins/views.py index 8944a0a9b..e44b25bc4 100644 --- a/app/plugins/views.py +++ b/app/plugins/views.py @@ -6,7 +6,7 @@ from django.http import HttpResponse, Http404 from .functions import get_plugin_by_name, get_active_plugins -from django.conf.urls import url +from django.urls import re_path from django.views.static import serve from urllib.parse import urlparse @@ -26,7 +26,7 @@ def app_view_handler(request, plugin_name=None): # Try mountpoints first for mount_point in plugin.app_mount_points(): - view, args, kwargs = try_resolve_url(request, url(r'^/plugins/{}/{}'.format(plugin_name, mount_point.url), + view, args, kwargs = try_resolve_url(request, re_path(r'^/plugins/{}/{}'.format(plugin_name, mount_point.url), mount_point.view, *mount_point.args, **mount_point.kwargs)) @@ -35,7 +35,7 @@ def app_view_handler(request, plugin_name=None): # Try public assets if os.path.exists(plugin.get_path("public")) and plugin.serve_public_assets(request): - view, args, kwargs = try_resolve_url(request, url('^/plugins/{}/(.*)'.format(plugin_name), + view, args, kwargs = try_resolve_url(request, re_path('^/plugins/{}/(.*)'.format(plugin_name), serve, {'document_root': plugin.get_path("public")})) if view: @@ -50,7 +50,7 @@ def api_view_handler(request, plugin_name=None): raise Http404("Plugin not found") for mount_point in plugin.api_mount_points(): - view, args, kwargs = try_resolve_url(request, url(r'^/api/plugins/{}/{}'.format(plugin_name, mount_point.url), + view, args, kwargs = try_resolve_url(request, re_path(r'^/api/plugins/{}/{}'.format(plugin_name, mount_point.url), mount_point.view, *mount_point.args, **mount_point.kwargs)) @@ -64,6 +64,6 @@ def root_url_patterns(): result = [] for p in get_active_plugins(): for mount_point in p.root_mount_points(): - result.append(url(mount_point.url, mount_point.view, *mount_point.args, **mount_point.kwargs)) + result.append(path(mount_point.url, mount_point.view, *mount_point.args, **mount_point.kwargs)) return result \ No newline at end of file diff --git a/app/urls.py b/app/urls.py index 97eb9e552..f651ae014 100644 --- a/app/urls.py +++ b/app/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import url, include +from django.urls import path, re_path, include from django.views.i18n import JavaScriptCatalog from .views import app as app_views, public as public_views, dev as dev_views @@ -20,31 +20,31 @@ sync_plugin_db() urlpatterns = [ - url(r'^$', app_views.index, name='index'), - url(r'^welcome/$', app_views.welcome, name='welcome'), - url(r'^dashboard/$', app_views.dashboard, name='dashboard'), - url(r'^map/project/(?P[^/.]+)/task/(?P[^/.]+)/$', app_views.map, name='map'), - url(r'^map/project/(?P[^/.]+)/$', app_views.map, name='map'), - url(r'^3d/project/(?P[^/.]+)/task/(?P[^/.]+)/$', app_views.model_display, name='model_display'), + path('', app_views.index, name='index'), + path('welcome', app_views.welcome, name='welcome'), + path('dashboard', app_views.dashboard, name='dashboard'), + re_path('map/project/(?P[^/.]+)/task/(?P[^/.]+)/', app_views.map, name='map'), + re_path('map/project/(?P[^/.]+)/', app_views.map, name='map'), + re_path('3d/project/(?P[^/.]+)/task/(?P[^/.]+)/', app_views.model_display, name='model_display'), - url(r'^public/task/(?P[^/.]+)/map/$', public_views.map, name='public_map'), - url(r'^public/task/(?P[^/.]+)/iframe/map/$', public_views.map_iframe, name='public_iframe_map'), - url(r'^public/task/(?P[^/.]+)/3d/$', public_views.model_display, name='public_3d'), - url(r'^public/task/(?P[^/.]+)/iframe/3d/$', public_views.model_display_iframe, name='public_iframe_3d'), - url(r'^public/task/(?P[^/.]+)/json/$', public_views.task_json, name='public_json'), + re_path('public/task/(?P[^/.]+)/map/', public_views.map, name='public_map'), + re_path('public/task/(?P[^/.]+)/iframe/map/', public_views.map_iframe, name='public_iframe_map'), + re_path('public/task/(?P[^/.]+)/3d/', public_views.model_display, name='public_3d'), + re_path('public/task/(?P[^/.]+)/iframe/3d/', public_views.model_display_iframe, name='public_iframe_3d'), + re_path('public/task/(?P[^/.]+)/json/', public_views.task_json, name='public_json'), - url(r'^processingnode/([\d]+)/$', app_views.processing_node, name='processing_node'), + re_path('processingnode/([\d]+)/', app_views.processing_node, name='processing_node'), - url(r'^api/', include("app.api.urls")), + path('api', include("app.api.urls")), - url(r'^plugins/(?P[^/.]+)/(.*)$', app_view_handler), + re_path('plugins/(?P[^/.]+)/(.*)', app_view_handler), - url(r'^about/$', app_views.about, name='about'), - url(r'^dev-tools/(?P.*)$', dev_views.dev_tools, name='dev_tools'), + path('about', app_views.about, name='about'), + re_path('dev-tools/(?P.*)', dev_views.dev_tools, name='dev_tools'), # TODO: add caching: https://docs.djangoproject.com/en/3.1/topics/i18n/translation/#note-on-performance - url(r'^jsi18n/', JavaScriptCatalog.as_view(packages=['app']), name='javascript-catalog'), - url(r'^i18n/', include('django.conf.urls.i18n')), + path('jsi18n', JavaScriptCatalog.as_view(packages=['app']), name='javascript-catalog'), + path('i18n', include('django.conf.urls.i18n')), ] + root_url_patterns() handler404 = app_views.handler404 diff --git a/app/views/app.py b/app/views/app.py index 4256234a0..6ad16abeb 100644 --- a/app/views/app.py +++ b/app/views/app.py @@ -11,7 +11,7 @@ from app.models import Project, Task from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext_lazy as _ from django import forms from app.views.utils import get_permissions from webodm import settings diff --git a/app/views/dev.py b/app/views/dev.py index 90e8f1d43..b474cd4ad 100644 --- a/app/views/dev.py +++ b/app/views/dev.py @@ -3,7 +3,7 @@ from django.shortcuts import render from django.contrib import messages -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext_lazy as _ from django import forms from webodm import settings from django.http import JsonResponse diff --git a/app/views/public.py b/app/views/public.py index 13bc5905b..fcfaf2205 100644 --- a/app/views/public.py +++ b/app/views/public.py @@ -2,7 +2,7 @@ from django.http import Http404 from django.http import JsonResponse from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext_lazy as _ from django.shortcuts import render from app.api.tasks import TaskSerializer diff --git a/coreplugins/editshortlinks/api.py b/coreplugins/editshortlinks/api.py index 120b43bf4..82d6607a8 100644 --- a/coreplugins/editshortlinks/api.py +++ b/coreplugins/editshortlinks/api.py @@ -9,7 +9,7 @@ from app.plugins import GlobalDataStore from django.http import Http404 from django.shortcuts import redirect -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ import logging diff --git a/locale b/locale index 751d81ab6..599f937f7 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit 751d81ab61a7bf4a335d29b1d129e68be1403d82 +Subproject commit 599f937f7e98064f6c201d01259ec14293f885a5 diff --git a/nodeodm/migrations/0010_alter_processingnode_available_options.py b/nodeodm/migrations/0010_alter_processingnode_available_options.py new file mode 100644 index 000000000..bb5ee85dc --- /dev/null +++ b/nodeodm/migrations/0010_alter_processingnode_available_options.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.14 on 2024-08-20 05:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nodeodm', '0009_auto_20210610_1850'), + ] + + operations = [ + migrations.AlterField( + model_name='processingnode', + name='available_options', + field=models.JSONField(default=dict, help_text='Description of the options that can be used for processing', verbose_name='Available Options'), + ), + ] diff --git a/nodeodm/models.py b/nodeodm/models.py index 270abf036..a06f55b8f 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals from django.db import models -from django.contrib.postgres import fields from django.utils import timezone from django.dispatch import receiver from guardian.models import GroupObjectPermissionBase @@ -25,7 +24,7 @@ class ProcessingNode(models.Model): api_version = models.CharField(verbose_name=_("API Version"), max_length=32, null=True, help_text=_("API version used by the node")) last_refreshed = models.DateTimeField(verbose_name=_("Last Refreshed"), null=True, help_text=_("When was the information about this node last retrieved?")) queue_count = models.PositiveIntegerField(verbose_name=_("Queue Count"), default=0, help_text=_("Number of tasks currently being processed by this node (as reported by the node itself)")) - available_options = fields.JSONField(verbose_name=_("Available Options"), default=dict, help_text=_("Description of the options that can be used for processing")) + available_options = models.JSONField(verbose_name=_("Available Options"), default=dict, help_text=_("Description of the options that can be used for processing")) token = models.CharField(verbose_name=_("Token"), max_length=1024, blank=True, default="", help_text=_("Token to use for authentication. If the node doesn't have authentication, you can leave this field blank.")) max_images = models.PositiveIntegerField(verbose_name=_("Max Images"), help_text=_("Maximum number of images accepted by this node."), blank=True, null=True) engine_version = models.CharField(verbose_name=_("Engine Version"), max_length=32, null=True, help_text=_("Engine version used by the node.")) diff --git a/requirements.txt b/requirements.txt index 10900789c..ca95efee2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,64 +1,31 @@ -amqp==2.5.2 -appdirs==1.4.0 -APScheduler==3.2.0 -billiard==3.6.3.0 -celery==4.4.0 -coreapi>=2.3.3 -Django==2.2.27 -django-appconf==1.0.2 +celery==5.4.0 +numpy==1.24.4 +scipy==1.10.1 +Django==4.2.14 django-codemirror2==0.2 -django-colorfield==0.1.15 -django-cors-headers==3.0.2 -django-filter==2.4.0 -django-guardian==1.4.9 -django-imagekit==4.0.1 -django-redis==4.12.1 -django-webpack-loader==0.6.0 -djangorestframework==3.13.1 -djangorestframework-jwt==1.9.0 -djangorestframework-guardian==0.3.0 -drf-nested-routers==0.11.1 -funcsigs==1.0.2 -futures==3.1.1 -gunicorn==19.7.1 -itypes==1.1.0 -kombu==4.6.7 -Markdown==3.3.4 -olefile==0.44 -openapi-codec==1.1.7 -packaging==16.8 -piexif==1.0.13 -pilkit==2.0 -Pillow==8.3.2 -pip-autoremove==0.9.0 -psycopg2-binary==2.8.6 -PyJWT==1.5.3 -pydantic==1.10.8 +django-colorfield==0.11.0 +django_filter==2.4.0 +django-guardian==2.4.0 +django_imagekit==5.0.0 +django_redis==4.12.1 +django-webpack-loader==0.7.0 +djangorestframework==3.15.2 +djangorestframework-simplejwt==5.3.1 +drf_nested_routers==0.94.1 +django-cors-headers==4.4.0 +drf_yasg==1.21.7 +piexif==1.1.3 pyodm==1.5.11 -pyparsing==2.4.7 -pytz==2020.1 -rcssmin==1.0.6 -redis==3.2.0 -requests-toolbelt==0.9.1 -requests>=2.21.0 -rfc3987==1.3.7 -rjsmin==1.0.12 -simplejson==3.10.0 -six==1.11.0 -strict-rfc3339==0.7 +pytz==2024.1 +requests==2.32.3 tzlocal==1.3 -uritemplate==3.0.0 -vine==1.3.0 -webcolors==1.5 -rio-tiler==2.1.2 -rio-color==1.0.4 -rio-cogeo==2.3.1 -rasterio==1.2.9 ; sys_platform == 'linux' or sys_platform == 'darwin' -https://github.com/OpenDroneMap/WebODM/releases/download/v1.9.7/rasterio-1.2.10-cp39-cp39-win_amd64.whl ; sys_platform == "win32" +morecantile==5.3.0 +rio_tiler==6.6.1 +rio_cogeo==5.3.3 +rasterio==1.3.10 +urllib3==1.26.19 +psycopg2==2.9.9 +# GDAL-python will be installed if Python3 is found during the process of building GDAL from source. +GDAL==3.8.5 ; sys_platform == 'darwin' https://github.com/OpenDroneMap/WebODM/releases/download/v1.9.7/GDAL-3.3.3-cp39-cp39-win_amd64.whl ; sys_platform == "win32" -Shapely==1.7.0 ; sys_platform == "win32" -eventlet==0.32.0 ; sys_platform == "win32" -pyopenssl==19.1.0 ; sys_platform == "win32" -numpy==1.26.2 -scipy==1.11.3 -drf-yasg==1.20.0 +pyproj==3.5.0 diff --git a/webodm/settings.py b/webodm/settings.py index 112b9d520..bb2d97824 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -15,6 +15,7 @@ import datetime import tzlocal +from datetime import timedelta from django.contrib.messages import constants as messages # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -108,7 +109,7 @@ 'guardian', 'rest_framework', 'rest_framework_nested', - 'drf_yasg', + # 'drf_yasg', 'webpack_loader', 'corsheaders', 'colorfield', @@ -137,6 +138,8 @@ # Database # https://docs.djangoproject.com/en/1.10/ref/settings/#databases +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + DATABASES = { 'default': { 'ENGINE': os.environ.get('WO_DATABASE_ENGINE', 'django.contrib.gis.db.backends.postgis'), @@ -314,24 +317,24 @@ 'rest_framework.permissions.DjangoObjectPermissions', ], 'DEFAULT_FILTER_BACKENDS': [ - 'rest_framework_guardian.filters.ObjectPermissionsFilter', 'django_filters.rest_framework.DjangoFilterBackend', 'rest_framework.filters.OrderingFilter', ], 'DEFAULT_AUTHENTICATION_CLASSES': ( - 'rest_framework.authentication.SessionAuthentication', 'rest_framework.authentication.BasicAuthentication', - 'rest_framework_jwt.authentication.JSONWebTokenAuthentication', - 'app.api.authentication.JSONWebTokenAuthenticationQS', + 'rest_framework_simplejwt.authentication.JWTAuthentication', ), 'PAGE_SIZE': 10, 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', } -JWT_AUTH = { - 'JWT_EXPIRATION_DELTA': datetime.timedelta(hours=6), +SIMPLE_JWT = { + "ACCESS_TOKEN_LIFETIME": timedelta(days=7), + "REFRESH_TOKEN_LIFETIME": timedelta(days=1), + "UPDATE_LAST_LOGIN": True } + # Celery CELERY_BROKER_URL = os.environ.get('WO_BROKER', 'redis://localhost') CELERY_RESULT_BACKEND = os.environ.get('WO_BROKER', 'redis://localhost') diff --git a/webodm/urls.py b/webodm/urls.py index d92938ffb..f42b9ff6f 100644 --- a/webodm/urls.py +++ b/webodm/urls.py @@ -15,46 +15,45 @@ """ import os -from django.conf.urls import include, url -from django.urls import re_path +from django.urls import re_path, include, path from django.contrib import admin from . import settings from django.views.static import serve from rest_framework import permissions -from drf_yasg.views import get_schema_view -from drf_yasg import openapi +# from drf_yasg.views import get_schema_view +# from drf_yasg import openapi admin.site.site_header = 'WebODM Administration' -schema_view = get_schema_view( - openapi.Info( - title="WebODM API", - default_version='v1.0.0', - description="WebODM API", - #terms_of_service="", - #contact=openapi.Contact(email=""), - ), - public=True, - permission_classes=[permissions.AllowAny], -) +# schema_view = get_schema_view( +# openapi.Info( +# title="WebODM API", +# default_version='v1.0.0', +# description="WebODM API", +# #terms_of_service="", +# #contact=openapi.Contact(email=""), +# ), +# public=True, +# permission_classes=[permissions.AllowAny], +# ) urlpatterns = [ - url(r'^', include('app.urls')), - url(r'^', include('django.contrib.auth.urls')), - url(r'^admin/', admin.site.urls), - re_path(r'^swagger(?P\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0), name='schema-json'), - re_path(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), - re_path(r'^redoc/$', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'), + path('', include('app.urls')), + path('', include('django.contrib.auth.urls')), + path('admin/', admin.site.urls), + # re_path(r'^swagger(?P\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0), name='schema-json'), + # re_path(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), + # re_path(r'^redoc/$', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'), ] if settings.DEBUG or settings.FORCE_MEDIA_STATICFILES: urlpatterns += [ # Expose imagekit generated files and settings file uploads - url(r'^media/CACHE/(?P.*)$', serve, { + re_path(r'^media/CACHE/(?P.*)$', serve, { 'document_root': os.path.join(settings.MEDIA_ROOT, 'CACHE') }), - url(r'^media/settings/(?P.*)$', serve, { + re_path(r'^media/settings/(?P.*)$', serve, { 'document_root': os.path.join(settings.MEDIA_ROOT, 'settings') }), From f32feae0dee0ad952471149109a5fb3a089b6767 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Mon, 26 Aug 2024 02:18:39 +0800 Subject: [PATCH 13/16] feat: create postgis_raster extensions with Django --- app/migrations/0001_initial.py | 2 ++ db/docker-entrypoint.sh | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/migrations/0001_initial.py b/app/migrations/0001_initial.py index a96fd4040..facd2f192 100644 --- a/app/migrations/0001_initial.py +++ b/app/migrations/0001_initial.py @@ -8,6 +8,7 @@ from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.contrib.postgres.operations import CreateExtension class Migration(migrations.Migration): @@ -21,6 +22,7 @@ class Migration(migrations.Migration): ] operations = [ + CreateExtension("postgis_raster"), migrations.CreateModel( name='ImageUpload', fields=[ diff --git a/db/docker-entrypoint.sh b/db/docker-entrypoint.sh index eb6d88f86..9c6e940e2 100755 --- a/db/docker-entrypoint.sh +++ b/db/docker-entrypoint.sh @@ -321,7 +321,6 @@ _main() { fi exec "$@" - docker_process_sql --dbname="$POSTGRES_DB" <<<'CREATE extension postgis_raster CASCADE;' } From 44fc1bab964d836ccfda91ad0bc26a92e2897aab Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 27 Aug 2024 08:44:07 +0800 Subject: [PATCH 14/16] fix: add slash behind path routes --- app/api/urls.py | 2 +- app/urls.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/api/urls.py b/app/api/urls.py index f4e6c11e1..c2e3ca191 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -31,7 +31,7 @@ admin_router.register(r'admin/profiles', AdminProfileViewSet, basename='admin-profiles') urlpatterns = [ - re_path(r'processingnodes/options/$', ProcessingNodeOptionsView.as_view()), + path('processingnodes/options/', ProcessingNodeOptionsView.as_view()), re_path(r'^', include(router.urls)), re_path(r'^', include(tasks_router.urls)), diff --git a/app/urls.py b/app/urls.py index f651ae014..a7e968c51 100644 --- a/app/urls.py +++ b/app/urls.py @@ -21,8 +21,8 @@ urlpatterns = [ path('', app_views.index, name='index'), - path('welcome', app_views.welcome, name='welcome'), - path('dashboard', app_views.dashboard, name='dashboard'), + path('welcome/', app_views.welcome, name='welcome'), + path('dashboard/', app_views.dashboard, name='dashboard'), re_path('map/project/(?P[^/.]+)/task/(?P[^/.]+)/', app_views.map, name='map'), re_path('map/project/(?P[^/.]+)/', app_views.map, name='map'), re_path('3d/project/(?P[^/.]+)/task/(?P[^/.]+)/', app_views.model_display, name='model_display'), @@ -35,16 +35,16 @@ re_path('processingnode/([\d]+)/', app_views.processing_node, name='processing_node'), - path('api', include("app.api.urls")), + path('api/', include("app.api.urls")), re_path('plugins/(?P[^/.]+)/(.*)', app_view_handler), - path('about', app_views.about, name='about'), + path('about/', app_views.about, name='about'), re_path('dev-tools/(?P.*)', dev_views.dev_tools, name='dev_tools'), # TODO: add caching: https://docs.djangoproject.com/en/3.1/topics/i18n/translation/#note-on-performance - path('jsi18n', JavaScriptCatalog.as_view(packages=['app']), name='javascript-catalog'), - path('i18n', include('django.conf.urls.i18n')), + path('jsi18n/', JavaScriptCatalog.as_view(packages=['app']), name='javascript-catalog'), + path('i18n/', include('django.conf.urls.i18n')), ] + root_url_patterns() handler404 = app_views.handler404 From fe395466ba0de1e650f3071c54b46adc109ee99e Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 27 Aug 2024 14:06:46 +0800 Subject: [PATCH 15/16] refactor: update Docker base image to Ubuntu 22.04 --- Dockerfile | 32 +++++++++++++++++--------------- db/Dockerfile | 7 +++---- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 57b608771..0f2ef5b35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 MAINTAINER Piero Toffanin ARG TEST_BUILD @@ -6,7 +6,7 @@ ARG DEBIAN_FRONTEND=noninteractive ENV PYTHONUNBUFFERED=1 ENV PYTHONPATH=$PYTHONPATH:/webodm ENV NODE_MAJOR=20 -ENV PYTHON_MAJOR=3.9 +ENV PYTHON_MAJOR=3.10 ENV GDAL_VERSION=3.8.5 ENV LD_LIBRARY_PATH=/usr/local/lib @@ -19,47 +19,49 @@ RUN apt-get -o Acquire::Retries=3 -qq update > /dev/null && \ # Install PDAL, letsencrypt, psql, cron apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends binutils pdal certbot gettext tzdata libproj-dev libpq-dev > /dev/null && \ - + # Install Python in target version - apt-get -qq autoremove -y python3 && \ - apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends python$PYTHON_MAJOR-dev python$PYTHON_MAJOR-full && \ + apt-get -qq autoremove -y python3 > /dev/null && \ + apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends python$PYTHON_MAJOR-dev python$PYTHON_MAJOR-full > /dev/null && \ ln -s /usr/bin/python$PYTHON_MAJOR /usr/bin/python && \ curl https://bootstrap.pypa.io/get-pip.py | python && \ - + echo $(pip -V) && \ - echo $(python -V) && \ + echo $(python -V) && \ # Build GDAL from source wget --no-check-certificate -q https://github.com/OSGeo/gdal/releases/download/v$GDAL_VERSION/gdal-$GDAL_VERSION.tar.gz && \ tar -xzf gdal-$GDAL_VERSION.tar.gz && \ cd gdal-$GDAL_VERSION && mkdir build && cd build && \ - cmake .. && cmake --build . -j$(nproc) --target install && \ + cmake .. > /dev/null && cmake --build . -j$(nproc) --target install > /dev/null && \ cd / && rm -rf gdal-$GDAL_VERSION gdal-$GDAL_VERSION.tar.gz && \ # Install pip reqs cd /webodm && \ - pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \ pip install --quiet -U pip && \ - pip install --quiet -r requirements.txt "boto3==1.34.145" && \ + pip install -r requirements.txt "boto3==1.34.145" > /dev/null && \ # Install Node.js using new Node install method - apt-get -o Acquire::Retries=3 -qq install -y ca-certificates gnupg && \ + apt-get -o Acquire::Retries=3 -qq install -y ca-certificates gnupg > /dev/null && \ mkdir -p /etc/apt/keyrings && \ curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y nodejs && \ + apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y nodejs > /dev/null && \ + # Setup cron - apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends nginx cron && \ + apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends nginx cron > /dev/null && \ ln -s /webodm/nginx/crontab /var/spool/cron/crontabs/root && chmod 0644 /webodm/nginx/crontab && service cron start && chmod +x /webodm/nginx/letsencrypt-autogen.sh && \ /webodm/nodeodm/setup.sh && /webodm/nodeodm/cleanup.sh && cd /webodm && \ - npm install --quiet -g webpack@5.89.0 && npm install --quiet -g webpack-cli@5.1.4 && npm install --quiet && webpack --mode production && \ + npm install --quiet -g webpack@5.89.0 > /dev/null && npm install --quiet -g webpack-cli@5.1.4 > /dev/null && npm install --quiet > /dev/null && webpack --mode production > /dev/null && \ echo "UTC" > /etc/timezone && \ python manage.py collectstatic --noinput && \ python manage.py rebuildplugins && \ python manage.py translate build --safe && \ + # Cleanup apt-get remove -y g++ python2 && apt-get autoremove -y && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ - rm /webodm/webodm/secret_key.py + rm /webodm/webodm/secret_key.py && \ + mkdir -p /webodm/app/media/tmp VOLUME /webodm/app/media diff --git a/db/Dockerfile b/db/Dockerfile index d5bd85b94..99010dd39 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 MAINTAINER Piero Toffanin ENV POSTGRES_PASSWORD=postgres @@ -20,7 +20,7 @@ COPY docker-entrypoint.sh /usr/local/bin/ RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat # Setup system -RUN apt-get update; \ +RUN apt-get update > /dev/null; \ set -eux; \ groupadd -r postgres --gid=999; \ useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; \ @@ -64,12 +64,11 @@ RUN apt-get update; \ # this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA"; \ # Build PostGIS from source - apt-get install -o Acquire::Retries=3 -y --no-install-recommends libgdal-dev libjson-c-dev protobuf-c-compiler libprotobuf-c-dev; \ + apt-get install -o Acquire::Retries=3 -y --no-install-recommends libgdal-dev libjson-c-dev protobuf-c-compiler libprotobuf-c-dev > /dev/null; \ cd /staging; \ wget --no-check-certificate -q https://download.osgeo.org/postgis/source/postgis-$POSTGIS_VERSION.tar.gz; \ wget --no-check-certificate -q -O /usr/include/json-c/json_object_private.h https://raw.githubusercontent.com/json-c/json-c/json-c-0.16/json_object_private.h; \ tar -zxf postgis-$POSTGIS_VERSION.tar.gz; \ - sed -i 's/#error.*/#define ACCEPT_USE_OF_DEPRECATED_PROJ_API_H 1/' /usr/include/proj_api.h; \ cd /staging/postgis-$POSTGIS_VERSION; \ ./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config --with-gdalconfig=/usr/bin/gdal-config > /dev/null; \ make; \ From dc6a15b7a912ca5c9cf04625921e1593f39c9830 Mon Sep 17 00:00:00 2001 From: Ntkskwk Date: Tue, 27 Aug 2024 14:11:25 +0800 Subject: [PATCH 16/16] refactor: update docker actions --- .github/workflows/build-and-publish.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index bc756fd34..f59a334fb 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -16,26 +16,26 @@ jobs: with: submodules: 'recursive' - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v5 with: images: opendronemap/webodm_webapp - tag-semver: | - {{version}} + tags: | + type=semver,pattern={{version}} - name: Build and push Docker image id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: - file: ./Dockerfile + context: . platforms: linux/amd64,linux/arm64 push: true no-cache: true