Merge pull request #661 from getodk/next

Release v2024.2

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     machine:
-      image: ubuntu-2004:202201-02
+      image: ubuntu-2004:current
 
     steps:
       - checkout
@@ -37,4 +37,9 @@ jobs:
               appropriate/curl -4 --insecure --retry 20 --retry-delay 2 --retry-connrefused https://localhost/v1/projects \
               | tee /dev/tty \
               | grep -q '\[\]'
-            docker compose exec -T service npx pm2 list | grep -c "online" | grep -q 4 || exit 1
+      - run:
+          name: Verify pm2
+          command: |
+            docker compose exec -T service npx pm2 list \
+              | tee /dev/tty \
+              | grep -c "online" | grep -q 4

.env.template

@@ -39,3 +39,9 @@ HTTPS_PORT=443
 # SENTRY_ORG_SUBDOMAIN=
 # SENTRY_KEY=
 # SENTRY_PROJECT=
+
+# Optional: configure S3-compatible storage for binary files
+# S3_SERVER=
+# S3_ACCESS_KEY=
+# S3_SECRET_KEY=
+# S3_BUCKET_NAME=

.github/workflows/ghcr.yml

@@ -0,0 +1,58 @@
+name: GHCR
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+    tags: ["v*.*.*"]
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      matrix:
+        image: [nginx, service]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+          submodules: recursive
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }}
+
+      - name: Set up QEMU emulator for multi-arch images
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push ${{ matrix.image }} Docker image
+        uses: docker/build-push-action@v5
+        with:
+            file: ${{ matrix.image }}.dockerfile
+            context: .
+            push: true
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
+            platforms: 'linux/amd64,linux/arm64'

.github/workflows/test-nginx.yml

@@ -0,0 +1,24 @@
+name: Test nginx config
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  build:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 20.17.0
+    - run: cd test && npm i
+    - run: cd test && ./run-tests.sh
+
+    - if: always()
+      run: docker logs test-nginx-1
+    - if: always()
+      run: docker logs test-service-1
+    - if: always()
+      run: docker logs test-enketo-1

docker-compose.yml

@@ -26,7 +26,7 @@ services:
       POSTGRES_PASSWORD: odk
       POSTGRES_DB: odk
   mail:
-    image: "ixdotai/smtp:v0.5.2"
+    image: "ixdotai/smtp:v0.5.4"
     volumes:
       - ./files/mail/rsa.private:/etc/exim4/dkim.key.temp:ro
     environment:
@@ -70,15 +70,17 @@ services:
       - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137}
       - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee}
       - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632}
+      - S3_SERVER=${S3_SERVER:-}
+      - S3_ACCESS_KEY=${S3_ACCESS_KEY:-}
+      - S3_SECRET_KEY=${S3_SECRET_KEY:-}
+      - S3_BUCKET_NAME=${S3_BUCKET_NAME:-}
     command: [ "wait-for-it", "${DB_HOST:-postgres14}:5432", "--", "./start-odk.sh" ]
     restart: always
     logging:
       driver: local
   nginx:
     build:
       context: .
-      args:
-        - OIDC_ENABLED=${OIDC_ENABLED:-false}
       dockerfile: nginx.dockerfile
     depends_on:
       - service
@@ -90,6 +92,11 @@ services:
       - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137}
       - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee}
       - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632}
+      - OIDC_ENABLED=${OIDC_ENABLED:-false}
+    volumes:
+      - ./files/local/customssl/:/etc/customssl/live/local/:ro
+      - ./files/nginx/odk.conf.template:/usr/share/odk/nginx/odk.conf.template:ro
+      - ./files/nginx/client-config.json.template:/usr/share/odk/nginx/client-config.json.template:ro
     ports:
       - "${HTTP_PORT:-80}:80"
       - "${HTTPS_PORT:-443}:443"
@@ -101,7 +108,7 @@ services:
       options:
         max-file: "30"
   pyxform:
-    image: 'ghcr.io/getodk/pyxform-http:v2.0.3'
+    image: 'ghcr.io/getodk/pyxform-http:v2.1.0'
     restart: always
   secrets:
     volumes:

enketo.dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/enketo/enketo:7.2.0
+FROM ghcr.io/enketo/enketo:7.4.0
 
 ENV ENKETO_SRC_DIR=/srv/src/enketo/packages/enketo-express
 WORKDIR ${ENKETO_SRC_DIR}

files/nginx/client-config.json.template

@@ -0,0 +1,3 @@
+{
+  "oidcEnabled": $OIDC_ENABLED
+}

files/nginx/odk.conf.template

@@ -67,6 +67,10 @@ server {
       include /usr/share/odk/nginx/common-headers.conf;
       add_header Cache-Control no-cache;
     }
+    location /client-config.json {
+      include /usr/share/odk/nginx/common-headers.conf;
+      add_header Cache-Control no-cache;
+    }
     location /index.html {
       include /usr/share/odk/nginx/common-headers.conf;
       add_header Cache-Control no-cache;

files/nginx/setup-odk.sh

@@ -1,5 +1,15 @@
 #!/bin/bash
 
+
+echo "writing client config..."
+if [[ $OIDC_ENABLED != 'true' ]] && [[ $OIDC_ENABLED != 'false' ]]; then
+  echo 'OIDC_ENABLED must be either true or false'
+  exit 1
+fi
+
+envsubst < /usr/share/odk/nginx/client-config.json.template > /usr/share/nginx/html/client-config.json
+
+
 DH_PATH=/etc/dh/nginx.pem
 if [ "$SSL_TYPE" != "upstream" ] && [ ! -s "$DH_PATH" ]; then
   openssl dhparam -out "$DH_PATH" 2048
@@ -17,7 +27,9 @@ fi
 
 # start from fresh templates in case ssl type has changed
 echo "writing fresh nginx templates..."
+# redirector.conf gets deleted if using upstream SSL so copy it back
 cp /usr/share/odk/nginx/redirector.conf /etc/nginx/conf.d/redirector.conf
+
 CNAME=$( [ "$SSL_TYPE" = "customssl" ] && echo "local" || echo "$DOMAIN") \
 envsubst '$SSL_TYPE $CNAME $SENTRY_ORG_SUBDOMAIN $SENTRY_KEY $SENTRY_PROJECT' \
   < /usr/share/odk/nginx/odk.conf.template \

files/prebuild/build-frontend.sh

@@ -1,4 +1,4 @@
 #!/bin/bash -eu
 cd client
 npm clean-install --no-audit --fund=false --update-notifier=false
-VUE_APP_OIDC_ENABLED="$OIDC_ENABLED" npm run build
+npm run build

files/service/config.json.template

@@ -44,6 +44,13 @@
         "orgSubdomain": "${SENTRY_ORG_SUBDOMAIN}",
         "key": "${SENTRY_KEY}",
         "project": "${SENTRY_PROJECT}"
+      },
+      "s3blobStore": {
+        "server": "${S3_SERVER}",
+        "accessKey": "${S3_ACCESS_KEY}",
+        "secretKey": "${S3_SECRET_KEY}",
+        "bucketName": "${S3_BUCKET_NAME}",
+        "requestTimeout": 60000
       }
     }
   }

files/service/crontab

@@ -1,3 +1,5 @@
-0 4 * * * root /usr/odk/purge-forms.sh
+0 5 * * * root /usr/odk/upload-blobs.sh
+0 4 * * * root /usr/odk/purge.sh
 0 3 * * * root /usr/odk/run-analytics.sh
+0 2 * * * root /usr/odk/process-backlog.sh
 0 1 * * 0 root /usr/odk/reap-sessions.sh

files/service/scripts/process-backlog.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eu
+
+cd /usr/odk
+/usr/local/bin/node lib/bin/process-backlog.js >/proc/1/fd/1 2>/proc/1/fd/2
+

files/service/scripts/purge.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eu
+
+cd /usr/odk
+/usr/local/bin/node lib/bin/purge.js >/proc/1/fd/1 2>/proc/1/fd/2
+

files/service/scripts/start-odk.sh

@@ -4,7 +4,7 @@ echo "generating local service configuration.."
 
 ENKETO_API_KEY=$(cat /etc/secrets/enketo-api-key) \
 BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https://"${DOMAIN}":"${HTTPS_PORT}" ) \
-envsubst '$DOMAIN $BASE_URL $SYSADMIN_EMAIL $ENKETO_API_KEY $DB_HOST $DB_USER $DB_PASSWORD $DB_NAME $DB_SSL $EMAIL_FROM $EMAIL_HOST $EMAIL_PORT $EMAIL_SECURE $EMAIL_IGNORE_TLS $EMAIL_USER $EMAIL_PASSWORD $OIDC_ENABLED $OIDC_ISSUER_URL $OIDC_CLIENT_ID $OIDC_CLIENT_SECRET $SENTRY_ORG_SUBDOMAIN $SENTRY_KEY $SENTRY_PROJECT' \
+envsubst '$DOMAIN $BASE_URL $SYSADMIN_EMAIL $ENKETO_API_KEY $DB_HOST $DB_USER $DB_PASSWORD $DB_NAME $DB_SSL $EMAIL_FROM $EMAIL_HOST $EMAIL_PORT $EMAIL_SECURE $EMAIL_IGNORE_TLS $EMAIL_USER $EMAIL_PASSWORD $OIDC_ENABLED $OIDC_ISSUER_URL $OIDC_CLIENT_ID $OIDC_CLIENT_SECRET $SENTRY_ORG_SUBDOMAIN $SENTRY_KEY $SENTRY_PROJECT $S3_SERVER $S3_ACCESS_KEY $S3_SECRET_KEY $S3_BUCKET_NAME' \
     < /usr/share/odk/config.json.template \
     > /usr/odk/config/local.json
 

files/service/scripts/upload-blobs.sh

@@ -0,0 +1,4 @@
+#!/bin/sh -eu
+
+cd /usr/odk
+/usr/local/bin/node lib/bin/s3.js upload-pending >/proc/1/fd/1 2>/proc/1/fd/2

nginx.dockerfile

@@ -1,34 +1,38 @@
-FROM node:20.12.2-slim as intermediate
+FROM node:20.17.0-slim AS intermediate
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
         git \
+        gettext-base \
     && rm -rf /var/lib/apt/lists/*
 
 COPY ./ ./
 RUN files/prebuild/write-version.sh
-ARG OIDC_ENABLED
-RUN OIDC_ENABLED="$OIDC_ENABLED" files/prebuild/build-frontend.sh
+RUN files/prebuild/build-frontend.sh
 
 
 
 # when upgrading, look for upstream changes to redirector.conf
 # also, confirm setup-odk.sh strips out HTTP-01 ACME challenge location
-FROM jonasal/nginx-certbot:5.0.1
+FROM jonasal/nginx-certbot:5.4.0
 
 EXPOSE 80
 EXPOSE 443
 
-VOLUME [ "/etc/dh", "/etc/selfsign", "/etc/nginx/conf.d" ]
-ENTRYPOINT [ "/bin/bash", "/scripts/setup-odk.sh" ]
+# Persist Diffie-Hellman parameters and/or selfsign key
+VOLUME [ "/etc/dh", "/etc/selfsign" ]
 
 RUN apt-get update && apt-get install -y netcat-openbsd
 
 RUN mkdir -p /usr/share/odk/nginx/
 
 COPY files/nginx/setup-odk.sh /scripts/
-COPY files/local/customssl/*.pem /etc/customssl/live/local/
-COPY files/nginx/*.conf* /usr/share/odk/nginx/
+RUN chmod +x /scripts/setup-odk.sh
+
+COPY files/nginx/redirector.conf /usr/share/odk/nginx/
+COPY files/nginx/common-headers.conf /usr/share/odk/nginx/
 
 COPY --from=intermediate client/dist/ /usr/share/nginx/html
 COPY --from=intermediate /tmp/version.txt /usr/share/nginx/html
+
+ENTRYPOINT [ "/scripts/setup-odk.sh" ]

secrets.dockerfile

@@ -1,3 +1,3 @@
-FROM node:20.12.2-slim
+FROM node:20.17.0-slim
 
 COPY files/enketo/generate-secrets.sh ./

service.dockerfile

@@ -1,8 +1,8 @@
-ARG node_version=20.12.2
+ARG node_version=20.17.0
 
 
 
-FROM node:${node_version}-slim as pgdg
+FROM node:${node_version}-slim AS pgdg
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
         ca-certificates \
@@ -17,7 +17,7 @@ RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(grep -oP 'VERSION_CODEN
 
 
 
-FROM node:${node_version}-slim as intermediate
+FROM node:${node_version}-slim AS intermediate
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
         git \

test/.gitignore

@@ -0,0 +1 @@
+/node_modules/

test/files/nginx-test/acme-challenge

@@ -0,0 +1 @@
+hi-from-letsencrypt

test/files/nginx-test/http_root/index.html

@@ -0,0 +1 @@
+hi:/index.html

test/files/nginx-test/http_root/should-be-cached.txt

@@ -0,0 +1 @@
+hi:/should-be-cached.txt

test/files/nginx-test/http_root/version.txt

@@ -0,0 +1 @@
+hi:/version.txt

test/mock-http-server/.gitignore

@@ -0,0 +1 @@
+/node_modules/

test/mock-http-server/index.js

@@ -0,0 +1,37 @@
+const express = require('express');
+
+const port = process.env.PORT || 80;
+const log = (...args) => console.log('[mock-http-server]', ...args);
+
+const requests = [];
+
+const app = express();
+
+app.get('/health',      withStdLogging((req, res) => res.send('OK')));
+app.get('/request-log', withStdLogging((req, res) => res.json(requests)));
+app.get('/reset',       withStdLogging((req, res) => {
+  requests.length = 0;
+  res.json('OK');
+}));
+
+app.get('/*', ok('GET'));
+app.post('/*', ok('POST'));
+// TODO add more methods as required
+
+app.listen(port, () => {
+  log(`Listening on port: ${port}`);
+});
+
+function withStdLogging(fn) {
+  return (req, res) => {
+    console.log(new Date(), req.method, req.path);
+    return fn(req, res);
+  };
+}
+
+function ok(method) {
+  return withStdLogging((req, res) => {
+    requests.push({ method, path:req.path });
+    res.send('OK');
+  });
+}