Platform upgrades (#1469)

.dockerignore

@@ -9,6 +9,7 @@
 !pyproject.toml
 
 !build.js
+!imagemagick-policy.xml
 !package.json
 !package-lock.json
 !setup.cfg

.github/workflows/tests.yml

@@ -9,7 +9,7 @@ jobs:
       contents: read
 
     container:
-      image: ghcr.io/weasyl/ci-base-image@sha256:9a366824361bf3123b28364ab6a4bd039ac765fafd5a1baab07f5c7c4485c007
+      image: ghcr.io/weasyl/ci-base-image@sha256:960e621bcdde15bdf9b1dd29c8b45dec93927aa67f2e7f4ab3f763a16f2b99f4
       options: --user 1001
 
     services:
@@ -61,6 +61,7 @@ jobs:
         run: |
           cp ci/site.config.txt config/
           cp config/weasyl-staff.example.py config/weasyl-staff.py
+          cat imagemagick-policy.xml > /usr/etc/ImageMagick-6/policy.xml
           printf %.8s "$GITHUB_SHA" > version.txt
 
       - name: Build assets

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM docker.io/library/node:16-alpine3.16 AS asset-builder
+FROM docker.io/library/node:18-alpine3.20 AS asset-builder
 RUN --mount=type=cache,id=apk,target=/var/cache/apk,sharing=locked \
     ln -s /var/cache/apk /etc/apk/cache && apk upgrade && apk add \
     sassc runit
@@ -16,34 +16,88 @@ COPY assets assets
 RUN node build.js
 
 
-FROM docker.io/library/alpine:3.16 AS mozjpeg
+FROM docker.io/library/alpine:3.20 AS mozjpeg
 RUN --mount=type=cache,id=apk,target=/var/cache/apk,sharing=locked \
     ln -s /var/cache/apk /etc/apk/cache && apk upgrade && apk add \
     musl-dev gcc make \
     cmake nasm
 RUN adduser -S build -h /mozjpeg-build
 WORKDIR /mozjpeg-build
 USER build
-RUN wget https://github.com/mozilla/mozjpeg/archive/refs/tags/v4.0.3.tar.gz
-RUN echo '59c2d65af28d4ef68b9e5c85215cf3b26f4ac5c98e3ae76ba5febceec97fa5ab28cc13496e3f039f11cae767c5466bbf798038f83b310134c13d2e9a6bf5467e  v4.0.3.tar.gz' | sha512sum -c && tar xf v4.0.3.tar.gz
-WORKDIR /mozjpeg-build/mozjpeg-4.0.3
+RUN wget https://github.com/mozilla/mozjpeg/archive/refs/tags/v4.1.5.tar.gz
+RUN echo '90e1b0067740b161398d908e90b976eccc2ee7174496ce9693ba3cdf4727559ecff39744611657d847dd83164b80993152739692a5233aca577ebd052efaf501  v4.1.5.tar.gz' | sha512sum -c && tar xf v4.1.5.tar.gz
+WORKDIR /mozjpeg-build/mozjpeg-4.1.5
 RUN cmake -DENABLE_STATIC=0 -DPNG_SUPPORTED=0 -DCMAKE_INSTALL_PREFIX=/mozjpeg-build/package-root .
 RUN cmake --build . --parallel --target install
 
 
-FROM docker.io/library/python:3.10-alpine3.16 AS bdist
-# imagemagick6-dev: sanpera
-# libjpeg-turbo-dev, libwebp-dev, zlib-dev: Pillow
+FROM docker.io/library/alpine:3.20 AS imagemagick6-build
+RUN --network=none adduser -S build -h /imagemagick6-build
+RUN --mount=type=cache,id=apk,target=/var/cache/apk,sharing=locked \
+    ln -s /var/cache/apk /etc/apk/cache && apk upgrade && apk add \
+    musl-dev gcc make \
+    lcms2-dev \
+    libpng-dev \
+    libxml2-dev \
+    libwebp-dev \
+    zlib-dev
+WORKDIR /imagemagick6-build
+COPY --from=mozjpeg --chown=root:root /mozjpeg-build/package-root/include/ /usr/include/
+COPY --from=mozjpeg --chown=root:root /mozjpeg-build/package-root/lib64/ /usr/lib/
+USER build
+RUN wget https://imagemagick.org/archive/releases/ImageMagick-6.9.13-17.tar.xz
+RUN --network=none echo '655d8faa4387fd840e2a082633f55d961b3f6bb3c4909debec8272e7abbf9da4afb9994628a493229b41cbc17baba765812cf3d02fc69dd0eb2f2511e85b31c0  ImageMagick-6.9.13-17.tar.xz' | sha512sum -c && xzcat ImageMagick-6.9.13-17.tar.xz | tar x
+WORKDIR /imagemagick6-build/ImageMagick-6.9.13-17
+# `CFLAGS`, `LDFLAGS`: abuild defaults with `-O2` instead of `-Os`, as used by Alpine 3.16 imagemagick6 package
+# no `--enable-hdri`: doesn’t seem to work with sanpera, even though we’re building it from source?
+# `--with-cache=32GiB`: let other places (like policy.xml) set the limit, and definitely don’t choose whether to write files based on detecting available memory
+# `--with-xml`: for XMP metadata
+RUN --network=none ./configure \
+    --prefix=/usr \
+    --with-security-policy=websafe \
+    --disable-static \
+    --enable-shared \
+    --disable-deprecated \
+    --disable-docs \
+    --disable-cipher \
+    --with-cache=32GiB \
+    --without-magick-plus-plus \
+    --without-perl \
+    --without-bzlib \
+    --without-dps \
+    --without-djvu \
+    --without-flif \
+    --without-freetype \
+    --without-heic \
+    --without-jbig \
+    --without-openjp2 \
+    --without-lqr \
+    --without-lzma \
+    --without-openexr \
+    --without-pango \
+    --without-raw \
+    --without-raqm \
+    --without-tiff \
+    --without-wmf \
+    --with-xml \
+    --without-x \
+    --without-zstd \
+    CFLAGS='-O2 -fstack-clash-protection -Wformat -Werror=format-security' \
+    LDFLAGS='-Wl,--as-needed,-O1,--sort-common'
+RUN --network=none make -j"$(nproc)"
+RUN --network=none make install DESTDIR="$HOME/package-root"
+
+
+FROM docker.io/library/python:3.10-alpine3.20 AS bdist
+# libwebp-dev, zlib-dev: Pillow
 # libffi-dev, openssl-dev: cryptography
 # libmemcached-dev: pylibmc
 # libxml2-dev, libxslt-dev: lxml
 # postgresql-dev: psycopg2
 RUN --mount=type=cache,id=apk,target=/var/cache/apk,sharing=locked \
     ln -s /var/cache/apk /etc/apk/cache && apk upgrade && apk add \
     musl-dev gcc g++ make \
-    imagemagick6-dev \
     libffi-dev \
-    libjpeg-turbo-dev \
     libmemcached-dev \
     libwebp-dev \
     libxml2-dev libxslt-dev \
@@ -55,6 +109,7 @@ WORKDIR /weasyl
 USER weasyl
 COPY --from=mozjpeg --chown=root:root /mozjpeg-build/package-root/include/ /usr/include/
 COPY --from=mozjpeg --chown=root:root /mozjpeg-build/package-root/lib64/ /usr/lib/
+COPY --from=imagemagick6-build --chown=root:root /imagemagick6-build/package-root/ /
 COPY poetry-requirements.txt ./
 RUN --network=none python3 -m venv .poetry-venv
 RUN --mount=type=cache,id=pip,target=/weasyl/.cache/pip,sharing=locked,uid=1000 \
@@ -83,19 +138,21 @@ RUN --mount=type=cache,id=poetry,target=/weasyl/.cache/pypoetry,sharing=locked,u
     .poetry-venv/bin/poetry install --only=dev
 
 
-FROM docker.io/library/python:3.10-alpine3.16 AS package
+FROM docker.io/library/python:3.10-alpine3.20 AS package
+# gcc (libgomp), lcms2, libpng, libxml2, libwebp*: ImageMagick
 RUN --mount=type=cache,id=apk,target=/var/cache/apk,sharing=locked \
     ln -s /var/cache/apk /etc/apk/cache && apk upgrade && apk add \
-    imagemagick6-libs \
+    gcc lcms2 libpng libxml2 libwebpdemux libwebpmux \
     libffi \
     libmemcached-libs \
     libpq \
-    libwebp \
     libxslt
 RUN adduser -S weasyl -h /weasyl
 WORKDIR /weasyl
 USER weasyl
 COPY --from=mozjpeg --chown=root:root /mozjpeg-build/package-root/lib64/ /usr/lib/
+COPY --from=imagemagick6-build --chown=root:root /imagemagick6-build/package-root/ /
+COPY --chown=root:root imagemagick-policy.xml /usr/etc/ImageMagick-6/policy.xml
 
 COPY --from=bdist /weasyl/.venv .venv
 COPY --from=assets /weasyl-build/build build
@@ -117,7 +174,7 @@ COPY assets assets
 CMD pytest -x libweasyl.test libweasyl.models.test && pytest -x weasyl.test
 STOPSIGNAL SIGINT
 
-FROM docker.io/library/alpine:3.16 AS flake8
+FROM docker.io/library/alpine:3.20 AS flake8
 RUN --mount=type=cache,id=apk,target=/var/cache/apk,sharing=locked \
     ln -s /var/cache/apk /etc/apk/cache && apk upgrade && apk add \
     py3-flake8

containers/postgres/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/library/postgres:13-alpine3.16
+FROM docker.io/library/postgres:13-alpine3.20
 COPY \
     00-hstore.sql \
     01-test.sql \

docker-compose.yml

@@ -68,6 +68,8 @@ services:
         target: /weasyl/storage/temp
       - type: tmpfs
         target: /tmp
+      - type: tmpfs
+        target: /magick-tmp
     networks:
       - external-web
       - nginx-web
@@ -77,7 +79,7 @@ services:
     read_only: false
 
   memcached:
-    image: docker.io/library/memcached:1.6-alpine3.16
+    image: docker.io/library/memcached:1.6-alpine3.20
     command: --memory-limit=64
     networks:
       - web-memcached
@@ -115,7 +117,7 @@ services:
 
   configure:
     profiles: [ configure ]
-    image: docker.io/library/alpine:3.16
+    image: docker.io/library/alpine:3.20
     entrypoint:
       - sh
       - -c
@@ -213,6 +215,8 @@ services:
         target: /weasyl/testing
       - type: tmpfs
         target: /tmp
+      - type: tmpfs
+        target: /magick-tmp
     networks:
       - test-postgres
     read_only: true

imagemagick-policy.xml

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE policymap [
+<!ELEMENT policymap (policy)*>
+<!ATTLIST policymap xmlns CDATA #FIXED "">
+<!ELEMENT policy EMPTY>
+<!ATTLIST policy xmlns CDATA #FIXED "">
+<!ATTLIST policy domain NMTOKEN #REQUIRED>
+<!ATTLIST policy name NMTOKEN #IMPLIED>
+<!ATTLIST policy pattern CDATA #IMPLIED>
+<!ATTLIST policy rights NMTOKEN #IMPLIED>
+<!ATTLIST policy stealth NMTOKEN #IMPLIED>
+<!ATTLIST policy value CDATA #IMPLIED>
+]>
+<!--
+Adapted from the websafe ImageMagick security policy at <https://github.com/ImageMagick/ImageMagick6/blob/f20b5721ce14bb95b46b01b4f4f14258962ebd36/config/policy-websafe.xml>.
+
+    - locks down permitted types even more
+    - removes or relaxes most DoS limits, which we should manage with application-level quotas
+-->
+<policymap>
+  <!-- Set maximum parallel threads. -->
+  <policy domain="resource" name="thread" value="16"/>
+  <!-- Set maximum time to live in seconds or mnemonics, e.g. "2 minutes". When
+       this limit is exceeded, an exception is thrown and processing stops. -->
+  <policy domain="resource" name="time" value="55"/>
+  <!-- Set maximum number of open pixel cache files. When this limit is
+       exceeded, any subsequent pixels cached to disk are closed and reopened
+       on demand. -->
+  <policy domain="resource" name="file" value="768"/>
+  <!-- Set maximum amount of memory in bytes to allocate for the pixel cache
+       from the heap. When this limit is exceeded, the image pixels are cached
+       to memory-mapped disk. -->
+  <policy domain="resource" name="memory" value="2GiB"/>
+  <!-- Set maximum amount of memory map in bytes to allocate for the pixel
+       cache. When this limit is exceeded, the image pixels are cached to
+       disk. -->
+  <policy domain="resource" name="map" value="2GiB"/>
+  <!-- Set the maximum width * height of an image that can reside in the pixel
+       cache memory. Images that exceed the area limit are cached to disk. -->
+  <policy domain="resource" name="area" value="270MP"/>
+  <!-- Set maximum amount of disk space in bytes permitted for use by the pixel
+       cache. When this limit is exceeded, the pixel cache is not be created
+       and an exception is thrown. -->
+  <policy domain="resource" name="disk" value="0"/>
+  <!-- Set the maximum length of an image sequence.  When this limit is
+       exceeded, an exception is thrown. -->
+  <policy domain="resource" name="list-length" value="3600"/>
+  <!-- Set the maximum width of an image.  When this limit is exceeded, an
+       exception is thrown. -->
+  <policy domain="resource" name="width" value="16384"/>
+  <!-- Set the maximum height of an image.  When this limit is exceeded, an
+       exception is thrown. -->
+  <policy domain="resource" name="height" value="16384"/>
+  <!-- Do not create temporary files in the default shared directories, instead
+       specify a private area to store only ImageMagick temporary files. -->
+  <policy domain="resource" name="temporary-path" value="/magick-tmp/"/>
+  <!-- Force memory initialization by memory mapping select memory
+       allocations. -->
+  <policy domain="cache" name="memory-map" value="anonymous"/>
+  <!-- Do not permit any delegates to execute. -->
+  <policy domain="delegate" rights="none" pattern="*"/>
+  <!-- Do not permit any image filters to load. -->
+  <policy domain="filter" rights="none" pattern="*"/>
+  <!-- Don't read/write from/to stdin/stdout. -->
+  <policy domain="path" rights="none" pattern="-"/>
+  <!-- don't read sensitive paths. -->
+  <policy domain="path" rights="none" pattern="/*"/>
+  <!-- ImageMagick still copies some input to a temporary file to make it seekable. This copying doesn't count as a write. (TODO: determine why the input - an `open(..., "rb")` file handle - isn't recognized as already seekable.) -->
+  <policy domain="path" rights="read" pattern="/magick-tmp/magick-*"/>
+  <!-- Indirect reads are not permitted. -->
+  <policy domain="path" rights="none" pattern="@*"/>
+  <!-- Deny all image modules and specifically exempt reading or writing
+       web-safe image formats. -->
+  <policy domain="module" rights="none" pattern="*" />
+  <policy domain="module" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP}"/>
+  <!-- This policy sets the number of times to replace content of certain
+       memory buffers and temporary files before they are freed or deleted. -->
+  <policy domain="system" name="shred" value="1"/>
+  <!-- Enable the initialization of buffers with zeros, resulting in a minor
+       performance penalty but with improved security. -->
+  <policy domain="system" name="memory-map" value="anonymous"/>
+  <!-- Set the maximum amount of memory in bytes that are permitted for
+       allocation requests. -->
+  <policy domain="system" name="max-memory-request" value="2GiB"/>
+</policymap>

libweasyl/files.py

@@ -96,7 +96,7 @@ def file_type_for_category(data, category):
         try:
             im = images.from_buffer(data)
         except SanperaError:
-            raise UnknownFileFormat('The image data provided could not be decoded.')
+            raise UnknownFileFormat('The image data provided could not be decoded. Image files must be in the GIF, JPG, or PNG formats.')
         fmt = im.original_format.decode()
         if fmt == 'JPEG':
             fmt = 'JPG'

libweasyl/test/test_files.py

@@ -57,10 +57,10 @@ def test_file_type_for_category_invalid_visual_file():
 
 def test_file_type_for_category_invalid_file_format():
     """
-    ``InvalidFileFormat`` is raised if the image data isn't a GIF, JPG, or PNG.
+    A file format error is raised if the image data isn't a GIF, JPG, or PNG. It might be ``InvalidFileFormat`` if the file was parsed as a known format, or ``UnknownFileFormat`` if parsing failed, including by being denied by a security policy.
     """
     data = datadir.join('1x70.bmp').read(mode='rb')
-    with pytest.raises(exceptions.InvalidFileFormat):
+    with pytest.raises((exceptions.InvalidFileFormat, exceptions.UnknownFileFormat)):
         files.file_type_for_category(data, Category.visual)