From c0655a489a468bbb63a22e0bff2d77d576f2913b Mon Sep 17 00:00:00 2001 From: Ilche Bedelovski Date: Mon, 29 Jan 2024 17:54:51 +0100 Subject: [PATCH 1/5] SDDI base Docker build CKAN 2.10.3 --- sddi-base/Dockerfile | 183 +++++++++++++++++++++++++++++++------------ 1 file changed, 131 insertions(+), 52 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 7936f24..9dfa6d0 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -1,14 +1,60 @@ ############################################################################### # Build stage ############################################################################### -ARG CKAN_VERSION_BUILD_STAGE=2.9.9-dev +ARG CKAN_VERSION_BUILD_STAGE=2.10.1-dev ARG CKAN_VERSION_BUILD_SPATIAL=2.9.9-focal -ARG CKAN_VERSION_RUNTIME_STAGE=2.9.9-focal +ARG CKAN_VERSION_RUNTIME_STAGE=2.10.1-focal FROM ckan/ckan-base:${CKAN_VERSION_BUILD_STAGE} as extbuild USER root +RUN apk add --no-cache \ + # python3 \ + # python3-dev \ + git \ + curl \ + postgresql-dev \ + linux-headers \ + gcc \ + make \ + g++ \ + zlib-dev \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + pcre-dev \ + pcre \ + geos \ + libffi-dev \ + libxml2-dev \ + libxslt-dev \ + openssl-dev \ + rust \ + cargo + +ARG PYTHON_VERSION=3.9.9 + +# download and extract python sources +RUN cd /opt \ + && wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \ + && tar xzf Python-${PYTHON_VERSION}.tgz + +# build python and remove left-over sources +RUN cd /opt/Python-${PYTHON_VERSION} \ + && ./configure --prefix=/usr --enable-optimizations --with-ensurepip=install \ + && make install \ + && rm /opt/Python-${PYTHON_VERSION}.tgz /opt/Python-${PYTHON_VERSION} -rf + +RUN set -ex && \ + pip install --no-cache-dir -U pip && \ + rm -rf /var/lib/apt/lists/* + +RUN pip install markupsafe==2.0.1 && \ + pip install wheel + # ckanext-hierarchy ########################################################### ARG CKANEXT_HIERARCHY_VERSION="v1.2.0" ENV CKANEXT_HIERARCHY_VERSION=${CKANEXT_HIERARCHY_VERSION} @@ -100,45 +146,38 @@ RUN set -ex && \ pip wheel --wheel-dir=/wheels \ git+${CKANEXT_PASSWORD_POLICY_GITHUB_URL}.git@${CKANEXT_PASSWORD_POLICY_VERSION}#egg=ckanext-password-policy -# ckanext-spatial ############################################################# -FROM ghcr.io/keitaroinc/ckan:${CKAN_VERSION_BUILD_SPATIAL} as extbuild-spatial - -ARG CKANEXT_SPATIAL_VERSION="c2118b9" -ENV CKANEXT_SPATIAL_VERSION=${CKANEXT_SPATIAL_VERSION} +# ckanext-harvest ########################################################### -USER root +ARG CKANEXT_HARVEST_VERSION="v1.5.6" +ENV CKANEXT_HARVEST_VERSION=${CKANEXT_HARVEST_VERSION} +ENV CKANEXT_HARVEST_GITHUB_URL="https://github.com/ckan/ckanext-harvest.git" -# Install any system packages necessary to build extensions RUN set -ex && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - python3-dev python3-pip libxml2-dev libxslt1-dev libgeos-c1v5 python-is-python3 && \ - mkdir -p /wheels && \ - pip install -U pip + mkdir -p /wheels && \ + pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/dev-requirements.txt RUN set -ex && \ - pip install -r https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements.txt && \ - curl -o /wheels/ckanext-spatial.txt \ - https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements.txt && \ - pip install -r https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-postgis.txt && \ - curl -o /wheels/ckanext-spatial-postgis.txt \ - https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-postgis.txt && \ - ls -lah /wheels + pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/requirements.txt && \ + pip wheel --wheel-dir=/wheels git+https://github.com/ckan/ckanext-harvest.git@${CKANEXT_HARVEST_VERSION}#egg=ckanext-harvest && \ + curl -o /wheels/ckanext-harvest.txt https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/requirements.txt && \ + ls -lah /wheels + +# # ckanext-spatial ############################################################# +ENV CKANEXT_SPATIAL_GITHUB_URL="https://github.com/ckan/ckanext-spatial" +ENV CKANEXT_SPATIAL_VERSION="v2.1.1" RUN set -ex && \ + # pip wheel --wheel-dir=/wheels -r \ + # https://raw.githubusercontent.com/ckan/ckanext-spatial/v2.1.1/requirements.txt && \ + curl -o /wheels/ckanext-spatial-requirements.txt \ + https://raw.githubusercontent.com/ckan/ckanext-spatial/v2.1.1/requirements.txt && \ pip wheel --wheel-dir=/wheels \ - git+https://github.com/MarijaKnezevic/ckanext-spatial.git@${CKANEXT_SPATIAL_VERSION}#egg=ckanext-spatial + git+${CKANEXT_SPATIAL_GITHUB_URL}.git@${CKANEXT_SPATIAL_VERSION}#egg=ckanext-spatial ############################################################################### # Runtime stage ############################################################################### -FROM ghcr.io/keitaroinc/ckan:${CKAN_VERSION_RUNTIME_STAGE} as runtime - -ENV CKAN__PLUGINS "image_view text_view recline_view webpage_view datastore datapusher \ - hierarchy_display hierarchy_form display_group relation \ - spatial_metadata spatial_query datesearch repeating composite scheming_datasets \ - password_policy \ - envvars" +FROM ckan/ckan-base:2.10 as runtime # Extra env for compatibility with ckan/base Docker images for downstream k8s ENV CKAN_INI=${APP_DIR}/production.ini @@ -147,17 +186,55 @@ ENV TZ="UTC" USER root -# Install any system packages necessary to build extensions -RUN set -ex && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - libxml2-dev libxslt1-dev libgeos-c1v5 && \ - pip install --no-cache-dir -U pip && \ - rm -rf /var/lib/apt/lists/* +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl \ + postgresql-dev \ + linux-headers \ + gcc \ + make \ + g++ \ + zlib-dev \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + pcre-dev \ + pcre \ + geos \ + libffi-dev \ + libxml2-dev \ + libxslt-dev \ + openssl-dev \ + rust \ + cargo \ + proj-data + +ENV CKAN__PLUGINS "image_view text_view recline_view webpage_view datastore \ + harvest ckan_harvester \ + hierarchy_display hierarchy_form \ + # datapusher Token required \ + display_group \ + # relation ImportError: cannot import name 'Mapping' from 'collections' \ + spatial_metadata spatial_query \ + scheming_datasets \ + datesearch \ + composite \ + repeating \ + # password_policy No module named 'ckan.lib.repoze_plugins' \ + envvars" + +RUN pip install markupsafe==2.0.1 # Copy python wheels from build stage COPY --from=extbuild /wheels ${APP_DIR}/ext_wheels -COPY --from=extbuild-spatial /wheels ${APP_DIR}/ext_wheels + +# ckanext-harvest ########################################################### +RUN pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-harvest && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-harvest.txt # ckanext-hierarchy ########################################################### RUN set -ex && \ @@ -170,21 +247,20 @@ RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-grouphierarchy # ckanext-relation ############################################################ -RUN set -ex && \ - pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-relation.txt && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-relation +# RUN set -ex && \ +# pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-relation.txt && \ +# pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-relation # ckanext-spatial ############################################################# RUN set -ex && \ - pip install -r ${APP_DIR}/ext_wheels/ckanext-spatial.txt && \ - pip install -r ${APP_DIR}/ext_wheels/ckanext-spatial-postgis.txt && \ + pip install -r ${APP_DIR}/ext_wheels/ckanext-spatial-requirements.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-spatial -# ckanext-scheming ############################################################ +# # ckanext-scheming ############################################################ RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheming -# ckanext-datesearch ########################################################## +# # ckanext-datesearch ########################################################## RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-datesearch @@ -205,20 +281,23 @@ RUN set -ex && \ COPY --chown=ckan:ckan initScripts/ ${APP_DIR}/docker-afterinit.d COPY --chown=ckan:ckan who.ini ${APP_DIR}/who.ini +RUN set -ex && \ + ckan generate config ${APP_DIR}/production.ini + RUN set -ex && \ ckan config-tool "${CKAN_INI}" "ckan.plugins = ${CKAN__PLUGINS}" && \ - ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ - ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ + # ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ + # ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_schemas = ckanext.scheming:ckan_dataset.yaml" && \ ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json ckanext.repeating:presets.json ckanext.composite:presets.json" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_fallback = false" && \ ckan config-tool "${CKAN_INI}" "licenses_group_url = https://raw.githubusercontent.com/tum-gis/ckanext-grouphierarchy-sddi/main/ckanext/grouphierarchy/licenses_SDDI.json" && \ - ckan config-tool "${CKAN_INI}" "ckanext.password_policy.password_length = 12" && \ - ckan config-tool "${CKAN_INI}" "ckanext.password_policy.failed_logins = 3" && \ - ckan config-tool "${CKAN_INI}" "ckanext.password_policy.user_locked_time = 600" && \ - ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.type = custom" && \ - ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.custom.url = https://tile.openstreetmap.de/{z}/{x}/{y}.png" && \ - ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.attribution = OpenStreetMap contributors." && \ + # ckan config-tool "${CKAN_INI}" "ckanext.password_policy.password_length = 12" && \ + # ckan config-tool "${CKAN_INI}" "ckanext.password_policy.failed_logins = 3" && \ + # ckan config-tool "${CKAN_INI}" "ckanext.password_policy.user_locked_time = 600" && \ + # ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.type = custom" && \ + # ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.custom.url = https://tile.openstreetmap.de/{z}/{x}/{y}.png" && \ + # ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.attribution = OpenStreetMap contributors." && \ echo "${TZ}" > /etc/timezone && \ mkdir -p ${CKAN_STORAGE_PATH} && \ chown -R ckan:ckan ${APP_DIR} ${CKAN_STORAGE_PATH} && \ From 606e0919f06948faf3269ef8a56504165290506d Mon Sep 17 00:00:00 2001 From: Ilche Bedelovski Date: Thu, 1 Feb 2024 19:18:47 +0100 Subject: [PATCH 2/5] Dockerfile 2.10.3 Ubuntu slim based --- sddi-base/Dockerfile | 284 +++++++++++++++++++++++++++---------------- 1 file changed, 177 insertions(+), 107 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 9dfa6d0..193b1f8 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -5,55 +5,97 @@ ARG CKAN_VERSION_BUILD_STAGE=2.10.1-dev ARG CKAN_VERSION_BUILD_SPATIAL=2.9.9-focal ARG CKAN_VERSION_RUNTIME_STAGE=2.10.1-focal -FROM ckan/ckan-base:${CKAN_VERSION_BUILD_STAGE} as extbuild +FROM python:3.9-slim as ckanbuild -USER root +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.10.3 + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.10.3 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} -RUN apk add --no-cache \ - # python3 \ - # python3-dev \ +WORKDIR ${SRC_DIR} + +RUN apt-get update && apt-get install -y \ git \ curl \ - postgresql-dev \ - linux-headers \ + libpq-dev \ gcc \ make \ g++ \ - zlib-dev \ autoconf \ automake \ libtool \ patch \ musl-dev \ - pcre-dev \ - pcre \ - geos \ + libpcre3-dev \ + libpcre3 \ libffi-dev \ libxml2-dev \ - libxslt-dev \ - openssl-dev \ - rust \ - cargo + libxslt-dev -ARG PYTHON_VERSION=3.9.9 + # Cleanup to reduce image size +RUN apt-get clean && rm -rf /var/lib/apt/lists/* -# download and extract python sources -RUN cd /opt \ - && wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \ - && tar xzf Python-${PYTHON_VERSION}.tgz +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python -# build python and remove left-over sources -RUN cd /opt/Python-${PYTHON_VERSION} \ - && ./configure --prefix=/usr --enable-optimizations --with-ensurepip=install \ - && make install \ - && rm /opt/Python-${PYTHON_VERSION}.tgz /opt/Python-${PYTHON_VERSION} -rf +# Create the src directory +RUN mkdir -p ${SRC_DIR} -RUN set -ex && \ - pip install --no-cache-dir -U pip && \ - rm -rf /var/lib/apt/lists/* +# Downgrade setuptools so that CKAN requirements can be built +RUN pip3 install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan + +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh + +# Apply patches +RUN cd ${SRC_DIR} && ls -lah ${SRC_DIR} && bash ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git + +# Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736 +RUN echo 'Cython < 3.0' > /tmp/constraint.txt +RUN pip3_CONSTRAINT=/tmp/constraint.txt pip3 wheel --wheel-dir=/wheels PyYAML==5.4.1 + +# RUN pip3-compile ckan/requirements.in +RUN pip3 wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip3 wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3 + +FROM python:3.9-slim as extbuild + +USER root + +RUN apt-get update && apt-get install -y \ + git \ + curl \ + libpq-dev \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + libpcre3-dev \ + libpcre3 \ + libffi-dev \ + libxml2-dev \ + libxslt-dev -RUN pip install markupsafe==2.0.1 && \ - pip install wheel +RUN pip install -U markupsafe==2.0.1 + +# Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736 +RUN echo 'Cython < 3.0' > /tmp/constraint.txt +RUN pip3_CONSTRAINT=/tmp/constraint.txt pip3 wheel --wheel-dir=/wheels PyYAML==5.4.1 # ckanext-hierarchy ########################################################### ARG CKANEXT_HIERARCHY_VERSION="v1.2.0" @@ -147,7 +189,6 @@ RUN set -ex && \ git+${CKANEXT_PASSWORD_POLICY_GITHUB_URL}.git@${CKANEXT_PASSWORD_POLICY_VERSION}#egg=ckanext-password-policy # ckanext-harvest ########################################################### - ARG CKANEXT_HARVEST_VERSION="v1.5.6" ENV CKANEXT_HARVEST_VERSION=${CKANEXT_HARVEST_VERSION} ENV CKANEXT_HARVEST_GITHUB_URL="https://github.com/ckan/ckanext-harvest.git" @@ -162,75 +203,91 @@ RUN set -ex && \ curl -o /wheels/ckanext-harvest.txt https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/requirements.txt && \ ls -lah /wheels -# # ckanext-spatial ############################################################# +# ckanext-spatial ############################################################# ENV CKANEXT_SPATIAL_GITHUB_URL="https://github.com/ckan/ckanext-spatial" -ENV CKANEXT_SPATIAL_VERSION="v2.1.1" +ENV CKANEXT_SPATIAL_VERSION="release-v2.1.0" RUN set -ex && \ - # pip wheel --wheel-dir=/wheels -r \ - # https://raw.githubusercontent.com/ckan/ckanext-spatial/v2.1.1/requirements.txt && \ curl -o /wheels/ckanext-spatial-requirements.txt \ - https://raw.githubusercontent.com/ckan/ckanext-spatial/v2.1.1/requirements.txt && \ + https://raw.githubusercontent.com/ckan/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-py2.txt && \ pip wheel --wheel-dir=/wheels \ git+${CKANEXT_SPATIAL_GITHUB_URL}.git@${CKANEXT_SPATIAL_VERSION}#egg=ckanext-spatial -############################################################################### -# Runtime stage -############################################################################### -FROM ckan/ckan-base:2.10 as runtime +FROM python:3.9-slim + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV GIT_BRANCH=ckan-2.10.3 -# Extra env for compatibility with ckan/base Docker images for downstream k8s +# Setting the locale +ENV LC_ALL="en_US.UTF-8" +RUN apt-get update && apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Set timezone +RUN echo "UTC" > /etc/timezone ENV CKAN_INI=${APP_DIR}/production.ini ENV CKAN_STORAGE_PATH=/var/lib/ckan -ENV TZ="UTC" -USER root - -RUN apk add --no-cache \ - python3 \ - python3-dev \ +# Update the package lists and install required packages +RUN apt-get update && apt-get install -y \ + bash \ git \ + gettext \ curl \ - postgresql-dev \ - linux-headers \ - gcc \ - make \ - g++ \ - zlib-dev \ - autoconf \ - automake \ - libtool \ - patch \ + wget \ + unzip \ + postgresql-client \ + libmagic1 \ + libpcre3 \ + libxslt1.1 \ + libxml2 \ + tzdata \ + apache2-utils \ musl-dev \ - pcre-dev \ - pcre \ - geos \ - libffi-dev \ - libxml2-dev \ - libxslt-dev \ - openssl-dev \ - rust \ - cargo \ - proj-data - -ENV CKAN__PLUGINS "image_view text_view recline_view webpage_view datastore \ - harvest ckan_harvester \ - hierarchy_display hierarchy_form \ - # datapusher Token required \ - display_group \ - # relation ImportError: cannot import name 'Mapping' from 'collections' \ - spatial_metadata spatial_query \ - scheming_datasets \ - datesearch \ - composite \ - repeating \ - # password_policy No module named 'ckan.lib.repoze_plugins' \ - envvars" - -RUN pip install markupsafe==2.0.1 - -# Copy python wheels from build stage -COPY --from=extbuild /wheels ${APP_DIR}/ext_wheels + libssl-dev \ + proj-bin \ + libproj-dev \ + proj-data \ + python3-cffi + +# Cleanup to reduce image size +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +# Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736 +RUN echo 'Cython < 3.0' > /tmp/constraint.txt +RUN pip3_CONSTRAINT=/tmp/constraint.txt pip3 wheel --wheel-dir=/wheels PyYAML==5.4.1 + +# Create SRC_DIR +RUN mkdir -p ${SRC_DIR} && \ + # Link python to python3 + ln -s /usr/bin/python3 /usr/bin/python + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip3 install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0 + +# Create a local user and group to run the app +# Add a group with a specific GID (92) +RUN groupadd -g 92 ckan +# Add a user with a specific UID (92), home directory, and add to the ckan group +RUN useradd -u 92 -g ckan -M -d /srv/app -s /bin/bash ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip3 install -e /srv/app/src/ckan +RUN cp who.ini ${APP_DIR} +RUN pip3 install --no-index --find-links=/srv/app/wheels -r requirements.txt # ckanext-harvest ########################################################### RUN pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-harvest && \ @@ -246,21 +303,11 @@ RUN set -ex && \ pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-grouphierarchy.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-grouphierarchy -# ckanext-relation ############################################################ -# RUN set -ex && \ -# pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-relation.txt && \ -# pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-relation - -# ckanext-spatial ############################################################# -RUN set -ex && \ - pip install -r ${APP_DIR}/ext_wheels/ckanext-spatial-requirements.txt && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-spatial - -# # ckanext-scheming ############################################################ +# ckanext-scheming ############################################################ RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheming -# # ckanext-datesearch ########################################################## +# ckanext-datesearch ########################################################## RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-datesearch @@ -277,6 +324,11 @@ RUN set -ex && \ pip install -r ${APP_DIR}/ext_wheels/ckanext-password-policy.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-password-policy +# ckanext-spatial ############################################################# +RUN set -ex && \ + pip3 install -e 'git+https://github.com/ckan/ckanext-spatial.git#egg=ckanext-spatial' && \ + pip3 install -r 'https://raw.githubusercontent.com/ckan/ckanext-spatial/master/requirements.txt' + # Copy init scripts and additional files COPY --chown=ckan:ckan initScripts/ ${APP_DIR}/docker-afterinit.d COPY --chown=ckan:ckan who.ini ${APP_DIR}/who.ini @@ -286,22 +338,40 @@ RUN set -ex && \ RUN set -ex && \ ckan config-tool "${CKAN_INI}" "ckan.plugins = ${CKAN__PLUGINS}" && \ - # ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ - # ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ + ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ + ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_schemas = ckanext.scheming:ckan_dataset.yaml" && \ ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json ckanext.repeating:presets.json ckanext.composite:presets.json" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_fallback = false" && \ ckan config-tool "${CKAN_INI}" "licenses_group_url = https://raw.githubusercontent.com/tum-gis/ckanext-grouphierarchy-sddi/main/ckanext/grouphierarchy/licenses_SDDI.json" && \ - # ckan config-tool "${CKAN_INI}" "ckanext.password_policy.password_length = 12" && \ - # ckan config-tool "${CKAN_INI}" "ckanext.password_policy.failed_logins = 3" && \ - # ckan config-tool "${CKAN_INI}" "ckanext.password_policy.user_locked_time = 600" && \ - # ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.type = custom" && \ - # ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.custom.url = https://tile.openstreetmap.de/{z}/{x}/{y}.png" && \ - # ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.attribution = OpenStreetMap contributors." && \ + ckan config-tool "${CKAN_INI}" "ckanext.password_policy.password_length = 12" && \ + ckan config-tool "${CKAN_INI}" "ckanext.password_policy.failed_logins = 3" && \ + ckan config-tool "${CKAN_INI}" "ckanext.password_policy.user_locked_time = 600" && \ + ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.type = custom" && \ + ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.custom.url = https://tile.openstreetmap.de/{z}/{x}/{y}.png" && \ + ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.attribution = OpenStreetMap contributors." && \ echo "${TZ}" > /etc/timezone && \ mkdir -p ${CKAN_STORAGE_PATH} && \ chown -R ckan:ckan ${APP_DIR} ${CKAN_STORAGE_PATH} && \ # Remove wheels rm -rf ${APP_DIR}/ext_wheels -USER ckan +WORKDIR ${APP_DIR} + +# Create local storage folder +RUN mkdir -p ${CKAN_STORAGE_PATH} && \ + chown -R ckan:ckan ${CKAN_STORAGE_PATH} + +COPY setup/prerun.py ${APP_DIR} +COPY setup/start_ckan.sh ${APP_DIR} +ADD https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/wsgi.py ${APP_DIR} +RUN chmod 644 ${APP_DIR}/wsgi.py + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir /docker-entrypoint.d + +EXPOSE 5000 + +HEALTHCHECK --interval=60s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit CMD ["/srv/app/start_ckan.sh"] + +CMD ["/srv/app/start_ckan.sh"] From 2c1b4f67ff76681c4380443ff59382564fbd05e5 Mon Sep 17 00:00:00 2001 From: Ilche Bedelovski Date: Sat, 3 Feb 2024 22:19:10 +0100 Subject: [PATCH 3/5] CKAN 2.10.3 build and supervisor enable --- sddi-base/Dockerfile | 71 ++++++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 193b1f8..1bb4d78 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -1,10 +1,6 @@ ############################################################################### -# Build stage +# CKAN build stage ############################################################################### -ARG CKAN_VERSION_BUILD_STAGE=2.10.1-dev -ARG CKAN_VERSION_BUILD_SPATIAL=2.9.9-focal -ARG CKAN_VERSION_RUNTIME_STAGE=2.10.1-focal - FROM python:3.9-slim as ckanbuild # Used by Github Actions to tag the image with @@ -69,6 +65,9 @@ RUN pip3_CONSTRAINT=/tmp/constraint.txt pip3 wheel --wheel-dir=/wheels PyYAML==5 RUN pip3 wheel --wheel-dir=/wheels -r ckan/requirements.txt RUN pip3 wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3 +############################################################################### +# Extbuild stage +############################################################################### FROM python:3.9-slim as extbuild USER root @@ -111,6 +110,13 @@ RUN set -ex && \ curl -o /wheels/ckanext-hierarchy.txt https://raw.githubusercontent.com/ckan/ckanext-hierarchy/${CKANEXT_HIERARCHY_VERSION}/requirements.txt && \ ls -lah /wheels +# ckanext-envvars +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.2 + +RUN set -ex && \ + pip3 wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + # ckanext-grouphierarchy ###################################################### ARG CKANEXT_SDDI_VERSION="1.1.3" ENV CKANEXT_SDDI_VERSION=${CKANEXT_SDDI_VERSION} @@ -194,8 +200,8 @@ ENV CKANEXT_HARVEST_VERSION=${CKANEXT_HARVEST_VERSION} ENV CKANEXT_HARVEST_GITHUB_URL="https://github.com/ckan/ckanext-harvest.git" RUN set -ex && \ - mkdir -p /wheels && \ - pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/dev-requirements.txt + mkdir -p /wheels && \ + pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/dev-requirements.txt RUN set -ex && \ pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/requirements.txt && \ @@ -213,6 +219,9 @@ RUN set -ex && \ pip wheel --wheel-dir=/wheels \ git+${CKANEXT_SPATIAL_GITHUB_URL}.git@${CKANEXT_SPATIAL_VERSION}#egg=ckanext-spatial +############################################################################### +# Runtime stage +############################################################################### FROM python:3.9-slim ENV APP_DIR=/srv/app @@ -254,11 +263,14 @@ RUN apt-get update && apt-get install -y \ proj-bin \ libproj-dev \ proj-data \ - python3-cffi + python3-cffi \ + supervisor # Cleanup to reduce image size RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN pip install markupsafe==2.0.1 setuptools wheel + # Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736 RUN echo 'Cython < 3.0' > /tmp/constraint.txt RUN pip3_CONSTRAINT=/tmp/constraint.txt pip3 wheel --wheel-dir=/wheels PyYAML==5.4.1 @@ -269,29 +281,30 @@ RUN mkdir -p ${SRC_DIR} && \ ln -s /usr/bin/python3 /usr/bin/python # Get artifacts from build stages -COPY --from=ckanbuild /wheels /srv/app/wheels -COPY --from=extbuild /wheels /srv/app/ext_wheels -COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} +COPY --from=ckanbuild /wheels ${APP_DIR}/wheels +COPY --from=extbuild /wheels ${APP_DIR}/ext_wheels +COPY --from=ckanbuild ${APP_DIR}/src/ckan ${CKAN_DIR} # Additional install steps for build stages artifacts -RUN pip3 install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0 +RUN pip3 install --no-index --find-links=${APP_DIR}/wheels uWSGI==2.0.20 gevent==21.12.0 # Create a local user and group to run the app # Add a group with a specific GID (92) RUN groupadd -g 92 ckan # Add a user with a specific UID (92), home directory, and add to the ckan group -RUN useradd -u 92 -g ckan -M -d /srv/app -s /bin/bash ckan +RUN useradd -u 92 -g ckan -M -d ${APP_DIR} -s /bin/bash ckan WORKDIR ${CKAN_DIR} # Install CKAN -RUN pip3 install -e /srv/app/src/ckan +RUN pip3 install -e ${APP_DIR}/src/ckan RUN cp who.ini ${APP_DIR} -RUN pip3 install --no-index --find-links=/srv/app/wheels -r requirements.txt +RUN pip3 install --no-index --find-links=${APP_DIR}/wheels -r requirements.txt # ckanext-harvest ########################################################### -RUN pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-harvest && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-harvest.txt +RUN set -ex && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-harvest && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-harvest.txt # ckanext-hierarchy ########################################################### RUN set -ex && \ @@ -303,6 +316,10 @@ RUN set -ex && \ pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-grouphierarchy.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-grouphierarchy +# ckanext-envvars ############################################################ +RUN set -ex && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-envvars + # ckanext-scheming ############################################################ RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheming @@ -326,13 +343,26 @@ RUN set -ex && \ # ckanext-spatial ############################################################# RUN set -ex && \ - pip3 install -e 'git+https://github.com/ckan/ckanext-spatial.git#egg=ckanext-spatial' && \ - pip3 install -r 'https://raw.githubusercontent.com/ckan/ckanext-spatial/master/requirements.txt' + pip3 install -e 'git+https://github.com/ckan/ckanext-spatial.git#egg=ckanext-spatial' && \ + pip3 install -r 'https://raw.githubusercontent.com/ckan/ckanext-spatial/master/requirements.txt' # Copy init scripts and additional files COPY --chown=ckan:ckan initScripts/ ${APP_DIR}/docker-afterinit.d COPY --chown=ckan:ckan who.ini ${APP_DIR}/who.ini +ENV CKAN__PLUGINS "envvars image_view text_view recline_view webpage_view datastore \ + harvest ckan_harvester \ + hierarchy_display hierarchy_form \ + # datapusher Token required \ + # display_group ValueError: Cannot determine url for /usr/local/lib/python3.9/site-packages/ckanext/grouphierarchy/fanstatic/hierarchy_theme.css \ + # relation ImportError: cannot import name 'Mapping' from 'collections' \ + spatial_metadata spatial_query \ + scheming_datasets \ + datesearch \ + composite \ + repeating" + # password_policy No module named 'ckan.lib.repoze_plugins' \ + RUN set -ex && \ ckan generate config ${APP_DIR}/production.ini @@ -358,10 +388,13 @@ RUN set -ex && \ WORKDIR ${APP_DIR} +ENV UWSGI_HARAKIRI=50 + # Create local storage folder RUN mkdir -p ${CKAN_STORAGE_PATH} && \ chown -R ckan:ckan ${CKAN_STORAGE_PATH} +# Copy local scripts COPY setup/prerun.py ${APP_DIR} COPY setup/start_ckan.sh ${APP_DIR} ADD https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/wsgi.py ${APP_DIR} From 25c4519579249c6846838f4239514313772286f5 Mon Sep 17 00:00:00 2001 From: Ilche Bedelovski Date: Sat, 3 Feb 2024 22:24:05 +0100 Subject: [PATCH 4/5] Dockerfile dependencies --- sddi-base/patches/00_patch_sql_url.patch | 11 + .../02_patch_postgres_username_split.patch | 10 + sddi-base/scripts/apply_ckan_patches.sh | 5 + sddi-base/setup/prerun.py | 220 ++++++++++++++++++ sddi-base/setup/start_ckan.sh | 56 +++++ sddi-base/setup/supervisord.conf | 23 ++ sddi-base/setup/uwsgi.conf | 2 + 7 files changed, 327 insertions(+) create mode 100644 sddi-base/patches/00_patch_sql_url.patch create mode 100644 sddi-base/patches/02_patch_postgres_username_split.patch create mode 100644 sddi-base/scripts/apply_ckan_patches.sh create mode 100644 sddi-base/setup/prerun.py create mode 100755 sddi-base/setup/start_ckan.sh create mode 100644 sddi-base/setup/supervisord.conf create mode 100644 sddi-base/setup/uwsgi.conf diff --git a/sddi-base/patches/00_patch_sql_url.patch b/sddi-base/patches/00_patch_sql_url.patch new file mode 100644 index 0000000..f81eccb --- /dev/null +++ b/sddi-base/patches/00_patch_sql_url.patch @@ -0,0 +1,11 @@ +--- ckan/ckan/model/__init__.py ++++ ckan/ckan/model/__init__.py +@@ -276,7 +276,7 @@ class Repository(): + self.reset_alembic_output() + alembic_config = AlembicConfig(self._alembic_ini) + alembic_config.set_main_option( +- "sqlalchemy.url", config.get("sqlalchemy.url") ++ "sqlalchemy.url", config.get("sqlalchemy.url").replace('%', '%%') + ) + try: + sqlalchemy_migrate_version = self.metadata.bind.execute( \ No newline at end of file diff --git a/sddi-base/patches/02_patch_postgres_username_split.patch b/sddi-base/patches/02_patch_postgres_username_split.patch new file mode 100644 index 0000000..e743f6a --- /dev/null +++ b/sddi-base/patches/02_patch_postgres_username_split.patch @@ -0,0 +1,10 @@ +--- ckan/ckanext/datastore/backend/postgres.py ++++ ckan/ckanext/datastore/backend/postgres.py +@@ -1809,7 +1809,7 @@ class DatastorePostgresqlBackend(DatastoreBackend): + read only user. + ''' + write_connection = self._get_write_engine().connect() +- read_connection_user = sa_url.make_url(self.read_url).username ++ read_connection_user = sa_url.make_url(self.read_url).username.split("@")[0] + + drop_foo_sql = u'DROP TABLE IF EXISTS _foo' \ No newline at end of file diff --git a/sddi-base/scripts/apply_ckan_patches.sh b/sddi-base/scripts/apply_ckan_patches.sh new file mode 100644 index 0000000..66f12fa --- /dev/null +++ b/sddi-base/scripts/apply_ckan_patches.sh @@ -0,0 +1,5 @@ +#!/bin/bash +shopt -s nullglob +for patch in patches/*.patch; do + /usr/bin/patch -p0 -i $patch +done \ No newline at end of file diff --git a/sddi-base/setup/prerun.py b/sddi-base/setup/prerun.py new file mode 100644 index 0000000..18c2e76 --- /dev/null +++ b/sddi-base/setup/prerun.py @@ -0,0 +1,220 @@ +import os +import sys +import subprocess +import psycopg2 +try: + from urllib.request import urlopen + from urllib.error import URLError +except ImportError: + from urllib2 import urlopen + from urllib2 import URLError + +import time +import re +import json + +ckan_ini = os.environ.get("CKAN_INI", "/srv/app/production.ini") + +RETRY = 5 + + +def update_plugins(): + + plugins = os.environ.get("CKAN__PLUGINS", "") + print(("[prerun] Setting the following plugins in {}:".format(ckan_ini))) + print(plugins) + cmd = ["ckan", "config-tool", ckan_ini, "ckan.plugins = {}".format(plugins)] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + print("[prerun] Plugins set.") + + +def check_main_db_connection(retry=None): + + conn_str = os.environ.get("CKAN_SQLALCHEMY_URL") + if not conn_str: + print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db") + return + return check_db_connection(conn_str, retry) + + +def check_datastore_db_connection(retry=None): + + conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") + if not conn_str: + print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db") + return + return check_db_connection(conn_str, retry) + + +def check_db_connection(conn_str, retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print("[prerun] Giving up after 5 tries...") + sys.exit(1) + + try: + connection = psycopg2.connect(conn_str) + + except psycopg2.Error as e: + print(str(e)) + print("[prerun] Unable to connect to the database, waiting...") + time.sleep(10) + check_db_connection(conn_str, retry=retry - 1) + else: + connection.close() + + +def check_solr_connection(retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print("[prerun] Giving up after 5 tries...") + sys.exit(1) + + url = os.environ.get("CKAN_SOLR_URL", "") + search_url = '{url}/schema/name?wt=json'.format(url=url) + + try: + connection = urlopen(search_url) + except URLError as e: + print(str(e)) + print("[prerun] Unable to connect to solr, waiting...") + time.sleep(10) + check_solr_connection(retry=retry - 1) + else: + import re + conn_info = connection.read() + schema_name = json.loads(conn_info) + if 'ckan' in schema_name['name']: + print('[prerun] Succesfully connected to solr and CKAN schema loaded') + else: + print('[prerun] Succesfully connected to solr, but CKAN schema not found') + + +def init_db(): + + db_command = ["ckan", "-c", ckan_ini, "db", "init"] + print("[prerun] Initializing or upgrading db - start") + try: + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + print("[prerun] Initializing or upgrading db - end") + except subprocess.CalledProcessError as e: + if "OperationalError" in e.output: + print(e.output) + print("[prerun] Database not ready, waiting a bit before exit...") + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + + +def init_datastore_db(): + + conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") + if not conn_str: + print("[prerun] Skipping datastore initialization") + return + + datastore_perms_command = ["ckan", "-c", ckan_ini, "datastore", "set-permissions"] + + connection = psycopg2.connect(conn_str) + cursor = connection.cursor() + + print("[prerun] Initializing datastore db - start") + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, stdout=subprocess.PIPE + ) + + perms_sql = datastore_perms.stdout.read() + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub(b'\\\\connect "(.*)"', b"", perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print(notice) + + connection.commit() + + print("[prerun] Initializing datastore db - end") + print(datastore_perms.stdout.read()) + except psycopg2.Error as e: + print("[prerun] Could not initialize datastore") + print(str(e)) + + except subprocess.CalledProcessError as e: + if "OperationalError" in e.output: + print(e.output) + print("[prerun] Database not ready, waiting a bit before exit...") + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + name = os.environ.get("CKAN_SYSADMIN_NAME") + password = os.environ.get("CKAN_SYSADMIN_PASSWORD") + email = os.environ.get("CKAN_SYSADMIN_EMAIL") + + if name and password and email: + + # Check if user exists + command = ["ckan", "-c", ckan_ini, "user", "show", name] + + out = subprocess.check_output(command) + if b"User:None" not in re.sub(b"\s", b"", out): + print("[prerun] Sysadmin user exists, skipping creation") + return + + # Create user + command = [ + "ckan", + "-c", + ckan_ini, + "user", + "add", + name, + "password=" + password, + "email=" + email, + ] + + subprocess.call(command) + print("[prerun] Created user {0}".format(name)) + + # Make it sysadmin + command = ["ckan", "-c", ckan_ini, "sysadmin", "add", name] + + subprocess.call(command) + print("[prerun] Made user {0} a sysadmin".format(name)) + + # cleanup permissions + # We're running as root before pivoting to uwsgi and dropping privs + data_dir = "%s/storage" % os.environ['CKAN_STORAGE_PATH'] + + command = ["chown", "-R", "ckan:ckan", data_dir] + subprocess.call(command) + print("[prerun] Ensured storage directory is owned by ckan") + +if __name__ == "__main__": + + maintenance = os.environ.get("MAINTENANCE_MODE", "").lower() == "true" + + if maintenance: + print("[prerun] Maintenance mode, skipping setup...") + else: + check_main_db_connection() + init_db() + update_plugins() + check_datastore_db_connection() + init_datastore_db() + check_solr_connection() + create_sysadmin() diff --git a/sddi-base/setup/start_ckan.sh b/sddi-base/setup/start_ckan.sh new file mode 100755 index 0000000..9b46b1e --- /dev/null +++ b/sddi-base/setup/start_ckan.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +if [[ $CKAN__PLUGINS == *"datapusher"* ]]; then + # Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later) + echo "Setting a temporary value for ckan.datapusher.api_token" + ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx +fi + +# Set up the Secret key used by Beaker and Flask +# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var +if grep -qE "beaker.session.secret ?= ?$" production.ini +then + echo "Setting beaker.session.secret in ini file" + ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" + ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" +fi + +# Run the prerun script to init CKAN and create the default admin user +python3 prerun.py + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + done +fi + +# Set the common uwsgi options +UWSGI_OPTS="--plugins http,python \ + --socket /tmp/uwsgi.sock \ + --wsgi-file /srv/app/wsgi.py \ + --module wsgi:application \ + --uid 92 --gid 92 \ + --http 0.0.0.0:5000 \ + --master --enable-threads \ + --lazy-apps \ + -p 2 -L -b 32768 --vacuum \ + --harakiri $UWSGI_HARAKIRI" + +if [ $? -eq 0 ] +then + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi + uwsgi $UWSGI_OPTS +else + echo "[prerun] failed...not starting CKAN." +fi diff --git a/sddi-base/setup/supervisord.conf b/sddi-base/setup/supervisord.conf new file mode 100644 index 0000000..a3f6671 --- /dev/null +++ b/sddi-base/setup/supervisord.conf @@ -0,0 +1,23 @@ +[unix_http_server] +file = /tmp/supervisor.sock +chmod = 0777 +chown = nobody:nogroup + +[supervisord] +logfile = /tmp/supervisord.log +logfile_maxbytes = 50MB +logfile_backups=10 +loglevel = info +pidfile = /tmp/supervisord.pid +nodaemon = true +umask = 022 +identifier = supervisor + +[supervisorctl] +serverurl = unix:///tmp/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[include] +files = /etc/supervisord.d/*.conf diff --git a/sddi-base/setup/uwsgi.conf b/sddi-base/setup/uwsgi.conf new file mode 100644 index 0000000..6321d6d --- /dev/null +++ b/sddi-base/setup/uwsgi.conf @@ -0,0 +1,2 @@ +[uwsgi] +route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd From fee51b259cc87b4a65bfafce7bd5f9bee23b3ec5 Mon Sep 17 00:00:00 2001 From: Aleksandra Lazoroska Date: Wed, 16 Oct 2024 13:30:59 +0200 Subject: [PATCH 5/5] SDDI base Docker build CKAN 2.11.0 --- sddi-base/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 1bb4d78..3313c97 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -4,11 +4,11 @@ FROM python:3.9-slim as ckanbuild # Used by Github Actions to tag the image with -ENV IMAGE_TAG=2.10.3 +ENV IMAGE_TAG=2.11.0 # Set CKAN version to build ENV GIT_URL=https://github.com/ckan/ckan.git -ENV GIT_BRANCH=ckan-2.10.3 +ENV GIT_BRANCH=ckan-2.11.0 # Set src dirs ENV SRC_DIR=/srv/app/src @@ -229,7 +229,7 @@ ENV SRC_DIR=/srv/app/src ENV CKAN_DIR=${SRC_DIR}/ckan ENV DATA_DIR=/srv/app/data ENV PIP_SRC=${SRC_DIR} -ENV GIT_BRANCH=ckan-2.10.3 +ENV GIT_BRANCH=ckan-2.11.0 # Setting the locale ENV LC_ALL="en_US.UTF-8"