Dockerfile dependencies

sddi-base/patches/00_patch_sql_url.patch

@@ -0,0 +1,11 @@
+--- ckan/ckan/model/__init__.py
++++ ckan/ckan/model/__init__.py
+@@ -276,7 +276,7 @@ class Repository():
+         self.reset_alembic_output()
+         alembic_config = AlembicConfig(self._alembic_ini)
+         alembic_config.set_main_option(
+-            "sqlalchemy.url", config.get("sqlalchemy.url")
++            "sqlalchemy.url", config.get("sqlalchemy.url").replace('%', '%%')
+         )
+         try:
+             sqlalchemy_migrate_version = self.metadata.bind.execute(

sddi-base/patches/02_patch_postgres_username_split.patch

@@ -0,0 +1,10 @@
+--- ckan/ckanext/datastore/backend/postgres.py
++++ ckan/ckanext/datastore/backend/postgres.py
+@@ -1809,7 +1809,7 @@ class DatastorePostgresqlBackend(DatastoreBackend):
+         read only user.
+         '''
+         write_connection = self._get_write_engine().connect()
+-        read_connection_user = sa_url.make_url(self.read_url).username
++        read_connection_user = sa_url.make_url(self.read_url).username.split("@")[0]
+
+         drop_foo_sql = u'DROP TABLE IF EXISTS _foo'

sddi-base/scripts/apply_ckan_patches.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+shopt -s nullglob
+for patch in patches/*.patch; do
+    /usr/bin/patch -p0 -i $patch
+done

sddi-base/setup/prerun.py

@@ -0,0 +1,220 @@
+import os
+import sys
+import subprocess
+import psycopg2
+try:
+    from urllib.request import urlopen
+    from urllib.error import URLError
+except ImportError:
+    from urllib2 import urlopen
+    from urllib2 import URLError
+
+import time
+import re
+import json
+
+ckan_ini = os.environ.get("CKAN_INI", "/srv/app/production.ini")
+
+RETRY = 5
+
+
+def update_plugins():
+
+    plugins = os.environ.get("CKAN__PLUGINS", "")
+    print(("[prerun] Setting the following plugins in {}:".format(ckan_ini)))
+    print(plugins)
+    cmd = ["ckan", "config-tool", ckan_ini, "ckan.plugins = {}".format(plugins)]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    print("[prerun] Plugins set.")
+
+
+def check_main_db_connection(retry=None):
+
+    conn_str = os.environ.get("CKAN_SQLALCHEMY_URL")
+    if not conn_str:
+        print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db")
+        return
+    return check_db_connection(conn_str, retry)
+
+
+def check_datastore_db_connection(retry=None):
+
+    conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL")
+    if not conn_str:
+        print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db")
+        return
+    return check_db_connection(conn_str, retry)
+
+
+def check_db_connection(conn_str, retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print("[prerun] Giving up after 5 tries...")
+        sys.exit(1)
+
+    try:
+        connection = psycopg2.connect(conn_str)
+
+    except psycopg2.Error as e:
+        print(str(e))
+        print("[prerun] Unable to connect to the database, waiting...")
+        time.sleep(10)
+        check_db_connection(conn_str, retry=retry - 1)
+    else:
+        connection.close()
+
+
+def check_solr_connection(retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print("[prerun] Giving up after 5 tries...")
+        sys.exit(1)
+
+    url = os.environ.get("CKAN_SOLR_URL", "")
+    search_url = '{url}/schema/name?wt=json'.format(url=url)
+
+    try:
+        connection = urlopen(search_url)
+    except URLError as e:
+        print(str(e))
+        print("[prerun] Unable to connect to solr, waiting...")
+        time.sleep(10)
+        check_solr_connection(retry=retry - 1)
+    else:
+        import re
+        conn_info = connection.read()
+        schema_name = json.loads(conn_info)
+        if 'ckan' in schema_name['name']:
+            print('[prerun] Succesfully connected to solr and CKAN schema loaded')
+        else:
+            print('[prerun] Succesfully connected to solr, but CKAN schema not found')
+
+
+def init_db():
+
+    db_command = ["ckan", "-c", ckan_ini, "db", "init"]
+    print("[prerun] Initializing or upgrading db - start")
+    try:
+        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
+        print("[prerun] Initializing or upgrading db - end")
+    except subprocess.CalledProcessError as e:
+        if "OperationalError" in e.output:
+            print(e.output)
+            print("[prerun] Database not ready, waiting a bit before exit...")
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print(e.output)
+            raise e
+
+
+def init_datastore_db():
+
+    conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL")
+    if not conn_str:
+        print("[prerun] Skipping datastore initialization")
+        return
+
+    datastore_perms_command = ["ckan", "-c", ckan_ini, "datastore", "set-permissions"]
+
+    connection = psycopg2.connect(conn_str)
+    cursor = connection.cursor()
+
+    print("[prerun] Initializing datastore db - start")
+    try:
+        datastore_perms = subprocess.Popen(
+            datastore_perms_command, stdout=subprocess.PIPE
+        )
+
+        perms_sql = datastore_perms.stdout.read()
+        # Remove internal pg command as psycopg2 does not like it
+        perms_sql = re.sub(b'\\\\connect "(.*)"', b"", perms_sql)
+        cursor.execute(perms_sql)
+        for notice in connection.notices:
+            print(notice)
+
+        connection.commit()
+
+        print("[prerun] Initializing datastore db - end")
+        print(datastore_perms.stdout.read())
+    except psycopg2.Error as e:
+        print("[prerun] Could not initialize datastore")
+        print(str(e))
+
+    except subprocess.CalledProcessError as e:
+        if "OperationalError" in e.output:
+            print(e.output)
+            print("[prerun] Database not ready, waiting a bit before exit...")
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print(e.output)
+            raise e
+    finally:
+        cursor.close()
+        connection.close()
+
+
+def create_sysadmin():
+
+    name = os.environ.get("CKAN_SYSADMIN_NAME")
+    password = os.environ.get("CKAN_SYSADMIN_PASSWORD")
+    email = os.environ.get("CKAN_SYSADMIN_EMAIL")
+
+    if name and password and email:
+
+        # Check if user exists
+        command = ["ckan", "-c", ckan_ini, "user", "show", name]
+
+        out = subprocess.check_output(command)
+        if b"User:None" not in re.sub(b"\s", b"", out):
+            print("[prerun] Sysadmin user exists, skipping creation")
+            return
+
+        # Create user
+        command = [
+            "ckan",
+            "-c",
+            ckan_ini,
+            "user",
+            "add",
+            name,
+            "password=" + password,
+            "email=" + email,
+        ]
+
+        subprocess.call(command)
+        print("[prerun] Created user {0}".format(name))
+
+        # Make it sysadmin
+        command = ["ckan", "-c", ckan_ini, "sysadmin", "add", name]
+
+        subprocess.call(command)
+        print("[prerun] Made user {0} a sysadmin".format(name))
+
+        # cleanup permissions
+        # We're running as root before pivoting to uwsgi and dropping privs
+        data_dir = "%s/storage" % os.environ['CKAN_STORAGE_PATH']
+
+        command = ["chown", "-R", "ckan:ckan", data_dir]
+        subprocess.call(command)
+        print("[prerun] Ensured storage directory is owned by ckan")
+
+if __name__ == "__main__":
+
+    maintenance = os.environ.get("MAINTENANCE_MODE", "").lower() == "true"
+
+    if maintenance:
+        print("[prerun] Maintenance mode, skipping setup...")
+    else:
+        check_main_db_connection()
+        init_db()
+        update_plugins()
+        check_datastore_db_connection()
+        init_datastore_db()
+        check_solr_connection()
+        create_sysadmin()

sddi-base/setup/start_ckan.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+if [[ $CKAN__PLUGINS == *"datapusher"* ]]; then
+    # Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later)
+    echo "Setting a temporary value for ckan.datapusher.api_token"
+    ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx
+fi
+
+# Set up the Secret key used by Beaker and Flask
+# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var
+if grep -qE "beaker.session.secret ?= ?$" production.ini
+then
+    echo "Setting beaker.session.secret in ini file"
+    ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
+    ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}"
+    ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
+fi
+
+# Run the prerun script to init CKAN and create the default admin user
+python3 prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python3 "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+    done
+fi
+
+# Set the common uwsgi options
+UWSGI_OPTS="--plugins http,python \
+            --socket /tmp/uwsgi.sock \
+            --wsgi-file /srv/app/wsgi.py \
+            --module wsgi:application \
+            --uid 92 --gid 92 \
+            --http 0.0.0.0:5000 \
+            --master --enable-threads \
+            --lazy-apps \
+            -p 2 -L -b 32768 --vacuum \
+            --harakiri $UWSGI_HARAKIRI"
+
+if [ $? -eq 0 ]
+then
+    # Start supervisord
+    supervisord --configuration /etc/supervisord.conf &
+    # Start uwsgi
+    uwsgi $UWSGI_OPTS
+else
+  echo "[prerun] failed...not starting CKAN."
+fi

sddi-base/setup/supervisord.conf

@@ -0,0 +1,23 @@
+[unix_http_server]
+file = /tmp/supervisor.sock
+chmod = 0777
+chown = nobody:nogroup
+
+[supervisord]
+logfile = /tmp/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /tmp/supervisord.pid
+nodaemon = true
+umask = 022
+identifier = supervisor
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord.d/*.conf

sddi-base/setup/uwsgi.conf

@@ -0,0 +1,2 @@
+[uwsgi]
+route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd