Use hardcopy of the GHA jobs on main

Based on: 7d7ed2e

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,32 +7,46 @@ assignees: ''
 
 ---
 
-NOTE: If this is a _discussion starter_, you need any _installation help_ or have a _question_ on how to accomplish something,
+<!--
+NOTE: If this is a _discussion starter_, you need any _installation help_, or have a _question_ on how to accomplish something,
 rather post at [our discussion channel](https://www.domjudge.org/chat) or send an email to our
 [DOMjudge-devel mailinglist](https://www.domjudge.org/mailman/postorius/lists/domjudge-devel.domjudge.org/)
 instead of filing an issue here.
+-->
 
 ### Description of the problem
-> Replace this line with a short description.
+<!--
+Write here a short description.
+-->
 
 ### Your environment
-> Include details about your installation here.
-> - DOMjudge version (e.g. 7.0.0 or a github commit hash)
-> - Operating system / Linux distribution and version (e.g. Ubuntu 18.04)
-> - Webserver (e.g. Apache or nginx)
+<!--
+Include details about your installation here.
+- DOMjudge version (e.g. 7.0.0 or a github commit hash)
+- Operating system / Linux distribution and version (e.g. Ubuntu 18.04)
+- Webserver (e.g. Apache or nginx)
+-->
 
 ### Steps to reproduce
-> Replace this with a description how we can reproduce your bug.
-> - Step 1
-> - Step 2
-> - Step 3
+<!--
+Write here a description how we can reproduce your bug.
+1. Step 1
+1. Step 2
+1. Step 3
+-->
 
 ### Expected behaviour
-> Replace this line with what you would expect to happen.
+<!--
+Write here what you would expect to happen.
+-->
 
 ### Actual behaviour
-> Replace this line with what happens instead.
+<!--
+Write here what happens instead.
+-->
 
 ### Any other information that you want to share?
-> Please include webserver, symfony and judgedaemon log snippets here as appropriate.
-> Screenshots may help in case of UI bugs.
+<!--
+Please include webserver, symfony and judgedaemon log snippets here as appropriate.
+Screenshots may help in case of UI bugs.
+-->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -7,25 +7,36 @@ assignees: ''
 
 ---
 
-NOTE: If this is a _discussion starter_, you need any _installation help_ or have a _question_ on how to accomplish something,
+<!--
+NOTE: If this is a _discussion starter_, you need any _installation help_, or have a _question_ on how to accomplish something,
 rather post at [our discussion channel](https://www.domjudge.org/chat) or send an email to our
 [DOMjudge-devel mailinglist](https://www.domjudge.org/mailman/postorius/lists/domjudge-devel.domjudge.org/).
 
 Thank you for suggesting ways to improve DOMjudge. Before you file a feature
 request, it might be useful to discuss it first via the chat or mailing list
 linked above. We can then assess together whether there is
 not already a way to accomplish your goal with DOMjudge currently.
+-->
 
 ### Description of the enhancement request
-> Replace this line with a short description.
+<!--
+Write here a short description.
+-->
 
 ### The goal you want to achieve
-> Please elaborate on the (larger, higher level) goal you want to achieve with this enhancement, so we have a good understanding what this feature would be useful for and how it fits in DOMjudge as a whole.
+<!--
+Please elaborate on the (larger, higher level) goal you want to achieve with this enhancement, so we have a good understanding what this feature would be useful for and how it fits in DOMjudge as a whole.
+-->
 
 ### Expected behaviour
-> Replace this line with what you would expect to happen.
-> Please describe the workflow how you want this feature to work with expected start URL
-> - Step 1
-> - Step 2
+<!--
+Write here what you would expect to happen.
+For example describe the flow how you want this feature to work.
+1. Step 1
+2. Step 2
+-->
+
 ### Any other information that you want to share?
-> Screenshots may help in case of UI enhancements with annotated drawings.
+<!--
+Screenshots with annotated drawings may help in case of UI enhancements.
+-->

.github/dependabot.yml

@@ -44,3 +44,12 @@ updates:
     allow:
         - dependency-type: "all"
     target-branch: "8.2"
+  - package-ecosystem: "composer"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 0
+    versioning-strategy: lockfile-only
+    allow:
+        - dependency-type: "all"
+    target-branch: "8.3"

.github/jobs/baseinstall.sh

@@ -1,54 +1,31 @@
 #!/bin/sh
 
-# Functions to annotate the Github actions logs
-alias trace_on='set -x'
-alias trace_off='{ set +x; } 2>/dev/null'
-
-section_start_internal  () {
-    echo "::group::$1"
-    trace_on
-}
-
-section_end_internal () {
-    echo "::endgroup::"
-    trace_on
-}
-
-alias section_start='trace_off ; section_start_internal '
-alias section_end='trace_off ; section_end_internal '
+. .github/jobs/ci_settings.sh
 
 export version="$1"
+db=${2:-install}
+phpversion="${3:-8.1}"
+# If this script is called from unit-tests.sh, we use the test environment
+export APP_ENV="${4:-prod}"
 
-set -eux
+# In the test environment, we need to use a different database
+[ "$APP_ENV" = "prod" ] && DATABASE_NAME=domjudge || DATABASE_NAME=domjudge_test
 
-section_start "Update packages"
-sudo apt update
-section_end
+MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-root}
 
-section_start "Install needed packages"
-sudo apt install -y acl zip unzip nginx php php-fpm php-gd \
-                    php-cli php-intl php-mbstring php-mysql php-curl php-json \
-                    php-xml php-zip ntp make sudo debootstrap \
-                    libcgroup-dev lsof php-cli php-curl php-json php-xml \
-                    php-zip procps gcc g++ default-jre-headless \
-                    default-jdk-headless ghc fp-compiler autoconf automake bats \
-                    python3-sphinx python3-sphinx-rtd-theme rst2pdf fontconfig \
-                    python3-yaml latexmk curl
-section_end
+set -eux
 
-PHPVERSION=$(php -r 'echo PHP_MAJOR_VERSION.".".PHP_MINOR_VERSION."\n";')
-export PHPVERSION
+if [ -z "$phpversion" ]; then
+phpversion=$(php -r 'echo PHP_MAJOR_VERSION.".".PHP_MINOR_VERSION."\n";')
+fi
 
-section_start "Install composer"
-php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
-HASH="$(wget -q -O - https://composer.github.io/installer.sig)"
-php -r "if (hash_file('SHA384', 'composer-setup.php') === '$HASH') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
-sudo php composer-setup.php --install-dir=/usr/local/bin --filename=composer
-section_end
+show_phpinfo "$phpversion"
 
 section_start "Run composer"
 export APP_ENV="dev"
-composer install --no-scripts
+cd webapp
+composer install --no-scripts |tee "$ARTIFACTS"/composer_out.txt
+cd ..
 section_end
 
 section_start "Set simple admin password"
@@ -58,59 +35,127 @@ section_end
 
 section_start "Install domserver"
 make configure
-./configure --with-baseurl='https://localhost/domjudge/' --enable-doc-build=no --prefix="/opt/domjudge"
+if [ "$version" = "all" ]; then
+    # Note that we use http instead of https here as python requests doesn't
+    # like our self-signed cert. We should fix this separately.
+    ./configure \
+      --with-baseurl='http://localhost/domjudge/' \
+      --with-domjudge-user=domjudge \
+      --with-judgehost-chrootdir=/chroot/domjudge | tee "$ARTIFACTS"/configure.txt
+    make build-scripts domserver judgehost docs
+    make install-domserver install-judgehost install-docs
+else
+    ./configure \
+      --with-baseurl='https://localhost/domjudge/' \
+      --with-domjudge-user=root \
+      --enable-doc-build=no \
+      --enable-judgehost-build=no | tee "$ARTIFACTS"/configure.txt
+    make domserver
+    make install-domserver
+    rm -rf /opt/domjudge/domserver/webapp/public/doc
+    cp -r doc /opt/domjudge/domserver/webapp/public/
+    find /opt/domjudge/domserver -name DOMjudgelogo.pdf
+fi
+section_end
 
-make domserver
-sudo make install-domserver
+section_start "SQL settings"
+cat > ~/.my.cnf <<EOF
+[client]
+host=sqlserver
+user=root
+password=${MYSQL_ROOT_PASSWORD}
+EOF
+cat ~/.my.cnf
+
+mysql_root "CREATE DATABASE IF NOT EXISTS \`$DATABASE_NAME\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
+mysql_root "CREATE USER IF NOT EXISTS \`domjudge\`@'%' IDENTIFIED BY 'domjudge';"
+mysql_root "GRANT SELECT, INSERT, UPDATE, DELETE ON \`$DATABASE_NAME\`.* TO 'domjudge'@'%';"
+mysql_root "FLUSH PRIVILEGES;"
+
+# Show some MySQL debugging
+mysql_root "show databases"
+mysql_root "SELECT CURRENT_USER();"
+mysql_root "SELECT USER();"
+mysql_root "SELECT user,host FROM mysql.user"
+mysql_root "SET GLOBAL max_allowed_packet=1073741824"
+mysql_root "SHOW GLOBAL STATUS LIKE 'Connection_errors_%'"
+mysql_root "SHOW VARIABLES LIKE '%_timeout'"
+echo "unused:sqlserver:$DATABASE_NAME:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret
+mysql_user "SELECT CURRENT_USER();"
+mysql_user "SELECT USER();"
 section_end
 
-section_start "Explicit start mysql + install DB"
-sudo /etc/init.d/mysql start
+if [ "${db}" = "install" ]; then
+    section_start "Install DOMjudge database"
+    /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} bare-install
+    section_end
+elif [ "${db}" = "upgrade" ]; then
+    section_start "Upgrade DOMjudge database"
+    /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} upgrade
+    section_end
+fi
+
+section_start "Show PHP config"
+php -v | tee -a "$ARTIFACTS"/php.txt
+php -m | tee -a "$ARTIFACTS"/php.txt
+section_end
 
-/opt/domjudge/domserver/bin/dj_setup_database -uroot -proot bare-install
+section_start "Show general config"
+printenv | tee -a "$ARTIFACTS"/environment.txt
+cp /etc/os-release "$ARTIFACTS"/os-release.txt
+cp /proc/cmdline "$ARTIFACTS"/cmdline.txt
 section_end
 
 section_start "Setup webserver"
-sudo cp /opt/domjudge/domserver/etc/domjudge-fpm.conf /etc/php/$PHPVERSION/fpm/pool.d/domjudge.conf
+cp /opt/domjudge/domserver/etc/domjudge-fpm.conf /etc/php/"$phpversion"/fpm/pool.d/domjudge.conf
 
-sudo rm -f /etc/nginx/sites-enabled/*
-sudo cp /opt/domjudge/domserver/etc/nginx-conf /etc/nginx/sites-enabled/domjudge
+rm -f /etc/nginx/sites-enabled/*
+cp /opt/domjudge/domserver/etc/nginx-conf /etc/nginx/sites-enabled/domjudge
 
 openssl req -nodes -new -x509 -keyout /tmp/server.key -out /tmp/server.crt -subj "/C=NL/ST=Noord-Holland/L=Amsterdam/O=TestingForPR/CN=localhost"
-sudo cp /tmp/server.crt /usr/local/share/ca-certificates/
-sudo update-ca-certificates
+cp /tmp/server.crt /usr/local/share/ca-certificates/
+update-ca-certificates
 # shellcheck disable=SC2002
-cat "$(pwd)/.github/jobs/data/nginx_extra" | sudo tee -a /etc/nginx/sites-enabled/domjudge
-sudo nginx -t
+cat "$(pwd)/.github/jobs/data/nginx_extra" | tee -a /etc/nginx/sites-enabled/domjudge
+nginx -t
 section_end
 
 section_start "Show webserver is up"
-for service in nginx php${PHPVERSION}-fpm; do
-    sudo systemctl restart $service
-    sudo systemctl status  $service
+for service in nginx php${phpversion}-fpm; do
+    service "$service" restart
+    service "$service" status
 done
 section_end
 
-section_start "Install the example data"
-/opt/domjudge/domserver/bin/dj_setup_database -uroot -proot install-examples
-section_end
+if [ "${db}" = "install" ]; then
+    section_start "Install the example data"
+    if [ "$version" = "unit" ]; then
+	    # Make sure admin has no team associated so we will not insert submissions during unit tests.
+	    mysql_root "UPDATE user SET teamid=null WHERE userid=1;" $DATABASE_NAME
+    fi
+    /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} install-examples | tee -a "$ARTIFACTS/mysql.txt"
+    section_end
+fi
 
 section_start "Setup user"
 # We're using the admin user in all possible roles
-echo "DELETE FROM userrole WHERE userid=1;" | mysql -uroot -proot domjudge
+mysql_root "DELETE FROM userrole WHERE userid=1;" $DATABASE_NAME
 if [ "$version" = "team" ]; then
     # Add team to admin user
-    echo "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" | mysql -uroot -proot domjudge
-    echo "UPDATE user SET teamid = 1 WHERE userid = 1;" | mysql -uroot -proot domjudge
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME
+    mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME
 elif [ "$version" = "jury" ]; then
     # Add jury to admin user
-    echo "INSERT INTO userrole (userid, roleid) VALUES (1, 2);" | mysql -uroot -proot domjudge
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 2);" $DATABASE_NAME
 elif [ "$version" = "balloon" ]; then
     # Add balloon to admin user
-    echo "INSERT INTO userrole (userid, roleid) VALUES (1, 4);" | mysql -uroot -proot domjudge
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 4);" $DATABASE_NAME
 elif [ "$version" = "admin" ]; then
     # Add admin to admin user
-    echo "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" | mysql -uroot -proot domjudge
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME
+elif [ "$version" = "all" ] || [ "$version" = "unit" ]; then
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME
+    mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME
+    mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME
 fi
 section_end
-

.github/jobs/chroot.bats

@@ -0,0 +1,55 @@
+#!/usr/bin/env bats
+
+load 'assert'
+
+CHROOT="/chroot/domjudge"
+if [ -n "${GITHUB_REPOSITORY+x}" ]; then
+    CHROOT="/__w/domjudge/domjudge${CHROOT}"
+fi
+# Cleanup old dir
+rm -rf $CHROOT
+
+COMMANDARGS=""
+if [ -n "${ARCH+x}" ]; then
+    COMMANDARGS="-a $ARCH $COMMANDARGS"
+fi
+
+@test "help output" {
+    run ./dj_make_chroot -h
+    assert_success
+    assert_partial "Usage:"
+    assert_partial "Creates a chroot environment with Java JRE support using the"
+    assert_partial "Debian or Ubuntu GNU/Linux distribution."
+    assert_partial "Options"
+    assert_partial "Available architectures:"
+    assert_partial "Environment Overrides:"
+    assert_partial "This script must be run as root"
+}
+
+@test "Test chroot fails if unsupported architecture given" {
+    if [ -n "${ARCH+x}" ]; then
+        skip "Already an Arch set in the commands."
+    fi
+    for unsupported_arch in 'dom04' 'arm' '64' 'namd64' 'amd64v2'; do
+        run ./dj_make_chroot $COMMANDARGS -a $unsupported_arch
+        assert_failure
+        assert_partial "Error: Architecture $unsupported_arch not supported for"
+    done
+}
+
+# Creation of the chroot is slow so we run all tests inside 1 large test to speedup.
+@test "Test chroot works with args: $COMMANDARGS" {
+    run ./dj_make_chroot $COMMANDARGS
+    assert_partial "Done building chroot in $CHROOT"
+    assert_success
+    if [ -n "${ARCH+x}" ]; then
+        run ./dj_run_chroot "dpkg --print-architecture"
+        assert_partial "$ARCH"
+        assert_success
+    else
+        HOSTARCH=$(dpkg --print-architecture)
+        run ./dj_run_chroot "dpkg --print-architecture"
+        assert_partial "$HOSTARCH"
+        assert_success
+    fi
+}