Merge pull request apel#351 from Sae126V/python3-migration-build-script

Apel FPM build script

CHANGELOG

@@ -1,5 +1,3 @@
-Changelog for apel
-==================
  * Thu May 25 2024 Adrian Coveney <[email protected]> - 1.9.3-1
  - [parsers] Reduced verbosity of parser to only report a single time that logs have been skipped.
  - Fixed some unit tests so that they run properly.

scripts/apel-build.sh

@@ -0,0 +1,290 @@
+#!/bin/bash
+
+# Apel Build Script 1.0: FPM edition
+# Currently supports RPM ONLY.
+# Download ruby (if you're locked to 2.5, use RVM) and then run:
+# sudo gem install fpm -v 1.14.2
+# for RPM builds, you will also need:
+# sudo yum install rpm-build rpmlint | sudo apt-get install rpm
+# ./apel-build.sh rpm <version> <iteration> <python_root_dir>
+# e.g.
+# ./apel-build.sh rpm 1.9.2 1 /usr/lib/python2.7
+# If you're struggling finding the right version of Python to use, consider opening interpreter and:
+# import site; site.getsitepackages()
+
+set -e
+
+usage() {
+    echo "Usage: $0 [options] (rpm) <version> <iteration> <python_root_dir> "
+    echo -e "Build script for Apel.\n"
+    echo "  -h                    Displays help."
+    echo "  -v                    Verbose FPM output."
+    echo "  -s <source_dir>       Directory of source files.  Defaults to ~/rpmbuild/SOURCES."
+    echo -e "  -b <build_dir>        Directory of build files.  Defaults to ~/rpmbuild/BUILD.\n" 1>&2;
+    exit 1;
+}
+
+# Bool flags to prevent automatic overwrite of input
+SOURCE_ASSIGNED=0
+BUILD_ASSIGNED=0
+
+# Configurable options
+while getopts ":hs:b:v" o; do
+    case "${o}" in
+        h)  echo "APEL Help"
+            usage;
+            ;;
+        s)  s=${OPTARG}
+            SOURCE_DIR=$s
+            SOURCE_ASSIGNED=1
+            ;;
+        b)  b=${OPTARG}
+            BUILD_DIR=$b
+            BUILD_ASSIGNED=1
+            ;;
+        v)  VERBOSE="--verbose "
+            ;;
+        *)  usage;
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+# Check how any arguments there are
+if [ "$#" -ne 4 ]; then
+    echo "Expected 4 arguments, $# given."
+    usage;
+fi
+
+PACK_TYPE=$1
+VERSION=$2
+ITERATION=$3
+PYTHON_ROOT_DIR=$4 # i.e. /usr/lib/python2.7
+
+if [[ "$PACK_TYPE" = "rpm" ]]; then
+    LIB_EXTENSION="/site-packages"
+    if [[ "$SOURCE_ASSIGNED" = 0 ]]; then
+        SOURCE_DIR=~/rpmbuild/SOURCES
+    fi
+    if [[ "$BUILD_ASSIGNED" = 0 ]]; then
+        BUILD_DIR=~/rpmbuild/BUILD
+    fi
+else # If package type is NOT a rpm type, show an error message and exit
+    echo "$0 currently supports 'rpm' package type ONLY."
+    usage;
+fi
+
+# Directory cleaning and repository management
+mkdir -p "$SOURCE_DIR"
+mkdir -p "$BUILD_DIR"
+
+# Temporary directories for apel-parsers, apel-client, apel-server, apel-lib
+TEMP_DIR_FOR_PARSERS="/tmp/apel-parsers-build"
+TEMP_DIR_FOR_CLIENT="/tmp/apel-client-build"
+TEMP_DIR_FOR_SERVER="/tmp/apel-server-build"
+TEMP_DIR_FOR_LIB="/tmp/apel-lib-build"
+
+# Cleanup any previous directories
+rm -rf "$TEMP_DIR_FOR_PARSERS" "$TEMP_DIR_FOR_CLIENT" "$TEMP_DIR_FOR_SERVER" "$TEMP_DIR_FOR_LIB"
+rm -rf "${SOURCE_DIR:?}"/*
+rm -rf "${BUILD_DIR:?}"/*
+
+# Create necessary temporary directories
+# apel-parsers
+mkdir -p "$TEMP_DIR_FOR_PARSERS/etc/apel"
+mkdir -p $TEMP_DIR_FOR_PARSERS/usr/bin
+mkdir -p $TEMP_DIR_FOR_PARSERS/usr/share/apel
+
+# apel-client
+mkdir -p "$TEMP_DIR_FOR_CLIENT/usr/bin"
+mkdir -p "$TEMP_DIR_FOR_CLIENT/etc/logrotate.d"
+mkdir -p "$TEMP_DIR_FOR_CLIENT/etc/apel"
+mkdir -p "$TEMP_DIR_FOR_CLIENT/usr/share/apel"
+mkdir -p "$TEMP_DIR_FOR_CLIENT/var/log/apel"
+mkdir -p  "$TEMP_DIR_FOR_CLIENT/var/run/apel"
+
+# apel-server
+mkdir -p "$TEMP_DIR_FOR_SERVER/etc/apel"
+mkdir -p "$TEMP_DIR_FOR_SERVER/usr/bin"
+mkdir -p "$TEMP_DIR_FOR_SERVER/usr/share/apel"
+mkdir -p "$TEMP_DIR_FOR_SERVER/var/log/apel"
+mkdir -p "$TEMP_DIR_FOR_SERVER/var/run/apel"
+
+# apel-lib
+mkdir -p "$TEMP_DIR_FOR_LIB/$PYTHON_ROOT_DIR/$LIB_EXTENSION/apel/"
+
+APEL_DIR=apel-$VERSION-$ITERATION
+
+# Download and extract source
+TAR_FILE="${VERSION}-${ITERATION}.tar.gz"
+TAR_URL="https://github.com/apel/apel/archive/${TAR_FILE}"
+wget --no-check-certificate "$TAR_URL" -O "$SOURCE_DIR/$TAR_FILE"
+tar xvf "$SOURCE_DIR/$TAR_FILE" -C "$SOURCE_DIR"
+rm -f "$SOURCE_DIR/$TAR_FILE"
+
+# Get supplied Python version
+PY_VERSION="$(basename "$PYTHON_ROOT_DIR")"
+PY_NUM=${PY_VERSION#python}
+OS_EXTENSION="$(uname -r | grep -o 'el[7-9]' || echo '_all')"
+
+# Copy files into apel-parsers
+cp "$SOURCE_DIR/$APEL_DIR/conf/parser.cfg" "$TEMP_DIR_FOR_PARSERS/etc/apel/parser.cfg"
+cp "$SOURCE_DIR/$APEL_DIR/bin/parser.py" "$TEMP_DIR_FOR_PARSERS/usr/bin/apelparser"
+cp "$SOURCE_DIR/$APEL_DIR/scripts/htcondor_acc.sh" "$TEMP_DIR_FOR_PARSERS/usr/share/apel/htcondor_acc.sh"
+cp "$SOURCE_DIR/$APEL_DIR/scripts/slurm_acc.sh" "$TEMP_DIR_FOR_PARSERS/usr/share/apel/slurm_acc.sh"
+
+# Copy files into apel-client
+cp "$SOURCE_DIR/$APEL_DIR/bin/client.py" "$TEMP_DIR_FOR_CLIENT/usr/bin/apelclient"
+cp "$SOURCE_DIR/$APEL_DIR/conf/client.cfg" "$TEMP_DIR_FOR_CLIENT/etc/apel/client.cfg"
+cp "$SOURCE_DIR/$APEL_DIR/schemas/client.sql" "$TEMP_DIR_FOR_CLIENT/usr/share/apel/client.sql"
+cp "$SOURCE_DIR/$APEL_DIR/scripts/apel-client" "$TEMP_DIR_FOR_CLIENT/etc/logrotate.d/apel-client"
+
+# Copy files into apel-server
+cp "$SOURCE_DIR/$APEL_DIR/bin/dbloader.py"  "$TEMP_DIR_FOR_SERVER/usr/bin/apeldbloader"
+cp "$SOURCE_DIR/$APEL_DIR/bin/dbunloader.py"  "$TEMP_DIR_FOR_SERVER/usr/bin/apeldbunloader"
+cp "$SOURCE_DIR/$APEL_DIR/bin/summariser.py"  "$TEMP_DIR_FOR_SERVER/usr/bin/apelsummariser"
+cp "$SOURCE_DIR/$APEL_DIR/bin/retrieve_dns.py"  "$TEMP_DIR_FOR_SERVER/usr/bin/apelauth"
+cp "$SOURCE_DIR/$APEL_DIR/conf/"{auth,db,loader,summariser,unloader}.cfg  "$TEMP_DIR_FOR_SERVER/etc/apel/"
+cp "$SOURCE_DIR/$APEL_DIR/schemas/"{cloud,server-extra,server,storage}.sql  "$TEMP_DIR_FOR_SERVER/usr/share/apel/"
+cp "$SOURCE_DIR/$APEL_DIR/scripts/msg_status.py"  "$TEMP_DIR_FOR_SERVER/usr/share/apel/"
+cp "$SOURCE_DIR/$APEL_DIR/scripts/update-"*  "$TEMP_DIR_FOR_SERVER/usr/share/apel/"
+
+# Copy files into apel-lib
+cp -r "$SOURCE_DIR/$APEL_DIR/apel/"* "$TEMP_DIR_FOR_LIB/$PYTHON_ROOT_DIR/$LIB_EXTENSION/apel/"
+# In addition we make apel into compiled bytecode to replicate what rpm was doing.
+$PY_VERSION -m compileall "$TEMP_DIR_FOR_LIB/$PYTHON_ROOT_DIR/$LIB_EXTENSION/apel/"
+
+# Set up dependencies for apel-lib based on Python version
+if [[ ${PY_NUM:0:1} == "3" ]]; then
+    echo "Building $VERSION iteration $ITERATION for Python $PY_NUM as $PACK_TYPE."
+    FPM_PYTHON="--depends python3 \
+        --depends python3-pip \
+        --depends python3-ldap \
+        --depends openldap-devel \
+        --depends python3-dirq \
+        --depends python3-iso8601 \
+        --depends python3-mysqlclient \
+        --depends python3-future "
+    FPM_PYTHON_SERVER_DEPS="--depends python3-daemon"
+elif [[ ${PY_NUM:0:1} == "2" ]]; then
+    echo "Building $VERSION iteration $ITERATION for Python $PY_NUM as $PACK_TYPE."
+    # This dependencies is for python2 in el7 environment.
+    FPM_PYTHON="--depends python2 \
+        --depends python2-pip \
+        --depends python-ldap \
+        --depends openldap-devel \
+        --depends python-dirq \
+        --depends python-iso8601 \
+        --depends MySQL-python \
+        --depends python2-future "
+    FPM_PYTHON_SERVER_DEPS="--depends python-daemon"
+fi
+
+# Build the packages
+SOURCE_TYPE="dir"
+ARCH_TYPE="noarch"
+ADMIN_CONTACT_EMAIL="[email protected]"
+APEL_WIKI_URL="https://wiki.egi.eu/wiki/APEL"
+LICENSE="ASL 2.0"
+GROUP="Development/Languages"
+
+FPM_CORE="fpm -s $SOURCE_TYPE \
+    -t $PACK_TYPE \
+    -v $VERSION \
+    --iteration $ITERATION \
+    --architecture $ARCH_TYPE \
+    --license \"$LICENSE\" \
+    --url $APEL_WIKI_URL \
+    --category $GROUP \
+    --python-bin /usr/bin/$PY_VERSION \
+    --python-install-lib $PYTHON_ROOT_DIR$LIB_EXTENSION \
+    --\"$PACK_TYPE\"-dist $OS_EXTENSION \
+    -m \"Apel Administrators <$ADMIN_CONTACT_EMAIL>\" \
+    --rpm-user root \
+    --rpm-group root \
+    --prefix / \
+    -p $BUILD_DIR \
+    --depends \"shadow-utils\" \
+    --no-auto-depends "
+
+FPM_PARSERS_PACKAGE="-n apel-parsers \
+    -C $TEMP_DIR_FOR_PARSERS \
+    --depends \"apel-lib >= ${VERSION}\" \
+    --rpm-attr 755,root,root:/usr/bin/apelparser \
+    --rpm-attr 600,-,-:/etc/apel/parser.cfg \
+    --rpm-attr 755,root,root:/usr/share/apel/slurm_acc.sh \
+    --rpm-attr 755,root,root:/usr/share/apel/htcondor_acc.sh \
+    --config-files /etc/apel/parser.cfg \
+    --description \"Parsers for APEL system\" "
+
+FPM_CLIENT_PACKAGE="-n apel-client \
+    -C $TEMP_DIR_FOR_CLIENT \
+    --depends \"apel-lib >= ${VERSION}\" \
+    --depends \"apel-ssm >= 3.2.0\" \
+    --rpm-attr 755,root,root:/usr/bin/apelclient \
+    --rpm-attr 600,-,-:/etc/apel/client.cfg \
+    --config-files /etc/apel/client.cfg \
+    --config-files /etc/logrotate.d/apel-client \
+    --description \"APEL client package\" "
+
+FPM_SERVER_PACKAGE="-n apel-server \
+    --before-install \"$SOURCE_DIR/$APEL_DIR/scripts/apel-server-helper.sh\" \
+    -C $TEMP_DIR_FOR_SERVER \
+    --depends \"apel-lib >= ${VERSION}\" \
+    --rpm-attr 755,root,root:/usr/share/apel/msg_status.py \
+    --rpm-attr 755,root,root:/usr/bin/apeldbunloader \
+    --rpm-attr 755,root,root:/usr/bin/apeldbloader \
+    --rpm-attr 755,root,root:/usr/bin/apelsummariser \
+    --rpm-attr 755,root,root:/usr/bin/apelauth \
+    --rpm-attr 600,apel,apel:/etc/apel/db.cfg \
+    --config-files /etc/apel/summariser.cfg \
+    --config-files /etc/apel/unloader.cfg \
+    --config-files /etc/apel/loader.cfg \
+    --config-files /etc/apel/db.cfg \
+    --config-files /etc/apel/auth.cfg \
+    --description \"APEL server package\" "
+
+FPM_LIB_PACKAGE="-n apel-lib \
+    -C $TEMP_DIR_FOR_LIB \
+    --description \"Libraries required for Apel Client, Server and Parsers\" "
+
+# Construct and evaluate the FPM call
+BUILD_PACKAGE_COMMAND_PARSERS="${FPM_CORE}${FPM_PARSERS_PACKAGE}${VERBOSE} ."
+eval "$BUILD_PACKAGE_COMMAND_PARSERS"
+
+BUILD_PACKAGE_COMMAND_CLIENT="${FPM_CORE}${FPM_CLIENT_PACKAGE}${VERBOSE} ."
+eval "$BUILD_PACKAGE_COMMAND_CLIENT"
+
+BUILD_PACKAGE_COMMAND_SERVER="${FPM_CORE}${FPM_SERVER_PACKAGE}${FPM_PYTHON_SERVER_DEPS}${VERBOSE} ."
+eval "$BUILD_PACKAGE_COMMAND_SERVER"
+
+BUILD_PACKAGE_COMMAND_LIB="${FPM_CORE}${FPM_LIB_PACKAGE}${FPM_PYTHON}${VERBOSE} ."
+eval "$BUILD_PACKAGE_COMMAND_LIB"
+
+echo "== Generating pleaserun package =="
+
+FILE_TO_EXECUTE="/usr/bin/${PY_VERSION} /usr/bin/apeldbloader"
+
+# When installed, use pleaserun to perform system specific service setup
+fpm -s pleaserun -t "$PACK_TYPE" \
+    -n apeldbloader-service \
+    -v "$VERSION" \
+    --iteration "$ITERATION" \
+    --"$PACK_TYPE"-dist "$OS_EXTENSION" \
+    -m "Apel Administrators <$ADMIN_CONTACT_EMAIL>" \
+    --description "Accounting Processor for Event Logs (APEL) apeldbloader Daemon service" \
+    --architecture $ARCH_TYPE \
+    --no-auto-depends \
+    --depends "apel-lib >= $VERSION" \
+    --package "$BUILD_DIR" \
+    "$FILE_TO_EXECUTE"
+
+echo "== rpmlint performing checks =="
+
+# `dirname`: Output each NAME with its last non-slash component and trailing slashes removed.
+# For instance; Given $(dirname /root/rpmb/rpmbuild/source) will output "/root/rpmb/rpmbuild".
+# rpmlint: To check common problems in rpm packages.
+rpmlint "$(dirname "$SOURCE_DIR")"
+
+# Cleanup
+rm -rf "$TEMP_DIR_FOR_PARSERS" "$TEMP_DIR_FOR_CLIENT" "$TEMP_DIR_FOR_SERVER" "$TEMP_DIR_FOR_LIB"

scripts/apel-server-helper.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Create the apel user if it doesn't exist
+if ! getent passwd apel >/dev/null; then
+    useradd -r apel
+fi