From 7a7dbb10f967741c177d0315320e95c28649fd14 Mon Sep 17 00:00:00 2001 From: Antoine Delvaux Date: Mon, 13 Sep 2021 16:43:11 +0100 Subject: [PATCH] Improve proxy management and binary builds. --- Dockerfile | 8 +++---- README.md | 37 +++++++++++++++-------------- build-in-docker | 20 ++++++++++++---- docker-compose.yml | 4 ++-- ps-binary-builder | 4 ++-- ps-source-builder | 58 ++++++++++++++++++---------------------------- 6 files changed, 64 insertions(+), 67 deletions(-) diff --git a/Dockerfile b/Dockerfile index aff5b97..54716fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ### Globally scoped ARG, defined here to be available to use in FROM statements # Do we want to use a proxy? -ARG useproxy=no +ARG useproxy=without # TODO: move to pS provided base OS image # OS image to use as a base ARG OSimage=debian:stretch @@ -15,20 +15,20 @@ ENV DEBIAN_FRONTEND=noninteractive # If you want to use a proxy to speed up download both at build time and test time (docker run) # Trick built on top of https://medium.com/@tonistiigi/advanced-multi-stage-build-patterns-6f741b852fae -FROM pre-base AS base-proxy-yes +FROM pre-base AS base-with-proxy ARG proxy ENV http_proxy=http://${proxy} ENV https_proxy=https://${proxy} ENV no_proxy=localhost,127.0.0.1 -FROM pre-base AS base-proxy-no +FROM pre-base AS base-without-proxy ENV http_proxy= ENV https_proxy= ENV no_proxy= ### Systemd related setup # TODO: should be moved to dedicated image -FROM base-proxy-${useproxy} AS ps-base-image +FROM base-${useproxy}-proxy AS ps-base-image RUN echo "This Docker image is using proxy: ${https_proxy:-none}" RUN apt-get update && apt-get install -y \ apt-utils \ diff --git a/README.md b/README.md index 97da4f9..68e9cf4 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,10 @@ This directoy needs to be at the same level as the other perfSONAR repositories ## Build packages To build a Debian/Ubuntu package, run something like: -./build-in-docker -b 4.4.0 toolkit +./build-in-docker toolkit - * args and environment variables: - ** `-b` the branch of the repository you want to build (snapshot package) - ** `-t` the tag of the repository you want to build (staging/release package) - ** `ARCHES` can be set to a list of architectures to build for - -TODO: make it also work for pScheduler +* args and environment variables: + * `-t` the tag of the repository you want to build (staging/release package) The resulting packages will be located in a new directory at the same level as the other repositories called `build_results` @@ -20,9 +16,14 @@ The build will use the information from the `debian/gbp.conf` file to know on wh ### Examples - * Building `pscheduler-archiver-rabbitmq` on debian:stretch using a local proxy: -`export useproxy=yes proxy=172.17.0.1:3128 OSimage=debian:stretch REPO=perfsonar-minor-snapshot` -`package=pscheduler-archiver-rabbitmq ./build-in-docker -b 4.4.0 pscheduler` +* Building `perfsonar-graphs` located in the `../graphs` directory: +``` +./build-in-docker graphs +``` +* Building `pscheduler-archiver-rabbitmq` using a local proxy: +``` +proxy=172.17.0.1:3128 package=pscheduler-archiver-rabbitmq ./build-in-docker pscheduler +``` ## Test packages To test the installation of a resulting package, run something like: @@ -40,13 +41,13 @@ Fully Supported: You can run the tests by executing `test_install_instructions.sh $REPO $OS $BUNDLE` (with all args optional but the previous one always required) where: * `$REPO` is one of these (default is to use production): - ** `perfsonar-release` (production) - ** `perfsonar-patch-snapshot` - ** `perfsonar-minor-snapshot` - ** `perfsonar-patch-staging` + * `perfsonar-release` (production) + * `perfsonar-patch-snapshot` + * `perfsonar-minor-snapshot` + * `perfsonar-patch-staging` * `$OS` is one of these (default to test all): - ** `debian:stretch` - ** `debian:buster` - ** `ubuntu:xenial` - ** `ubuntu:bionic` + * `debian:stretch` + * `debian:buster` + * `ubuntu:xenial` + * `ubuntu:bionic` diff --git a/build-in-docker b/build-in-docker index 1b78eca..681aee3 100755 --- a/build-in-docker +++ b/build-in-docker @@ -7,6 +7,7 @@ show_help() { This script will build docker images and launch a git repo build in a container Requirements: - Needs to be in a directory at the same level as the gitrepo to be built + - The source branch to build from need to be checked out - Functionnal docker buildx setup Arguments understood: @@ -29,8 +30,8 @@ EOF } # Defaults -declare -a ARCHES=("linux/amd64") -#declare -a ARCHES=("linux/amd64" "linux/arm64") +#declare -a ARCHES=("linux/amd64") +declare -a ARCHES=("linux/amd64" "linux/arm64") #declare -a ARCHES=("linux/amd64" "linux/ppc64le") # Valid values for ARCH: # linux/amd64 @@ -42,6 +43,7 @@ declare -a ARCHES=("linux/amd64") build_docker_images=true source_and_binary=true need_shift=false +OTHEROPT=1 # Parsing options while getopts "dBDhb:t:ksu" OPT; do @@ -49,13 +51,15 @@ while getopts "dBDhb:t:ksu" OPT; do d) build_docker_images=false ; need_shift=true ;; B) source_and_binary=false ; need_shift=true ;; D) debug_build="bash" ; need_shift=true ;; +# The other options will get passed to ps-source-builder so we must not shift them + b|t) ((OTHEROPT+=2)) ;; + k|s|u) ((OTHEROPT+=1)) ;; h) show_help >&2 exit 1 ;; -# The other options will get passed to ps-source-builder esac done -[ $need_shift ] && shift +shift $((OPTIND-OTHEROPT)) # Get the gitrepo to build GITREPO=`echo $@ | awk '{print $NF}'` @@ -86,7 +90,13 @@ case $OSimage in focal) OSimage=ubuntu:$OSimage ;; stretch) OSimage=debian:$OSimage ;; esac -export OSimage REPO useproxy +export OSimage REPO +if [ -n "$proxy" ]; then + # If $proxy is set, then we will use it + export useproxy=with +else + export useproxy=without +fi # Initialise LOGS_PREFIX="logs/binary_build" diff --git a/docker-compose.yml b/docker-compose.yml index 96a61f9..f7423a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,8 @@ x-build-args: OSimage: "$OSimage" ARCH: "$ARCH" REPO: "$REPO" - useproxy: "${useproxy:-no}" - proxy: "$proxy" + useproxy: "${useproxy:-without}" + proxy: "${proxy:-}" x-tmpfs: &default-tmpfs diff --git a/ps-binary-builder b/ps-binary-builder index 7562285..f725d4c 100755 --- a/ps-binary-builder +++ b/ps-binary-builder @@ -11,7 +11,7 @@ REPO_LOC='/mnt/build/' BUILD_SRC=${REPO_LOC}'build_source' BUILD_RESULTS=${REPO_LOC}'build_results' SRC_DIR='source' -BUILD_DIR='build' +BUILD_DIR='binary' # Go where the source package is cd ${BUILD_SRC}/${GITREPO} @@ -22,7 +22,7 @@ cd ${BUILD_SRC}/${GITREPO} # Extract source package mkdir -p $BUILD_DIR -tar -x -C $BUILD_DIR -f *.orig.* +tar -x -C $BUILD_DIR --strip-components 1 -f *.orig.* tar -x -C $BUILD_DIR -f *.debian.* cd $BUILD_DIR diff --git a/ps-source-builder b/ps-source-builder index b4eb7dd..718d94d 100755 --- a/ps-source-builder +++ b/ps-source-builder @@ -5,13 +5,11 @@ shopt -s extglob # Usage info show_help() { cat << EOF - Usage: ${0##*/} [-b DEBIAN_BRANCH] [-t DEBIAN_TAG] [-s] [-u] [REPO] + Usage: ${0##*/} [-t DEBIAN_TAG] [-s] [-u] [REPO] This script builds a perfSONAR package out of a git REPO. The build happens for the $DIST Debian distro (\$DIST should be set in your ENV). Arguments understood: - -b: git debian branch to build from (default to master) - (upstream is read from gbp.conf) -k: keep locally built packages (can be useful to solve dependencies) -s: build only the source package -t: git tag to build from (defaults no tag), overrides any -b arg @@ -20,13 +18,13 @@ show_help() { (useful for subsequent builds with no change to sources) Some examples: - ./ps-cowbuilder-build -b 4.2.1 toolkit - ./ps-cowbuilder-build -t debian/stretch/4.2.1 toolkit - package=pscheduler-archiver-esmond ./ps-cowbuilder-build -b 4.3.0 pscheduler + ./${0##*/} toolkit + ./${0##*/} -t debian/stretch/4.2.1 toolkit + package=pscheduler-archiver-esmond ./${0##*/} pscheduler EOF # Other examples: to rebuild everything in the local repo: -# for p in `ls -al pscheduler/source/ | awk '/pscheduler/ {print $9}' | grep -vE "pscheduler-(docs|rpm)" | sort -r`; do package=$p ./ps-cowbuilder-build -b 4.3.0 pscheduler; done -# for p in jq python-icmperror python-jsonschema python-jsontemplate python-pyjq python-pyrsistent python-radix python-pscheduler `ls pscheduler/source | grep -E "^pscheduler" | grep -v "pscheduler-rpm"`; do package=$p ./ps-cowbuilder-build -b debian-python-3 pscheduler; done +# for p in `ls -al pscheduler/source/ | awk '/pscheduler/ {print $9}' | grep -vE "pscheduler-(docs|rpm)" | sort -r`; do package=$p ./ps-source-builder pscheduler; done +# for p in jq python-icmperror python-jsonschema python-jsontemplate python-pyjq python-pyrsistent python-radix python-pscheduler `ls pscheduler/source | grep -E "^pscheduler" | grep -v "pscheduler-rpm"`; do package=$p ./ps-source-builder pscheduler; done } # Defaults @@ -41,16 +39,14 @@ SRC_DIR='source' cd ${BUILD_SRC} # Default values -branch='master' unset tag unset source_only git_update=true keep_builds=false # Parsing options -while getopts "b:t:ksu" OPT; do +while getopts "t:ksu" OPT; do case $OPT in - b) branch=$OPTARG ;; k) keep_builds=true ;; s) source_only=true ;; t) tag=$OPTARG ;; @@ -71,34 +67,23 @@ if [ -z "$REPO" ]; then exit 1 fi -# Keep previous builds? -if ! $keep_builds ; then - rm -rf $REPO/!(source) -fi - -# Do we have a tag? -if [ -z "$tag" ]; then - # We don't have a tag, we build a snapshot from $branch - export branch - DIST="${branch##*\/}" -else - # We have a tag, say we will build from it - tagl="${tag#*\/}" - DIST="${tagl%%\/*}" - export tag -fi - # Do we want to clone the repo or do we have it already existing? if $git_update ; then - # Checkout source from local dev repo and merge upstream branch into ${DEBIAN_BRANCH} + # Checkout source from local dev repo rm -rf "${REPO}" git clone "${REPO_LOC}${REPO}" "${REPO}/${SRC_DIR}" cd "${REPO}/${SRC_DIR}" - if [ -n "$tag" ] ; then + # Do we have a tag? + if [ -n "$tag" ]; then + # We have a tag, say we will build from it (build-source-package.sh uses it) + export tag branch=`git branch --contains "tags/${tag}" | awk '{print $2}'` + else + # We build from current branch + branch=`git rev-parse --abbrev-ref HEAD` fi git checkout "$branch" -elif [[ -d "${REPO}/${SRC_DIR}" && ${branch} ]]; then +elif [ -d "${REPO}/${SRC_DIR}" ]; then echo "I'm using the existing ${REPO} directory." cd "${REPO}/${SRC_DIR}/.." rm -f .git @@ -107,21 +92,22 @@ elif [[ -d "${REPO}/${SRC_DIR}" && ${branch} ]]; then mkdir -p "${SRC_DIR}" cd "${SRC_DIR}" mv ../.git . - git checkout -B "${branch}" + # Restoring pristine repo git checkout -- * + branch=`git rev-parse --abbrev-ref HEAD` else echo "You asked me to reuse the existing ${REPO} directory but it doesn't seem right." exit 1 fi # Announce what we'll do! -echo -e "\nBuilding \033[1;32m${REPO}\033[0;37m from branch \033[1;36m${branch}\033[0;37m on \033[1;32m${ARCHES}\033[0;37m.\n" +echo -e "\nBuilding \033[1;32m${REPO}\033[0;37m from branch \033[1;36m${branch}\033[0;37m on \033[1;32m${ARCH}\033[0;37m.\n" [ $tag ] && echo -e "\033[1;32mThis is a \033[1;33mrelease\033[1;32m, tag \033[1;33m${tag}\033[1;32m will be built !\033[0m\n" -# TODO: keep local packages somewhere +# TODO: Pass this directory to build-source-package to solve dependencies if $keep_builds ; then - echo "To solve dependencies, I will also use packages from /var/cache/pbuilder/$DIST/result if any." + echo "To solve dependencies, I will also use packages from ${REPO_LOC}/build_results if any." else - rm -rf /var/cache/pbuilder/result/$DIST/* + rm -rf ${REPO_LOC}/build_results/* fi cd ..