Improve proxy management and binary builds.

Dockerfile

@@ -2,7 +2,7 @@
 
 ### Globally scoped ARG, defined here to be available to use in FROM statements
 # Do we want to use a proxy?
-ARG useproxy=no
+ARG useproxy=without
 # TODO: move to pS provided base OS image
 # OS image to use as a base
 ARG OSimage=debian:stretch
@@ -15,20 +15,20 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # If you want to use a proxy to speed up download both at build time and test time (docker run)
 # Trick built on top of https://medium.com/@tonistiigi/advanced-multi-stage-build-patterns-6f741b852fae
-FROM pre-base AS base-proxy-yes
+FROM pre-base AS base-with-proxy
 ARG proxy
 ENV http_proxy=http://${proxy}
 ENV https_proxy=https://${proxy}
 ENV no_proxy=localhost,127.0.0.1
 
-FROM pre-base AS base-proxy-no
+FROM pre-base AS base-without-proxy
 ENV http_proxy=
 ENV https_proxy=
 ENV no_proxy=
 
 ### Systemd related setup
 # TODO: should be moved to dedicated image
-FROM base-proxy-${useproxy} AS ps-base-image
+FROM base-${useproxy}-proxy AS ps-base-image
 RUN echo "This Docker image is using proxy: ${https_proxy:-none}"
 RUN apt-get update && apt-get install -y \
         apt-utils \

README.md

@@ -4,14 +4,10 @@ This directoy needs to be at the same level as the other perfSONAR repositories
 ## Build packages
 To build a Debian/Ubuntu package, run something like:
 
-./build-in-docker -b 4.4.0 toolkit
+./build-in-docker toolkit
 
- * args and environment variables:
- ** `-b` the branch of the repository you want to build (snapshot package)
- ** `-t` the tag of the repository you want to build (staging/release package)
- ** `ARCHES` can be set to a list of architectures to build for
-
-TODO: make it also work for pScheduler
+* args and environment variables:
+  * `-t` the tag of the repository you want to build (staging/release package)
 
 The resulting packages will be located in a new directory at the same level as the other repositories called `build_results`
 
@@ -20,9 +16,14 @@ The build will use the information from the `debian/gbp.conf` file to know on wh
 
 ### Examples
 
- * Building `pscheduler-archiver-rabbitmq` on debian:stretch using a local proxy:
-`export useproxy=yes proxy=172.17.0.1:3128 OSimage=debian:stretch REPO=perfsonar-minor-snapshot`
-`package=pscheduler-archiver-rabbitmq ./build-in-docker -b 4.4.0 pscheduler`
+* Building `perfsonar-graphs` located in the `../graphs` directory:
+```
+./build-in-docker graphs
+```
+* Building `pscheduler-archiver-rabbitmq` using a local proxy:
+```
+proxy=172.17.0.1:3128 package=pscheduler-archiver-rabbitmq ./build-in-docker pscheduler
+```
 
 ## Test packages
 To test the installation of a resulting package, run something like:
@@ -40,13 +41,13 @@ Fully Supported:
 You can run the tests by executing `test_install_instructions.sh $REPO $OS $BUNDLE` (with all args optional but the previous one always required) where:
 
  * `$REPO` is one of these (default is to use production):
- ** `perfsonar-release` (production)
- ** `perfsonar-patch-snapshot`
- ** `perfsonar-minor-snapshot`
- ** `perfsonar-patch-staging`
+   * `perfsonar-release` (production)
+   * `perfsonar-patch-snapshot`
+   * `perfsonar-minor-snapshot`
+   * `perfsonar-patch-staging`
  * `$OS` is one of these (default to test all):
- ** `debian:stretch`
- ** `debian:buster`
- ** `ubuntu:xenial`
- ** `ubuntu:bionic`
+   * `debian:stretch`
+   * `debian:buster`
+   * `ubuntu:xenial`
+   * `ubuntu:bionic`
 

build-in-docker

@@ -7,6 +7,7 @@ show_help() {
     This script will build docker images and launch a git repo build in a container
     Requirements:
         - Needs to be in a directory at the same level as the gitrepo to be built
+        - The source branch to build from need to be checked out
         - Functionnal docker buildx setup
 
     Arguments understood:
@@ -29,8 +30,8 @@ EOF
 }
 
 # Defaults
-declare -a ARCHES=("linux/amd64")
-#declare -a ARCHES=("linux/amd64" "linux/arm64")
+#declare -a ARCHES=("linux/amd64")
+declare -a ARCHES=("linux/amd64" "linux/arm64")
 #declare -a ARCHES=("linux/amd64" "linux/ppc64le")
 # Valid values for ARCH:
 # linux/amd64
@@ -42,20 +43,23 @@ declare -a ARCHES=("linux/amd64")
 build_docker_images=true
 source_and_binary=true
 need_shift=false
+OTHEROPT=1
 
 # Parsing options
 while getopts "dBDhb:t:ksu" OPT; do
     case $OPT in
         d) build_docker_images=false ; need_shift=true ;;
         B) source_and_binary=false ; need_shift=true ;;
         D) debug_build="bash" ; need_shift=true ;;
+# The other options will get passed to ps-source-builder so we must not shift them
+        b|t) ((OTHEROPT+=2)) ;;
+        k|s|u) ((OTHEROPT+=1)) ;;
         h)
             show_help >&2
             exit 1 ;;
-# The other options will get passed to ps-source-builder
     esac
 done
-[ $need_shift ] && shift
+shift $((OPTIND-OTHEROPT))
 
 # Get the gitrepo to build
 GITREPO=`echo $@ | awk '{print $NF}'`
@@ -86,7 +90,13 @@ case $OSimage in
     focal) OSimage=ubuntu:$OSimage ;;
     stretch) OSimage=debian:$OSimage ;;
 esac
-export OSimage REPO useproxy
+export OSimage REPO
+if [ -n "$proxy" ]; then
+    # If $proxy is set, then we will use it
+    export useproxy=with
+else
+    export useproxy=without
+fi
 
 # Initialise
 LOGS_PREFIX="logs/binary_build"

docker-compose.yml

@@ -10,8 +10,8 @@ x-build-args:
   OSimage: "$OSimage"
   ARCH: "$ARCH"
   REPO: "$REPO"
-  useproxy: "${useproxy:-no}"
-  proxy: "$proxy"
+  useproxy: "${useproxy:-without}"
+  proxy: "${proxy:-}"
 
 x-tmpfs:
   &default-tmpfs

ps-binary-builder

@@ -11,7 +11,7 @@ REPO_LOC='/mnt/build/'
 BUILD_SRC=${REPO_LOC}'build_source'
 BUILD_RESULTS=${REPO_LOC}'build_results'
 SRC_DIR='source'
-BUILD_DIR='build'
+BUILD_DIR='binary'
 # Go where the source package is
 cd ${BUILD_SRC}/${GITREPO}
 
@@ -22,7 +22,7 @@ cd ${BUILD_SRC}/${GITREPO}
 
 # Extract source package
 mkdir -p $BUILD_DIR
-tar -x -C $BUILD_DIR -f *.orig.*
+tar -x -C $BUILD_DIR --strip-components 1 -f *.orig.*
 tar -x -C $BUILD_DIR -f *.debian.*
 cd $BUILD_DIR
 

ps-source-builder

@@ -5,13 +5,11 @@ shopt -s extglob
 # Usage info
 show_help() {
     cat << EOF
-    Usage: ${0##*/} [-b DEBIAN_BRANCH] [-t DEBIAN_TAG] [-s] [-u] [REPO]
+    Usage: ${0##*/} [-t DEBIAN_TAG] [-s] [-u] [REPO]
     This script builds a perfSONAR package out of a git REPO.
     The build happens for the $DIST Debian distro (\$DIST should be set in your ENV).
 
     Arguments understood:
-        -b: git debian branch to build from (default to master)
-            (upstream is read from gbp.conf)
         -k: keep locally built packages (can be useful to solve dependencies)
         -s: build only the source package
         -t: git tag to build from (defaults no tag), overrides any -b arg
@@ -20,13 +18,13 @@ show_help() {
             (useful for subsequent builds with no change to sources)
 
     Some examples:
-    ./ps-cowbuilder-build -b 4.2.1 toolkit
-    ./ps-cowbuilder-build -t debian/stretch/4.2.1 toolkit
-    package=pscheduler-archiver-esmond ./ps-cowbuilder-build -b 4.3.0 pscheduler
+    ./${0##*/} toolkit
+    ./${0##*/} -t debian/stretch/4.2.1 toolkit
+    package=pscheduler-archiver-esmond ./${0##*/} pscheduler
 EOF
 #    Other examples: to rebuild everything in the local repo:
-#    for p in `ls -al pscheduler/source/ | awk '/pscheduler/ {print $9}' | grep -vE "pscheduler-(docs|rpm)" | sort -r`; do package=$p ./ps-cowbuilder-build -b 4.3.0 pscheduler; done
-#    for p in jq python-icmperror python-jsonschema python-jsontemplate python-pyjq python-pyrsistent python-radix python-pscheduler `ls pscheduler/source | grep -E "^pscheduler" | grep -v "pscheduler-rpm"`; do package=$p ./ps-cowbuilder-build -b debian-python-3 pscheduler; done
+#    for p in `ls -al pscheduler/source/ | awk '/pscheduler/ {print $9}' | grep -vE "pscheduler-(docs|rpm)" | sort -r`; do package=$p ./ps-source-builder pscheduler; done
+#    for p in jq python-icmperror python-jsonschema python-jsontemplate python-pyjq python-pyrsistent python-radix python-pscheduler `ls pscheduler/source | grep -E "^pscheduler" | grep -v "pscheduler-rpm"`; do package=$p ./ps-source-builder pscheduler; done
 }
 
 # Defaults
@@ -41,16 +39,14 @@ SRC_DIR='source'
 cd ${BUILD_SRC}
 
 # Default values
-branch='master'
 unset tag
 unset source_only
 git_update=true
 keep_builds=false
 
 # Parsing options
-while getopts "b:t:ksu" OPT; do
+while getopts "t:ksu" OPT; do
     case $OPT in
-        b) branch=$OPTARG ;;
         k) keep_builds=true ;;
         s) source_only=true ;;
         t) tag=$OPTARG ;;
@@ -71,34 +67,23 @@ if [ -z "$REPO" ]; then
     exit 1
 fi
 
-# Keep previous builds?
-if ! $keep_builds ; then
-    rm -rf $REPO/!(source)
-fi
-
-# Do we have a tag?
-if [ -z "$tag" ]; then
-    # We don't have a tag, we build a snapshot from $branch
-    export branch
-    DIST="${branch##*\/}"
-else
-    # We have a tag, say we will build from it
-    tagl="${tag#*\/}"
-    DIST="${tagl%%\/*}"
-    export tag
-fi
-
 # Do we want to clone the repo or do we have it already existing?
 if $git_update ; then
-    # Checkout source from local dev repo and merge upstream branch into ${DEBIAN_BRANCH}
+    # Checkout source from local dev repo
     rm -rf "${REPO}"
     git clone "${REPO_LOC}${REPO}" "${REPO}/${SRC_DIR}"
     cd "${REPO}/${SRC_DIR}"
-    if [ -n "$tag" ] ; then
+    # Do we have a tag?
+    if [ -n "$tag" ]; then
+        # We have a tag, say we will build from it (build-source-package.sh uses it)
+        export tag
         branch=`git branch --contains "tags/${tag}" | awk '{print $2}'`
+    else
+        # We build from current branch
+        branch=`git rev-parse --abbrev-ref HEAD`
     fi
     git checkout "$branch"
-elif [[ -d "${REPO}/${SRC_DIR}" && ${branch} ]]; then
+elif [ -d "${REPO}/${SRC_DIR}" ]; then
     echo "I'm using the existing ${REPO} directory."
     cd "${REPO}/${SRC_DIR}/.."
     rm -f .git
@@ -107,21 +92,22 @@ elif [[ -d "${REPO}/${SRC_DIR}" && ${branch} ]]; then
     mkdir -p "${SRC_DIR}"
     cd "${SRC_DIR}"
     mv ../.git .
-    git checkout -B "${branch}"
+    # Restoring pristine repo
     git checkout -- *
+    branch=`git rev-parse --abbrev-ref HEAD`
 else
     echo "You asked me to reuse the existing ${REPO} directory but it doesn't seem right."
     exit 1
 fi
 
 # Announce what we'll do!
-echo -e "\nBuilding \033[1;32m${REPO}\033[0;37m from branch \033[1;36m${branch}\033[0;37m on \033[1;32m${ARCHES}\033[0;37m.\n"
+echo -e "\nBuilding \033[1;32m${REPO}\033[0;37m from branch \033[1;36m${branch}\033[0;37m on \033[1;32m${ARCH}\033[0;37m.\n"
 [ $tag ] && echo -e "\033[1;32mThis is a \033[1;33mrelease\033[1;32m, tag \033[1;33m${tag}\033[1;32m will be built !\033[0m\n"
-# TODO: keep local packages somewhere
+# TODO: Pass this directory to build-source-package to solve dependencies
 if $keep_builds ; then
-    echo "To solve dependencies, I will also use packages from /var/cache/pbuilder/$DIST/result if any."
+    echo "To solve dependencies, I will also use packages from ${REPO_LOC}/build_results if any."
 else
-    rm -rf /var/cache/pbuilder/result/$DIST/*
+    rm -rf ${REPO_LOC}/build_results/*
 fi
 
 cd ..