(CI) enable debug & many small fixes (#737)

tue-robotics · May 8, 2024 · dc30cfc · dc30cfc
2 parents 270185a + 2ca613a
commit dc30cfc
Show file tree

Hide file tree

Showing 21 changed files with 305 additions and 206 deletions.
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
@@ -10,3 +10,5 @@ d71897d20d9816b2c86bb23d2d8ba78a24622a8d
 7b2835b380d6b63c9953ecb5b7f3258a98d08022
 # Make lsb_release.py black
 287a466e1b3e2018deac2d9214439c705325bc5c
+# (CI) consistent echo colors
+1009e550b338f36656483d934e5dde39ade13c53
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -118,23 +118,24 @@ jobs:
           # shellcheck disable=SC2296
           TARGET_IMAGE="${{ env.REGISTRY }}/tue-robotics/tue-env-ros-${{ matrix.ros-distro }}${IMAGE_FLAVOR}:${IMAGE_TAG}-${{ matrix.platform }}"
           SSH_KNOWN_HOSTS="github.com"
-          SSH_ARG=()
+          SSH_ARGS=()
           if [[ -n "${SSH_KEY}" ]]
           then
-              SSH_KEY_PATH="${HOME}/.ssh/ci_ssh_key"
+              SSH_KEY_PATH="${HOME}"/.ssh/ci_ssh_key
+              echo -e "\e[35;1mmkdir -p ${HOME}/.ssh\e[0m"
               mkdir -p "${HOME}"/.ssh
               echo "${SSH_KEY}" > "${SSH_KEY_PATH}"
               chmod 600 "${SSH_KEY_PATH}"
               eval "$(ssh-agent -s)"
-              SSH_ARG+=("--ssh" "--ssh-key=${SSH_KEY_PATH}")
+              SSH_ARGS+=("--ssh" "--ssh-key=${SSH_KEY_PATH}")
           fi
           if [[ -n "${SSH_KNOWN_HOSTS}" ]]
           then
-              echo -e "\e[35m\e[1mmkdir -p ${HOME}/.ssh\e[0m"
+              echo -e "\e[35;1mmkdir -p ${HOME}/.ssh\e[0m"
               mkdir -p "${HOME}"/.ssh
               for host in ${SSH_KNOWN_HOSTS}
               do
-                  echo -e "\e[35m\e[1mssh-keyscan -t rsa -H \"${host}\" 2>&1 | tee -a ${HOME}/.ssh/known_hosts\e[0m"
+                  echo -e "\e[35;1mssh-keyscan -t rsa -H \"${host}\" 2>&1 | tee -a ${HOME}/.ssh/known_hosts\e[0m"
                   ssh-keyscan -t rsa -H "${host}" 2>&1 | tee -a "${HOME}"/.ssh/known_hosts
               done
           fi
@@ -168,10 +169,10 @@ jobs:
               --virtualenv-include-system-site-packages="${VENV_INCLUDE_SYSTEM_SITE}" \
               --docker_user="${DOCKER_USER}" \
               --docker_user_id="${DOCKER_USER_ID}" \
-              "${SSH_ARG[@]}"
-          echo -e "\e[35m\e[1mimage=${TARGET_IMAGE} >> \${GITHUB_OUTPUT}\e[0m"
+              "${SSH_ARGS[@]}"
+          echo -e "\e[35;1mimage=${TARGET_IMAGE} >> \${GITHUB_OUTPUT}\e[0m"
           echo "image=${TARGET_IMAGE}" >> "${GITHUB_OUTPUT}"
-          echo -e "\e[35m\e[1mtag=${IMAGE_TAG} >> \${GITHUB_OUTPUT}\e[0m"
+          echo -e "\e[35;1mtag=${IMAGE_TAG} >> \${GITHUB_OUTPUT}\e[0m"
           echo "tag=${IMAGE_TAG}" >> "${GITHUB_OUTPUT}"
       - name: Generate artifacts
         id: artifacts
@@ -180,7 +181,7 @@ jobs:
           MATRIX_OUTPUT_KEY=image
           echo "${{ steps.script.outputs.image }}" > "${MATRIX_OUTPUT_KEY}"
           sha256sum "${MATRIX_OUTPUT_KEY}" | cut -d " " -f 1 | xargs -I{} echo "artifact={}" >> $GITHUB_OUTPUT
-          echo -e "\e[35m\e[1mkey=${MATRIX_OUTPUT_KEY} >> \$GITHUB_OUTPUT\e[0m"
+          echo -e "\e[35;1mkey=${MATRIX_OUTPUT_KEY} >> \${GITHUB_OUTPUT}\e[0m"
           echo "key=${MATRIX_OUTPUT_KEY}" >> $GITHUB_OUTPUT
       - name: Write image name
         if: github.event_name != 'pull_request'
@@ -208,7 +209,7 @@ jobs:
         id: read
         run: |
           result="$(find . -name ${{ needs.docker_generation_tue_env.outputs.key }} -exec cat {} \; | jq --slurp --raw-input 'split("\n")[:-1] | .[] | split(":") | {(.[0]) : [(.[1])]}' | jq -cs '[.[] | to_entries] | flatten | reduce .[] as $dot ({}; .[$dot.key] += $dot.value) | { ${{ needs.docker_generation_tue_env.outputs.key }} : . }' )"
-          echo -e "\e[35m\e[1mresult=${result} >> \$GITHUB_OUTPUT\e[0m"
+          echo -e "\e[35;1mresult=${result} >> \${GITHUB_OUTPUT}\e[0m"
           echo "result=${result}" >> $GITHUB_OUTPUT
       - name: Generate new manifest files
         run: |

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.25.5
+1.25.6
diff --git a/ci/azure-pipelines.example.yml b/ci/azure-pipelines.example.yml
@@ -67,13 +67,14 @@ steps:
     targetType: inline
     script: |
       # START SSH ONLY
+      echo -e "\e[35;1mmkdir -p ${HOME}/.ssh\e[0m"
       mkdir -p "${HOME}"/.ssh
       cp $(deployKey.secureFilePath) "${HOME}"/.ssh/ci_ssh_key
       if [[ -n "${SSH_KNOWN_HOSTS}" ]]
       then
         for host in ${SSH_KNOWN_HOSTS}
         do
-          echo -e "\e[35m\e[1mssh-keyscan -t rsa -H \"${host}\" 2>&1 | tee -a ${HOME}/.ssh/known_hosts\e[0m"
+          echo -e "\e[35;1mssh-keyscan -t rsa -H \"${host}\" 2>&1 | tee -a ${HOME}/.ssh/known_hosts\e[0m"
           ssh-keyscan -t rsa -H "${host}" 2>&1 | tee -a "${HOME}"/.ssh/known_hosts
         done
       fi

diff --git a/ci/azure-pipelines.multiple_pkgs.example.yml b/ci/azure-pipelines.multiple_pkgs.example.yml
@@ -110,13 +110,14 @@ stages:
               targetType: inline
               script: |
                 # START SSH ONLY
+                echo -e "\e[35;1mmkdir -p ${HOME}/.ssh\e[0m"
                 mkdir -p "${HOME}"/.ssh
                 cp $(deployKey.secureFilePath) "${HOME}"/.ssh/ci_ssh_key
                 if [[ -n "${SSH_KNOWN_HOSTS}" ]]
                 then
                   for host in ${SSH_KNOWN_HOSTS}
                   do
-                    echo -e "\e[35m\e[1mssh-keyscan -t rsa -H \"${host}\" 2>&1 | tee -a ${HOME}/.ssh/known_hosts\e[0m"
+                    echo -e "\e[35;1mssh-keyscan -t rsa -H \"${host}\" 2>&1 | tee -a ${HOME}/.ssh/known_hosts\e[0m"
                     ssh-keyscan -t rsa -H "${host}" 2>&1 | tee -a "${HOME}"/.ssh/known_hosts
                   done
                 fi

diff --git a/ci/build-docker-image.sh b/ci/build-docker-image.sh
@@ -7,8 +7,15 @@
 # Stop on errors
 set -o errexit
 
+function cleanup {
+  echo -e "\e[35;1mRemoving ./known_hosts\e[0m"
+  rm -f ./known_hosts
+}
+
+trap cleanup EXIT
+
 # Execute script only in a CI environment
-if [[ "$CI" != "true" ]]
+if [[ "${CI}" != "true" ]]
 then
     echo -e "\e[35;1mError!\e[0m Trying to execute a CI script in a non-CI environment. Exiting script."
     exit 1
@@ -227,7 +234,8 @@ then
     echo -e "\e[35;1mCreating a new docker context for multi-arch builds\e[0m"
     docker context create multiarch-environment
     echo -e "\e[35;1mCreating a new buildx builder for multi-arch builds\e[0m"
-    docker buildx create --name multiarchbuilder --driver docker-container --use multiarch-environment
+    docker buildx create --name multiarch-builder --driver docker-container --use multiarch-environment
+    echo -e "\e[35;1mdocker buildx ls\e[0m"
     docker buildx ls
 fi
 

diff --git a/ci/build-package.sh b/ci/build-package.sh
@@ -8,9 +8,9 @@
 set -o errexit
 
 # Execute script only in a CI environment
-if [ "$CI" != "true" ]
+if [[ "${CI}" != "true" ]]
 then
-    echo -e "\e[35m\e[1mError!\e[0m Trying to execute a CI script in a non-CI environment. Exiting script."
+    echo -e "\e[35;1mError!\e[0m Trying to execute a CI script in a non-CI environment. Exiting script."
     exit 1
 fi
 
@@ -21,53 +21,67 @@ do
         -p=* | --package=* )
             PACKAGE="${i#*=}" ;;
 
+        --debug )
+            DEBUG="true" ;;
+
         * )
             # unknown option
             if [[ -n "$i" ]]  # Ignore empty arguments
             then
-                echo -e "\e[35m\e[1mUnknown input argument '$i'. Check CI .yml file\e[0m"
+                echo -e "\e[35;1mUnknown input argument '$i'. Check CI .yml file\e[0m"
                 exit 1
             fi ;;
     esac
     shift
 done
 
-echo -e "\e[35m\e[1mPACKAGE     = ${PACKAGE}\e[0m"
+echo -e "\e[35;1mPACKAGE     = ${PACKAGE}\e[0m"
 ROS_VERSION=$(docker exec tue-env bash -c 'source ~/.bashrc; echo "${ROS_VERSION}"' | tr -d '\r')
-echo -e "\e[35m\e[1mROS_VERSION = ${ROS_VERSION}\e[0m"
+echo -e "\e[35;1mROS_VERSION = ${ROS_VERSION}\e[0m"
+echo -e "\e[35;1mDEBUG       = ${DEBUG}\e[0m"
 
 # If packages is non-zero, this is a multi-package repo. In multi-package repo, check if this package needs CI.
 # If a single-package repo, CI is always needed.
 # shellcheck disable=SC2153
 if [ -n "$PACKAGES" ] && ! echo "$PACKAGES" | grep -sqw "$PACKAGE"
 then
-    echo -e "\e[35m\e[1mNo changes in this package, so no need to run CI\e[0m"
+    echo -e "\e[35;1mNo changes in this package, so no need to run CI\e[0m"
     exit 0
 fi
 
 # Use docker environment variables in all exec commands instead of script variables
 # Compile the package
 if [[ "${ROS_VERSION}" == 1 ]]
 then
-    echo -e "\e[35m\e[1mCompile the package (catkin build -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCATKIN_ENABLE_TESTING=OFF)\e[0m"
-    docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}"/src/"${PACKAGE}" && /usr/bin/python3 "$(command -v catkin)" build --this --no-status -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCATKIN_ENABLE_TESTING=OFF'
+    ADDITIONAL_ARGS_CATKIN=()
+    if [[ ${DEBUG} == "true" ]]
+    then
+        ADDITIONAL_ARGS_CATKIN+=("--verbose")
+    fi
+    echo -e "\e[35;1mCompile the package (catkin build ${ADDITIONAL_ARGS_CATKIN[*]} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCATKIN_ENABLE_TESTING=OFF)\e[0m"
+    docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}"/src/"${PACKAGE}" && /usr/bin/python3 "$(command -v catkin)" build --this --no-status '"${ADDITIONAL_ARGS_CATKIN[*]}"' -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCATKIN_ENABLE_TESTING=OFF'
 else
-    echo -e "\e[35m\e[1mCheck for default mixin repo (colcon mixin list)\e[0m"
-    MIXIN_REPOS=$(docker exec tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon mixin list | grep -v "^- "' | tr -d '\r' | awk -F ": " '{print $1}')
+    ADDITIONAL_ARGS_COLCON=()
+    if [[ ${DEBUG} == "true" ]]
+    then
+        ADDITIONAL_ARGS_COLCON+=("--log-level" "debug")
+    fi
+    echo -e "\e[35;1mCheck for default mixin repo (colcon ${ADDITIONAL_ARGS_COLCON[*]} mixin list)\e[0m"
+    MIXIN_REPOS=$(docker exec tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon '"${ADDITIONAL_ARGS_COLCON[*]}"' mixin list | grep -v "^- "' | tr -d '\r' | awk -F ": " '{print $1}')
     if ! echo -e "${MIXIN_REPOS}" | grep "^default$" -q
     then
-        echo -e "\e[35m\e[1mAdd the default mixin repo (colcon mixin add default https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml)\e[0m"
-        docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon mixin add default https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml'
+        echo -e "\e[35;1mAdd the default mixin repo (colcon ${ADDITIONAL_ARGS_COLCON[*]} mixin add default https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml)\e[0m"
+        docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon '"${ADDITIONAL_ARGS_COLCON[*]}"' mixin add default https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml'
     else
-        echo -e "\e[35m\e[1mDefault mixin repo already exists\e[0m"
+        echo -e "\e[35;1mDefault mixin repo already exists\e[0m"
     fi
 
-    echo -e "\e[35m\e[1mUpdate colcon mixins (colcon mixin update)\e[0m"
-    docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon mixin update'
+    echo -e "\e[35;1mUpdate colcon mixins (colcon ${ADDITIONAL_ARGS_COLCON[*]} mixin update)\e[0m"
+    docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon '"${ADDITIONAL_ARGS_COLCON[*]}"' mixin update'
 
-    echo -e "\e[35m\e[1mDeleting the merged install directory\e[0m"
+    echo -e "\e[35;1mDeleting the merged install directory\e[0m"
     docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && rm -rf install'
 
-    echo -e "\e[35m\e[1mCompile the package (colcon build --mixin rel-with-deb-info build-testing-off)\e[0m"
-    docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon build --packages-up-to "${PACKAGE}" --mixin rel-with-deb-info build-testing-off --event-handlers desktop_notification- status- terminal_title-'
+    echo -e "\e[35;1mCompile the package (colcon ${ADDITIONAL_ARGS_COLCON[*]} build --mixin rel-with-deb-info build-testing-off)\e[0m"
+    docker exec -t tue-env bash -c 'source ~/.bashrc; cd "${TUE_SYSTEM_DIR}" && colcon '"${ADDITIONAL_ARGS_COLCON[*]}"' build --packages-up-to "${PACKAGE}" --mixin rel-with-deb-info build-testing-off --event-handlers desktop_notification- status- terminal_title-'
 fi
diff --git a/ci/commit-range/action.yml b/ci/commit-range/action.yml
@@ -31,6 +31,6 @@ runs:
           NEWEST=${{ github.event.pull_request.head.sha }}
           COMMIT_RANGE="${OLDEST}...${NEWEST}"
         fi
-        echo -e "\e[35m\e[1m COMMIT_RANGE = ${COMMIT_RANGE} \e[0m"
+        echo -e "\e[35;1m COMMIT_RANGE = ${COMMIT_RANGE} \e[0m"
         echo "commit-range=${COMMIT_RANGE}" >> $GITHUB_OUTPUT
       shell: bash
diff --git a/ci/gh-actions.example.yml b/ci/gh-actions.example.yml
@@ -1,6 +1,6 @@
 name: CI
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 jobs:
   tue-ci:

diff --git a/ci/gh-actions.multiple_pkgs.example.yml b/ci/gh-actions.multiple_pkgs.example.yml
@@ -1,6 +1,6 @@
 name: CI
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 jobs:
   matrix: