Skip to content

(CI) Improve SSH stuff #2585

(CI) Improve SSH stuff

(CI) Improve SSH stuff #2585

Workflow file for this run

name: CI
on: [push, pull_request, workflow_dispatch]
env:
REGISTRY: ghcr.io
jobs:
linting_python_black:
name: Black Linting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: psf/black@stable
with:
options: --check --diff --color -l 120
linting_shellcheck:
name: ShellCheck
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Shellcheck
run: |
shopt -s globstar
shellcheck --version
shellcheck **/*.bash **/*.sh
docker_generation_tue_env:
name: Docker Generation tue-env
runs-on: ubuntu-latest
needs: [linting_python_black, linting_shellcheck]
strategy:
fail-fast: false
matrix:
include:
- ros-distro: noetic
ros-version: 1
platform: "amd64"
base-image: "ubuntu:20.04"
- ros-distro: humble
ros-version: 2
platform: "amd64"
base-image: "ubuntu:22.04"
- ros-distro: rolling
ros-version: 2
platform: "amd64"
base-image: "ubuntu:22.04"
flavor: u22
- ros-distro: rolling
ros-version: 2
platform: "amd64"
base-image: "ubuntu:24.04"
flavor: u24
create-venv: true
docker-user-id: 1001
- ros-distro: noetic
ros-version: 1
platform: "amd64"
base-image: "nvidia/cuda:12.1.1-devel-ubuntu20.04"
flavor: cu12.1.1-devel
default-branch-only: true
maximize-build-space: true
outputs:
key: ${{ steps.artifacts.outputs.key }}
tag: ${{ steps.script.outputs.tag }}
steps:
- name: Maximize build space
if: ${{ matrix.maximize-build-space == true }}
uses: easimon/maximize-build-space@master
with:
build-mount-path: /var/lib/docker/
root-reserve-mb: 2048
temp-reserve-mb: 512
- name: Restart docker
run: sudo service docker restart
- uses: actions/checkout@v4
- name: Login to Docker registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Expose GitHub Runtime
uses: crazy-max/ghaction-github-runtime@v3
# Comment out when generating arm64
#- name: Install QEMU
# run: |
# sudo apt update -qq
# sudo apt install -qq -y qemu qemu-user-static
- name: Docker info
run: docker info
- name: Script
id: script
run: |
#! /usr/bin/env bash
GITHUB_REF=${GITHUB_REF#refs/heads/}
GITHUB_REF=${GITHUB_REF#refs/tags/}
BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF}}
# shellcheck disable=SC1083,SC2296
PULLREQUEST=${{ github.event.number }}
PULLREQUEST=${PULLREQUEST:-false}
# shellcheck disable=SC1083,SC2296
PUSH_IMAGE=${{ matrix.push-image }}
[[ ${PULLREQUEST} != "false" ]] && PUSH_IMAGE="false"
PUSH_IMAGE=${PUSH_IMAGE:-true}
# shellcheck disable=SC1083,SC2296
DEFAULT_BRANCH=${{ github.event.repository.default_branch }}
if [[ "$BRANCH" == "$DEFAULT_BRANCH" ]]
then
IMAGE_TAG="latest"
else
IMAGE_TAG="$(echo "${BRANCH}" | tr '[:upper:]' '[:lower:]' | sed -e 's:/:_:g')"
fi
# shellcheck disable=SC1083,SC2296
IMAGE_FLAVOR=${{ matrix.flavor }}
IMAGE_FLAVOR=${IMAGE_FLAVOR:+"-${IMAGE_FLAVOR}"}
# shellcheck disable=SC2296
TARGET_IMAGE="${{ env.REGISTRY }}/tue-robotics/tue-env-ros-${{ matrix.ros-distro }}${IMAGE_FLAVOR}:${IMAGE_TAG}-${{ matrix.platform }}"
SSH_KNOWN_HOSTS="github.com"
SSH_ARGS=()
if [[ -n "${SSH_KEY}" ]]
then
SSH_KEY_PATH="${HOME}"/.ssh/ci_ssh_key
echo -e "\e[35;1mmkdir -p ${HOME}/.ssh\e[0m"
mkdir -p "${HOME}"/.ssh
echo "${SSH_KEY}" > "${SSH_KEY_PATH}"
chmod 600 "${SSH_KEY_PATH}"
eval "$(ssh-agent -s)"
SSH_ARGS+=("--ssh" "--ssh-key=${SSH_KEY_PATH}")
fi
if [[ -n "${SSH_KNOWN_HOSTS}" ]]
then
echo -e "\e[35;1mmkdir -p ${HOME}/.ssh\e[0m"
mkdir -p "${HOME}"/.ssh
for host in ${SSH_KNOWN_HOSTS}
do
echo -e "\e[35;1mssh-keyscan -t rsa -H \"${host}\" 2>&1 | tee -a ${HOME}/.ssh/known_hosts\e[0m"
ssh-keyscan -t rsa -H "${host}" 2>&1 | tee -a "${HOME}"/.ssh/known_hosts
done
fi
# shellcheck disable=SC1083,SC2296
CREATE_VENV=${{ matrix.create-venv }}
CREATE_VENV=${CREATE_VENV:-false}
# shellcheck disable=SC1083,SC2296
VENV_INCLUDE_SYSTEM_SITE=${{ matrix.venv-include-system-site }}
VENV_INCLUDE_SYSTEM_SITE=${VENV_INCLUDE_SYSTEM_SITE:-true}
# shellcheck disable=SC1083,SC2296
DOCKER_USER=${{ matrix.docker-user }}
DOCKER_USER=${DOCKER_USER:-docker}
# shellcheck disable=SC1083,SC2296
DOCKER_USER_ID=${{ matrix.docker-user-id }}
DOCKER_USER_ID=${DOCKER_USER_ID:-1000}
# shellcheck disable=SC2296
ci/build-docker-image.sh \
--registry="${{ env.REGISTRY }}" \
--image="${TARGET_IMAGE}" \
--branch="${BRANCH}" \
--commit="${GITHUB_SHA}" \
--push_image="${PUSH_IMAGE}" \
--pull_request="${PULLREQUEST}" \
--ros_version="${{ matrix.ros-version }}" \
--ros_distro="${{ matrix.ros-distro }}" \
--base_image="${{ matrix.base-image }}" \
--docker_file="dockerfiles/tue-env.Dockerfile" \
--oauth2_token="${{ secrets.OAUTH2_TOKEN }}" \
--platforms="${{ matrix.platform }}" \
--create-virtualenv="${CREATE_VENV}" \
--virtualenv-include-system-site-packages="${VENV_INCLUDE_SYSTEM_SITE}" \
--docker_user="${DOCKER_USER}" \
--docker_user_id="${DOCKER_USER_ID}" \
"${SSH_ARGS[@]}"
echo -e "\e[35;1mimage=${TARGET_IMAGE} >> \${GITHUB_OUTPUT}\e[0m"
echo "image=${TARGET_IMAGE}" >> "${GITHUB_OUTPUT}"
echo -e "\e[35;1mtag=${IMAGE_TAG} >> \${GITHUB_OUTPUT}\e[0m"
echo "tag=${IMAGE_TAG}" >> "${GITHUB_OUTPUT}"
- name: Generate artifacts
id: artifacts
if: github.event_name != 'pull_request'
run: |
MATRIX_OUTPUT_KEY=image
echo "${{ steps.script.outputs.image }}" > "${MATRIX_OUTPUT_KEY}"
sha256sum "${MATRIX_OUTPUT_KEY}" | cut -d " " -f 1 | xargs -I{} echo "artifact={}" >> $GITHUB_OUTPUT
echo -e "\e[35;1mkey=${MATRIX_OUTPUT_KEY} >> \${GITHUB_OUTPUT}\e[0m"
echo "key=${MATRIX_OUTPUT_KEY}" >> $GITHUB_OUTPUT
- name: Write image name
if: github.event_name != 'pull_request'
uses: actions/upload-artifact@v4
with:
name: ${{ steps.artifacts.outputs.artifact }}
path: ${{ steps.artifacts.outputs.key }}
retention-days: 1
docker_manifest_generation_tue_env:
name: Docker Manifest Generation tue-env
runs-on: ubuntu-latest
needs: [docker_generation_tue_env]
if: github.event_name != 'pull_request'
steps:
- uses: actions/checkout@v4
- name: Login to Docker registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/download-artifact@v4
- name: Read matrix output
id: read
run: |
result="$(find . -name ${{ needs.docker_generation_tue_env.outputs.key }} -exec cat {} \; | jq --slurp --raw-input 'split("\n")[:-1] | .[] | split(":") | {(.[0]) : [(.[1])]}' | jq -cs '[.[] | to_entries] | flatten | reduce .[] as $dot ({}; .[$dot.key] += $dot.value) | { ${{ needs.docker_generation_tue_env.outputs.key }} : . }' )"
echo -e "\e[35;1mresult=${result} >> \${GITHUB_OUTPUT}\e[0m"
echo "result=${result}" >> $GITHUB_OUTPUT
- name: Generate new manifest files
run: |
ci/docker-manifest-generator.sh \
--images-json='${{ toJSON(steps.read.outputs.result) }}' \
--tag="${{ needs.docker_generation_tue_env.outputs.tag }}"