From cfb7e205b440f3e0959843ce1383594cfc9e12b7 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Sat, 27 Nov 2021 22:54:29 +0100 Subject: [PATCH 01/32] add cloud init ability for ubunut. Fix issues when setting hostname/user config while cloud-init still runs --- README.md | 5 +++++ rpi/create_image.sh | 30 +++++++++++++++++++++--------- rpi/user-data | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 9 deletions(-) mode change 100644 => 100755 rpi/create_image.sh create mode 100644 rpi/user-data diff --git a/README.md b/README.md index fcb55f1..f2bd9f0 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,11 @@ The scripts have been tested with the following operating systems: - Ubuntu 20.03 - Raspbian Buster +## Initial Config + +Update the content of riasc.yml and user-data on the boot partion of the sd card + + ## Usage See: https://riasc.eu/docs/setup/agent/manual diff --git a/rpi/create_image.sh b/rpi/create_image.sh old mode 100644 new mode 100755 index b07826f..a09d5f9 --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -139,25 +139,36 @@ glob chmod 755 /usr/local/bin/riasc-*.sh copy-in keys/ /boot/ -echo "Enable SSH on boot..." -touch /boot/ssh - -echo "Setting hostname..." -write /etc/hostname "${NODENAME}" +echo "Disable daily APT timers" +rm /etc/systemd/system/timers.target.wants/apt-daily-upgrade.timer +rm /etc/systemd/system/timers.target.wants/apt-daily.timer echo "Updating os-release" write-append /etc/os-release "VARIANT=\"RIasC\"\n" write-append /etc/os-release "BUILD_ID=\"$(date)\"\n" write-append /etc/os-release "DOCUMENTATION_URL=\"https://riasc.eu\"\n" +EOF + +case ${OS} in + ubuntu) +cat <> patch.fish +copy-in user-data /boot +EOF + ;; + *) +cat <> patch.fish +echo "Enable SSH on boot..." +touch /boot/ssh + +echo "Setting hostname..." +write /etc/hostname "${NODENAME}" echo "Enable systemd risac services..." ln-sf /etc/systemd/system/risac-update.service /etc/systemd/system/multi-user.target.wants/riasc-update.service ln-sf /etc/systemd/system/risac-set-hostname.service /etc/systemd/system/multi-user.target.wants/riasc-set-hostname.service - -echo "Disable daily APT timers" -rm /etc/systemd/system/timers.target.wants/apt-daily-upgrade.timer -rm /etc/systemd/system/timers.target.wants/apt-daily.timer EOF + ;; +esac if [ "${FLAVOR}" = "edgeflex" -a "${OS}" = "ubuntu" ]; then cat <> patch.fish @@ -169,6 +180,7 @@ fi echo "Patching image with guestfish..." guestfish < patch.fish + # Zip image echo "Zipping image..." rm -f ${RIASC_IMAGE_FILE}.zip diff --git a/rpi/user-data b/rpi/user-data new file mode 100644 index 0000000..f61a044 --- /dev/null +++ b/rpi/user-data @@ -0,0 +1,35 @@ +#cloud-config + +# This is the user-data configuration file for cloud-init. By default this sets +# up an initial user called "ubuntu" with password "ubuntu", which must be +# changed at first login. However, many additional actions can be initiated on +# first boot from this file. The cloud-init documentation has more details: +# +# https://cloudinit.readthedocs.io/ +# +# Some additional examples are provided in comments below the default +# configuration. + +# On first boot, set the (default) ubuntu user's password to "ubuntu" and +# expire user passwords +chpasswd: + expire: true + list: + - ubuntu:ubuntu + +# Enable password authentication with the SSH daemon +ssh_pwauth: true + +locale: C.UTF-8 +timezone: Europe/Berlin +hostname: exampleHost + +## Run arbitrary commands at rc.local like time +runcmd: + - [ ln, -sf, /etc/systemd/system/risac-update.service, /etc/systemd/system/multi-user.target.wants/riasc-update.service] + +power_state: + mode: reboot + message: Reboot after inital setup + timeout: 20 + condition: True From 0e72923566accd361b2b33bc642e9a1370155864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Thu, 23 Dec 2021 00:32:33 +0100 Subject: [PATCH 02/32] Changed create_image.sh --- rpi/create_image.sh | 58 ++++++++++++++------------------------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/rpi/create_image.sh b/rpi/create_image.sh index a09d5f9..522d7e8 100755 --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -6,34 +6,15 @@ SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) pushd ${SCRIPT_PATH} # Settings -NODENAME="${NODENAME:-riasc-agent}" +NODENAME="${NODENAME}" TOKEN="${TOKEN:-XXXXX}" -FLAVOR=${FLAVOR:-erigrid} +FLAVOR=${FLAVOR:edgeflex} +OS="ubuntu" -case ${FLAVOR} in - edgeflex) - OS="ubuntu" - ;; - - erigrid) - OS="raspios" - ;; -esac - -case ${OS} in - ubuntu) - IMAGE_FILE="ubuntu-20.04.2-preinstalled-server-arm64+raspi" - IMAGE_SUFFIX="img.xz" - IMAGE_URL="https://cdimage.ubuntu.com/releases/20.04.2/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" - ;; - - raspios) - IMAGE_FILE="2021-05-07-raspios-buster-armhf-lite" - IMAGE_SUFFIX="zip" - IMAGE_URL="https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-05-28/${IMAGE_FILE}.${IMAGE_SUFFIX}" - ;; -esac +IMAGE_FILE="ubuntu-20.04.2-preinstalled-server-arm64+raspi" +IMAGE_SUFFIX="img.xz" +IMAGE_URL="https://cdimage.ubuntu.com/releases/20.04.2/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" RIASC_IMAGE_FILE="$(date +%Y-%m-%d)-riasc-${OS}" @@ -44,10 +25,6 @@ function check_command() { fi } -# Show config -echo "Using hostname: ${NODENAME}" -echo "Using token: ${TOKEN}" - # Check that required commands exist echo "Check if required commands are installed..." check_command guestfish @@ -80,23 +57,24 @@ fi echo "Copying image..." cp ${IMAGE_FILE}.img ${RIASC_IMAGE_FILE}.img -# Prepare config -case ${FLAVOR} in - erigrid) - CONFIG_FILE="riasc.yaml" - ;; - *) - CONFIG_FILE="riasc.${FLAVOR}.yaml" - ;; -esac +CONFIG_FILE="riasc.${FLAVOR}.yaml" cp ../common/${CONFIG_FILE} riasc.yaml +# Check config +if [[-z ${NODENAME}]] + echo "No Nodename provided" + echo "Not patching Image" + echo "Please provide nodename with: 'export NODENAME=XXXX'" + exit 0 +fi + # Patch config sed -i \ - -e "s/XXXXX/${TOKEN}/g" \ - -e "s/riasc-agent/${NODENAME}/g" \ + -e "s/edgePMU/${NODENAME}/g" \ riasc.yaml + + # Prepare systemd-timesyncd config cat > fallback-ntp.conf < Date: Fri, 7 Jan 2022 04:30:46 +0100 Subject: [PATCH 03/32] flash iamge script --- common/riasc-update.sh | 4 + rpi/.gitignore | 1 + rpi/create_image.sh | 60 +++++--- rpi/flash_image.sh | 204 ++++++++++++++++++++++++++++ rpi/rootfs/etc/systemd/system/flash | 0 5 files changed, 250 insertions(+), 19 deletions(-) create mode 100755 rpi/flash_image.sh create mode 100644 rpi/rootfs/etc/systemd/system/flash diff --git a/common/riasc-update.sh b/common/riasc-update.sh index d2dc60b..eeb1d4c 100755 --- a/common/riasc-update.sh +++ b/common/riasc-update.sh @@ -155,6 +155,10 @@ if [ $(config '.ansible.verify_commit') == "true" ]; then ANSIBLE_OPTS+="--verify-commit" fi +if [ -z $(config '.ansible.branch')]; then + ANSIBLE_OPTS+="--checkout $(config '.ansible.playbook')" +fi + # Run Ansible playbook log "Running Ansible playbook..." ANSIBLE_FORCE_COLOR=1 \ diff --git a/rpi/.gitignore b/rpi/.gitignore index 40b333e..c0fba10 100644 --- a/rpi/.gitignore +++ b/rpi/.gitignore @@ -5,3 +5,4 @@ patch.fish riasc.yaml fallback-ntp.conf keys/ +edgepmu*/ \ No newline at end of file diff --git a/rpi/create_image.sh b/rpi/create_image.sh index 522d7e8..b3a16e6 100755 --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -6,15 +6,34 @@ SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) pushd ${SCRIPT_PATH} # Settings -NODENAME="${NODENAME}" +NODENAME="${NODENAME:-riasc-agent}" TOKEN="${TOKEN:-XXXXX}" -FLAVOR=${FLAVOR:edgeflex} -OS="ubuntu" +FLAVOR=${FLAVOR:-erigrid} -IMAGE_FILE="ubuntu-20.04.2-preinstalled-server-arm64+raspi" -IMAGE_SUFFIX="img.xz" -IMAGE_URL="https://cdimage.ubuntu.com/releases/20.04.2/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" +case ${FLAVOR} in + edgeflex) + OS="ubuntu" + ;; + + erigrid) + OS="raspios" + ;; +esac + +case ${OS} in + ubuntu) + IMAGE_FILE="ubuntu-20.04.2-preinstalled-server-arm64+raspi" + IMAGE_SUFFIX="img.xz" + IMAGE_URL="https://cdimage.ubuntu.com/releases/20.04.2/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" + ;; + + raspios) + IMAGE_FILE="2021-05-07-raspios-buster-armhf-lite" + IMAGE_SUFFIX="zip" + IMAGE_URL="https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-05-28/${IMAGE_FILE}.${IMAGE_SUFFIX}" + ;; +esac RIASC_IMAGE_FILE="$(date +%Y-%m-%d)-riasc-${OS}" @@ -25,6 +44,10 @@ function check_command() { fi } +# Show config +echo "Using hostname: ${NODENAME}" +echo "Using token: ${TOKEN}" + # Check that required commands exist echo "Check if required commands are installed..." check_command guestfish @@ -57,24 +80,23 @@ fi echo "Copying image..." cp ${IMAGE_FILE}.img ${RIASC_IMAGE_FILE}.img -CONFIG_FILE="riasc.${FLAVOR}.yaml" +# Prepare config +case ${FLAVOR} in + erigrid) + CONFIG_FILE="riasc.yaml" + ;; + *) + CONFIG_FILE="riasc.${FLAVOR}.yaml" + ;; +esac cp ../common/${CONFIG_FILE} riasc.yaml -# Check config -if [[-z ${NODENAME}]] - echo "No Nodename provided" - echo "Not patching Image" - echo "Please provide nodename with: 'export NODENAME=XXXX'" - exit 0 -fi - # Patch config sed -i \ - -e "s/edgePMU/${NODENAME}/g" \ + -e "s/XXXXX/${TOKEN}/g" \ + -e "s/riasc-agent/${NODENAME}/g" \ riasc.yaml - - # Prepare systemd-timesyncd config cat > fallback-ntp.conf < "vaultkey.secret" + +#GPG key pair +#TODO + +echo "Done" + +echo "Writing configuration" + +sed -i \ + -e "s/edgepmu/${NODENAME}/g" \ + riasc.yaml + +#Select branch +if [[ -n ${GIT_BRANCH} ]]; then + sed -i \ + -e "/url: /abranch: ${GIT_BRANCH}" riasc.yaml +else + sed -i "/url: /abranch: development" riasc.yaml #default develop +fi + +#Git token +if [[ -n ${GIT_TOKEN} ]]; then + sed -i "/url: /atoken: ${GIT_TOKEN}" riasc.yaml +fi + + +sed -i \ + -e "s/exampleHost/${NODENAME}/g" \ + user-data + +echo "Done" + +#Unzip the image file +#echo "Unzipping the image" +#unzip -p ${NODE_IMAGE_FILE}.zip > ${NODE_IMAGE_FILE}.img +#echo "Done" + +echo "Writing Patch file" +cat < edgeflex.fish +echo "Loading image..." +add ${NODE_IMAGE_FILE}.img + +echo "Start virtual environment..." +run + +echo "Available filesystems:" +list-filesystems + +echo "Mounting filesystems..." +mount /dev/sda2 / +mount /dev/sda1 /boot + +echo "Available space:" +df-h + +echo "Copy files into image..." +copy-in riasc.yaml /boot +copy-in user-data /boot +copy-in vaultkey.secret /boot +EOF + +#Write patch to image +echo "Patching image with guestfish..." +guestfish < edgeflex.fish + +# Zip image +#echo "Zipping image..." +#rm -f ${NODE_IMAGE_FILE}.zip +#zip ${NODE_IMAGE_FILE}.zip ${NODE_IMAGE_FILE}.img +#rm -f ${NODE_IMAGE_FILE}.img +#echo "Done" + +echo "Please write the new image to an SD card:" +lsblk |grep sd +echo " dd bs=1M if=${NODE_IMAGE_FILE}.img of=/dev/sdX" diff --git a/rpi/rootfs/etc/systemd/system/flash b/rpi/rootfs/etc/systemd/system/flash new file mode 100644 index 0000000..e69de29 From 0f3ae466ef4c0367b649f946eb799c6cae8586a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Sun, 9 Jan 2022 18:23:16 +0100 Subject: [PATCH 04/32] Flash image script and support for branches --- common/riasc-update.sh | 4 ++-- rpi/flash_image.sh | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/common/riasc-update.sh b/common/riasc-update.sh index eeb1d4c..3d45a24 100755 --- a/common/riasc-update.sh +++ b/common/riasc-update.sh @@ -155,8 +155,8 @@ if [ $(config '.ansible.verify_commit') == "true" ]; then ANSIBLE_OPTS+="--verify-commit" fi -if [ -z $(config '.ansible.branch')]; then - ANSIBLE_OPTS+="--checkout $(config '.ansible.playbook')" +if ! [ -z $(config '.ansible.branch') ]; then + ANSIBLE_OPTS+="--checkout $(config '.ansible.branch')" fi # Run Ansible playbook diff --git a/rpi/flash_image.sh b/rpi/flash_image.sh index 49df44e..b765a7a 100755 --- a/rpi/flash_image.sh +++ b/rpi/flash_image.sh @@ -117,7 +117,7 @@ mkdir ${NODENAME} echo "Copying files" NODE_IMAGE_FILE="${NODENAME}_IMAGE" cp ${IMAGE_FILE} "${NODENAME}/${NODE_IMAGE_FILE}.img" -cp ${SSL_CERT_FILE} "${NODENAME}/openssl.conf" +cp ${SSL_CERT_FILE} "${NODENAME}/acs-lab.conf" cp ${CONFIG_PATH} "${NODENAME}/riasc.yaml" cp "user-data" "${NODENAME}/user-data" echo "Done" @@ -142,14 +142,14 @@ sed -i \ #Select branch if [[ -n ${GIT_BRANCH} ]]; then sed -i \ - -e "/url: /abranch: ${GIT_BRANCH}" riasc.yaml + -e "/url: /a branch: ${GIT_BRANCH}" riasc.yaml else - sed -i "/url: /abranch: development" riasc.yaml #default develop + sed -i "/url: /a branch: development" riasc.yaml #default develop fi #Git token if [[ -n ${GIT_TOKEN} ]]; then - sed -i "/url: /atoken: ${GIT_TOKEN}" riasc.yaml + sed -i "/url: /a token: ${GIT_TOKEN}" riasc.yaml fi @@ -186,6 +186,9 @@ echo "Copy files into image..." copy-in riasc.yaml /boot copy-in user-data /boot copy-in vaultkey.secret /boot + +mkdir /boot/openvpn/ +copy-in acs-lab.conf /boot/openvpn EOF #Write patch to image From 0b5daaab9a91b1e5fdc11e4e5a4c8152b8258681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Sun, 9 Jan 2022 21:21:47 +0100 Subject: [PATCH 05/32] Fix branch and access token options --- rpi/flash_image.sh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/rpi/flash_image.sh b/rpi/flash_image.sh index b765a7a..96dafd1 100755 --- a/rpi/flash_image.sh +++ b/rpi/flash_image.sh @@ -12,22 +12,25 @@ ASK_CONFIRM=true #Get User input usage(){ echo "Usage:" - echo " -I [Path to Image: -I /path/to/image/]" - echo " -N [Hostname to use: -N name]" - echo " -T [Gitlab Token: -T GITLAB/HUB Token]" #Currently not used - echo " -S [Path to SSL Cert: -S /path/to/cert]" + echo " -I [Path to Image: -I /path/to/image/]" + echo " -N [Hostname to use: -N name]" + echo " -T [Git !project! acces token: -T GITLAB token]" + echo " -B [Git branch -B development]" + echo " -S [Path to SSL Cert: -S /path/to/cert]" echo " -y [Dont ask for confirmations]" exit } -while getopts ":I:N:T:S::y" opt +while getopts ":I:N:T:B:S::y" opt do case "${opt}" in I) IMAGE_FILE=${OPTARG};; S) SSL_CERT_FILE=${OPTARG} ;; N) NODENAME=${OPTARG} ;; y) ASK_CONFIRM=false ;; + T) GIT_TOKEN=${OPTARG} ;; + B) GIT_BRANCH=${OPTARG} ;; *) echo "Unknown argument ${OPTARG}" usage ;; :) usage ;; @@ -131,6 +134,8 @@ openssl rand -hex 128 > "vaultkey.secret" #GPG key pair #TODO +echo ${GIT_TOKEN} > "git_token.secret" + echo "Done" echo "Writing configuration" @@ -142,17 +147,14 @@ sed -i \ #Select branch if [[ -n ${GIT_BRANCH} ]]; then sed -i \ - -e "/url: /a branch: ${GIT_BRANCH}" riasc.yaml -else - sed -i "/url: /a branch: development" riasc.yaml #default develop + -e "/url: /a\\\\tbranch: ${GIT_BRANCH}" riasc.yaml fi #Git token if [[ -n ${GIT_TOKEN} ]]; then - sed -i "/url: /a token: ${GIT_TOKEN}" riasc.yaml + sed -i -e "s/git.rwth-aachen/pmu-acs:${GIT_TOKEN}@git.rwth-aachen/g" riasc.yaml fi - sed -i \ -e "s/exampleHost/${NODENAME}/g" \ user-data @@ -186,6 +188,7 @@ echo "Copy files into image..." copy-in riasc.yaml /boot copy-in user-data /boot copy-in vaultkey.secret /boot +copy-in git_token.secret /boot mkdir /boot/openvpn/ copy-in acs-lab.conf /boot/openvpn From 6d8727f14e9739e3cffba029193ea7fcfec3fb6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Sun, 9 Jan 2022 22:33:37 +0100 Subject: [PATCH 06/32] Changed vault pass generation --- rpi/flash_image.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rpi/flash_image.sh b/rpi/flash_image.sh index 96dafd1..899c573 100755 --- a/rpi/flash_image.sh +++ b/rpi/flash_image.sh @@ -129,7 +129,12 @@ pushd ${NODENAME} echo "Generating secrets" #Ansible vault key (Currently not used but its there) -openssl rand -hex 128 > "vaultkey.secret" +VAULT_KEY=$(openssl rand -hex 128) +cat < vaultkey.secret +#!/bin/bash +echo "${VAULT_KEY}" +EOF + #GPG key pair #TODO From 33b2f6e0bfd5f78bbd6f4a0f13dfcbe19061bbcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Sun, 9 Jan 2022 22:44:22 +0100 Subject: [PATCH 07/32] Add automatic vault decryption --- common/riasc-update.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/riasc-update.sh b/common/riasc-update.sh index 3d45a24..53f8e05 100755 --- a/common/riasc-update.sh +++ b/common/riasc-update.sh @@ -150,13 +150,14 @@ ANSIBLE_EXTRA_VARS="$(config --tojson --indent 0 .ansible.variables)" ANSIBLE_OPTS=" --url $(config .ansible.url)" ANSIBLE_OPTS+=" --inventory $(config .ansible.inventory)" ANSIBLE_OPTS+=" $(config '.ansible.extra_args // [ ] | join(" ")')" +ANSIBLE_OPTS+=" --vault-password-file /boot/firmware/vaultkey.secret" if [ $(config '.ansible.verify_commit') == "true" ]; then ANSIBLE_OPTS+="--verify-commit" fi if ! [ -z $(config '.ansible.branch') ]; then - ANSIBLE_OPTS+="--checkout $(config '.ansible.branch')" + ANSIBLE_OPTS+=" --checkout $(config '.ansible.branch')" fi # Run Ansible playbook From 19a4b1622a1c2691b6abbbc0b051dcb3541b5dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Wed, 12 Jan 2022 14:38:02 +0100 Subject: [PATCH 08/32] Rename flash_image to update image --- rpi/{flash_image.sh => update_image.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename rpi/{flash_image.sh => update_image.sh} (100%) diff --git a/rpi/flash_image.sh b/rpi/update_image.sh similarity index 100% rename from rpi/flash_image.sh rename to rpi/update_image.sh From d5e661f1e5a7f54b8e5c9e80277b8d9785f9b5ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Wed, 12 Jan 2022 20:19:18 +0100 Subject: [PATCH 09/32] Add git options --- rpi/update_image.sh | 115 ++++++++++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 42 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 899c573..d8e4360 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -4,12 +4,16 @@ set -e SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) pushd ${SCRIPT_PATH} -#Predefined settings +#======================= Predefined settings ===================== CONFIG_FILE="riasc.edgeflex.yaml" SSL_SEARCH_PATH="./ssl" ASK_CONFIRM=true -#Get User input +GIT_SERVER="git@git.rwth-aachen.de" +GIT_ANSIBLE_REPO=${GIT_SERVER}":acs/public/software/pmu/pmu-ansible.git" +GIT_PASS=${GIT_SERVER}":acs/public/software/pmu/pmu_pass.git" + +#========================= Get User input ========================= usage(){ echo "Usage:" echo " -I [Path to Image: -I /path/to/image/]" @@ -18,19 +22,25 @@ usage(){ echo " -B [Git branch -B development]" echo " -S [Path to SSL Cert: -S /path/to/cert]" echo " -y [Dont ask for confirmations]" + echo "" + echo "Credentials for ansible/pass repo" + echo " -U [${GIT_SERVER} username -B myName]" + echo " -P [${GIT_SERVER} pass/token -P Token]" exit } -while getopts ":I:N:T:B:S::y" opt +while getopts ":I:N:T:B:S:yU:P:" opt do case "${opt}" in I) IMAGE_FILE=${OPTARG};; S) SSL_CERT_FILE=${OPTARG} ;; N) NODENAME=${OPTARG} ;; y) ASK_CONFIRM=false ;; - T) GIT_TOKEN=${OPTARG} ;; - B) GIT_BRANCH=${OPTARG} ;; + T) PMU_GIT_TOKEN=${OPTARG} ;; + B) PMU_GIT_BRANCH=${OPTARG} ;; + U) GIT_USERNAME=${OPTARG} + P) GIT_PASS=${OPTARG} *) echo "Unknown argument ${OPTARG}" usage ;; :) usage ;; @@ -48,30 +58,18 @@ if ! [[ -r ${IMAGE_FILE} ]]; then #TODO: Check if this is a .zip file usage fi -#Ensure nodename is supplied +#Ensure Nodename is supplied if ! [[ -n ${NODENAME} ]]; then echo "No node name supplied" usage fi -#if ! [[ -n ${GIT_TOKEN} ]]; then -# echo "No git acces token supplied" -# usage -#fi - - -#Try to find ssl cert file automatically if not supplied -if ! [[ -n ${SSL_CERT_FILE} ]]; then - if [[ -r "${SSL_SEARCH_PATH}/gate-TCP4-1184-${NODENAME}.pmu.acs-lab.eonerc.rwth-aachen.de-config.ovpn" ]]; then - SSL_CERT_FILE="${SSL_SEARCH_PATH}/gate-TCP4-1184-${NODENAME}.pmu.acs-lab.eonerc.rwth-aachen.de-config.ovpn" - echo "Automatically found SSL cert file: ${SSL_CERT_FILE}" - else - echo "No SSL cert file supplied. Could not find one either." - usage - fi +#Ensure git project token is supplied +if ! [[ -n ${PMU_GIT_TOKEN} ]]; then + echo "No git acces token supplied" + usage fi - #Ensure SSL cert file is found if ! [[ -r ${SSL_CERT_FILE} ]]; then echo "SSL cert file '${SSL_CERT_FILE}' does not exist" @@ -87,16 +85,21 @@ else exit fi -#Confirm settings +exit +#========================= Check if we can access Git repos ========================= +#Dont do for now +#Check if repos (pass + ansible) are available + +#================================= Confirm settings ================================= echo "Gathered following configuration:" echo "Nodename: ${NODENAME}" echo "Image: ${IMAGE_FILE}" echo "SSL_CERT: ${SSL_CERT_FILE}" echo "Config: ${CONFIG_PATH}" -if [[ -n ${GIT_BRANCH} ]]; then -echo "Branch: ${GIT_BRANCH}" +if [[ -n ${PMU_GIT_BRANCH} ]]; then +echo "Branch: ${PMU_GIT_BRANCH}" fi if [[ ${ASK_CONFIRM} == true ]]; then @@ -108,7 +111,9 @@ if [[ ${ASK_CONFIRM} == true ]]; then fi -#Start patching: +#============================== Setup to create patch ============================== + +#1. Create temp directory echo "Creating temporary work directory" if [[ -d ${NODENAME} ]]; then echo "Allready exists. Deleting" @@ -116,7 +121,7 @@ if [[ -d ${NODENAME} ]]; then fi mkdir ${NODENAME} -#copy what we need +#2. Copy files to work directory echo "Copying files" NODE_IMAGE_FILE="${NODENAME}_IMAGE" cp ${IMAGE_FILE} "${NODENAME}/${NODE_IMAGE_FILE}.img" @@ -125,24 +130,34 @@ cp ${CONFIG_PATH} "${NODENAME}/riasc.yaml" cp "user-data" "${NODENAME}/user-data" echo "Done" +#3. Make sure repos are here + + +#4. Enter working directory pushd ${NODENAME} +#5. Generate secrets and write to files echo "Generating secrets" -#Ansible vault key (Currently not used but its there) + +#Vault Key VAULT_KEY=$(openssl rand -hex 128) cat < vaultkey.secret #!/bin/bash echo "${VAULT_KEY}" EOF +#Git token +echo ${PMU_GIT_TOKEN} > "git_token.secret" #TODO: braucht man das?? -#GPG key pair -#TODO - -echo ${GIT_TOKEN} > "git_token.secret" +#SNMP key +SNMP_KEY=$(openssl rand -hex 10) +echo ${SNMP_KEY} > "snmp.secret" #TODO: braucht man das?? echo "Done" +#============================ Edit Files for Boot partition ============================ + +#1. Edit configuration files echo "Writing configuration" sed -i \ @@ -150,28 +165,42 @@ sed -i \ riasc.yaml #Select branch -if [[ -n ${GIT_BRANCH} ]]; then +if [[ -n ${PMU_GIT_BRANCH} ]]; then sed -i \ - -e "/url: /a\\\\tbranch: ${GIT_BRANCH}" riasc.yaml + -e "/url: /a\\\\tbranch: ${PMU_GIT_BRANCH}" riasc.yaml fi #Git token -if [[ -n ${GIT_TOKEN} ]]; then - sed -i -e "s/git.rwth-aachen/pmu-acs:${GIT_TOKEN}@git.rwth-aachen/g" riasc.yaml +if [[ -n ${PMU_GIT_TOKEN} ]]; then + sed -i -e "s/git.rwth-aachen/pmu-acs:${PMU_GIT_TOKEN}@git.rwth-aachen/g" riasc.yaml fi +#Node Name sed -i \ -e "s/exampleHost/${NODENAME}/g" \ user-data echo "Done" -#Unzip the image file -#echo "Unzipping the image" -#unzip -p ${NODE_IMAGE_FILE}.zip > ${NODE_IMAGE_FILE}.img -#echo "Done" +#============================== Edit Files in git repo ============================= + +#1. Encrypt with ansible +#OpenVPN. +#SNMP Key + +#2. Write variables to ansible-repo + +#3. Commit and push ansible-repo +#4. Encrypt password with PGP key + +#5. Push to pass repo + +#================================== Write to Image ================================== + +#1. Write patch file echo "Writing Patch file" + cat < edgeflex.fish echo "Loading image..." add ${NODE_IMAGE_FILE}.img @@ -199,17 +228,19 @@ mkdir /boot/openvpn/ copy-in acs-lab.conf /boot/openvpn EOF -#Write patch to image +#2. Write patch to image echo "Patching image with guestfish..." guestfish < edgeflex.fish -# Zip image +#3. Zip image + #echo "Zipping image..." #rm -f ${NODE_IMAGE_FILE}.zip #zip ${NODE_IMAGE_FILE}.zip ${NODE_IMAGE_FILE}.img #rm -f ${NODE_IMAGE_FILE}.img #echo "Done" +#4. Final outputs echo "Please write the new image to an SD card:" lsblk |grep sd echo " dd bs=1M if=${NODE_IMAGE_FILE}.img of=/dev/sdX" From 5d316158c9b2f2b1e15c9f9eaa65e4da2f5a5995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Mon, 17 Jan 2022 16:34:00 +0100 Subject: [PATCH 10/32] Added git Automatic permission check Automatic pull Automatic project token gen --- rpi/update_image.sh | 103 +++++++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index d8e4360..b5857f4 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -1,56 +1,68 @@ #!/bin/bash set -e +set -x SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) +SCRIPT_OWNER=$(stat -c '%U' ${SCRIPT_PATH}) pushd ${SCRIPT_PATH} + #======================= Predefined settings ===================== CONFIG_FILE="riasc.edgeflex.yaml" SSL_SEARCH_PATH="./ssl" ASK_CONFIRM=true +DEBUG=false + +GIT_SERVER="git.rwth-aachen.de" +GIT_USE_KEY=false +GIT_MIN_ACCESS_LEVEL=40 + +GIT_ANSIBLE_REPO_NAME="pmu-ansible" +GIT_ANSIBLE_REPO_ID=67607 +GIT_ANSIBLE_REPO="${GIT_SERVER}/acs/public/software/pmu/${GIT_ANSIBLE_REPO_NAME}.git" -GIT_SERVER="git@git.rwth-aachen.de" -GIT_ANSIBLE_REPO=${GIT_SERVER}":acs/public/software/pmu/pmu-ansible.git" -GIT_PASS=${GIT_SERVER}":acs/public/software/pmu/pmu_pass.git" +GIT_PASS_REPO_NAME="PMU_pass" +GIT_PASS_REPO_ID=67640 +GIT_PASS_REPO="${GIT_SERVER}/acs/public/software/pmu/${GIT_PASS_REPO_NAME}.git" #========================= Get User input ========================= usage(){ echo "Usage:" echo " -I [Path to Image: -I /path/to/image/]" echo " -N [Hostname to use: -N name]" - echo " -T [Git !project! acces token: -T GITLAB token]" echo " -B [Git branch -B development]" - echo " -S [Path to SSL Cert: -S /path/to/cert]" + echo " -S [Path to SSL Cert: -S /path/to/cert]" echo " -y [Dont ask for confirmations]" echo "" echo "Credentials for ansible/pass repo" - echo " -U [${GIT_SERVER} username -B myName]" + echo " -U [${GIT_SERVER} username -U myName]" echo " -P [${GIT_SERVER} pass/token -P Token]" exit } -while getopts ":I:N:T:B:S:yU:P:" opt +while getopts ":I:N:B:S:U::P::yd" opt do case "${opt}" in I) IMAGE_FILE=${OPTARG};; S) SSL_CERT_FILE=${OPTARG} ;; N) NODENAME=${OPTARG} ;; y) ASK_CONFIRM=false ;; - T) PMU_GIT_TOKEN=${OPTARG} ;; B) PMU_GIT_BRANCH=${OPTARG} ;; - U) GIT_USERNAME=${OPTARG} - P) GIT_PASS=${OPTARG} + U) GIT_USERNAME=${OPTARG} ;; + P) GIT_PASS=${OPTARG} ;; + d) DEBUG=true ;; *) echo "Unknown argument ${OPTARG}" usage ;; :) usage ;; esac done -if [ $OPTIND -eq 1 ]; then - echo "Not enougth options" - usage; -fi + +#if [ $OPTIND -eq 1 ]; then +# echo "Not enougth options" +# usage; +#fi #Ensure RIASC Image file is found if ! [[ -r ${IMAGE_FILE} ]]; then #TODO: Check if this is a .zip file @@ -64,12 +76,6 @@ if ! [[ -n ${NODENAME} ]]; then usage fi -#Ensure git project token is supplied -if ! [[ -n ${PMU_GIT_TOKEN} ]]; then - echo "No git acces token supplied" - usage -fi - #Ensure SSL cert file is found if ! [[ -r ${SSL_CERT_FILE} ]]; then echo "SSL cert file '${SSL_CERT_FILE}' does not exist" @@ -85,11 +91,25 @@ else exit fi -exit #========================= Check if we can access Git repos ========================= -#Dont do for now +GIT_API_URL="https://${GIT_SERVER}/api/v4" +GIT_API_AUTH_HEADER="--header 'PRIVATE-TOKEN: ${GIT_PASS}'" + +#Check Permissions on ansible repo via gitlab api +GIT_ANSIBLE_PERM_J=$(curl -s --header "PRIVATE-TOKEN: ${GIT_PASS}" ${GIT_API_URL}/projects/${GIT_ANSIBLE_REPO_ID} | jq -r '.permissions') +if [[ $(echo ${GIT_ANSIBLE_PERM_J} | jq -r '.group_access.access_level') -lt ${GIT_MIN_ACCESS_LEVEL} ]] && [[ $(echo ${GIT_ANSIBLE_PERM_J} | jq -r '.project_access.access_level') -lt ${GIT_MIN_ACCESS_LEVEL} ]]; then + echo "You appear to not have the right permissions for the ${GIT_ANSIBLE_REPO_NAME} repository" + echo "Please make sure that you have an access of at least ${GIT_MIN_ACCESS_LEVEL}" + exit +fi -#Check if repos (pass + ansible) are available +#Check Permissions on pass repo via gitlab api +GIT_PASS_PERM_J=$(curl -s --header "PRIVATE-TOKEN: ${GIT_PASS}" ${GIT_API_URL}/projects/${GIT_PASS_REPO_ID} | jq -r '.permissions') +if [[ $(echo ${GIT_PASS_PERM_J} | jq -r '.group_access.access_level') -lt ${GIT_MIN_ACCESS_LEVEL} ]] && [[ $(echo ${GIT_PASS_PERM_J} | jq -r '.project_access.access_level') -lt ${GIT_MIN_ACCESS_LEVEL} ]]; then + echo "You appear to not have the right permissions for the ${GIT_PASS_REPO_NAME} repository" + echo "Please make sure that you have an access of at least ${GIT_MIN_ACCESS_LEVEL}" + exit +fi #================================= Confirm settings ================================= echo "Gathered following configuration:" @@ -97,9 +117,10 @@ echo "Nodename: ${NODENAME}" echo "Image: ${IMAGE_FILE}" echo "SSL_CERT: ${SSL_CERT_FILE}" echo "Config: ${CONFIG_PATH}" +echo "Git User: ${GIT_USERNAME}" if [[ -n ${PMU_GIT_BRANCH} ]]; then -echo "Branch: ${PMU_GIT_BRANCH}" + echo "Branch: ${PMU_GIT_BRANCH}" fi if [[ ${ASK_CONFIRM} == true ]]; then @@ -130,12 +151,13 @@ cp ${CONFIG_PATH} "${NODENAME}/riasc.yaml" cp "user-data" "${NODENAME}/user-data" echo "Done" -#3. Make sure repos are here - - -#4. Enter working directory +#3. Enter working directory pushd ${NODENAME} +#4. Make sure repos are here +git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_PASS_REPO}" +git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_ANSIBLE_REPO}" -b ${PMU_GIT_BRANCH:main} + #5. Generate secrets and write to files echo "Generating secrets" @@ -147,6 +169,15 @@ echo "${VAULT_KEY}" EOF #Git token +#request git token from API TODO: check if old token exists and delte +TOKEN_RESP_J=$(curl -s --request POST --header "PRIVATE-TOKEN: ${GIT_PASS}" --header "Content-Type:application/json" --data "{ \"name\":\"${NODENAME}\", \"scopes\":[\"read_repository\"]}" "${GIT_API_URL}/projects/${GIT_ANSIBLE_REPO_ID}/access_tokens") +PMU_GIT_TOKEN=$(echo ${TOKEN_RESP_J} | jq -r '.token') +echo ${PMU_GIT_TOKEN} +if [ -z ${PMU_GIT_TOKEN} ]; then + echo "Error while creating git access token" + exit +fi + echo ${PMU_GIT_TOKEN} > "git_token.secret" #TODO: braucht man das?? #SNMP key @@ -171,9 +202,7 @@ if [[ -n ${PMU_GIT_BRANCH} ]]; then fi #Git token -if [[ -n ${PMU_GIT_TOKEN} ]]; then - sed -i -e "s/git.rwth-aachen/pmu-acs:${PMU_GIT_TOKEN}@git.rwth-aachen/g" riasc.yaml -fi +sed -i -e "s/git.rwth-aachen/pmu-acs:${PMU_GIT_TOKEN}@git.rwth-aachen/g" riasc.yaml #Node Name sed -i \ @@ -240,6 +269,18 @@ guestfish < edgeflex.fish #rm -f ${NODE_IMAGE_FILE}.img #echo "Done" + +#4. Clean up +if [[ DEBUG == false ]]; then +echo "removing stuff" +rm "acs-lab.conf" +rm "edgeflex.fish" +rm "riasc.yaml" +rm "user-data" +rm "*.secret" +fi + + #4. Final outputs echo "Please write the new image to an SD card:" lsblk |grep sd From cab9cb5cf8178e2f05d6aa0c358c2f15af9c2111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Mon, 17 Jan 2022 23:01:03 +0100 Subject: [PATCH 11/32] WIP: encrypting files with vaul --- rpi/update_image.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index b5857f4..e0dd95e 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -180,9 +180,9 @@ fi echo ${PMU_GIT_TOKEN} > "git_token.secret" #TODO: braucht man das?? -#SNMP key -SNMP_KEY=$(openssl rand -hex 10) -echo ${SNMP_KEY} > "snmp.secret" #TODO: braucht man das?? +#SNMP pass +SNMP_PASS=$(openssl rand -hex 10) +echo ${SNMP_PASS} > "snmp.secret" #TODO: braucht man das?? echo "Done" @@ -215,9 +215,14 @@ echo "Done" #1. Encrypt with ansible #OpenVPN. -#SNMP Key +ansible-vault encrypt --vault-password-file ./vaultkey.secret acs-lab.conf +#SNMP +SNMP_PASS_VAULT=$(ansible-vault encrypt_string --vault-password-file ./vaultkey.secret --name SNMP_PASS ${SNMP_PASS}) +echo ${SNMP_PASS_VAULT} #2. Write variables to ansible-repo +#make sure host bin exists + #3. Commit and push ansible-repo @@ -226,7 +231,7 @@ echo "Done" #5. Push to pass repo #================================== Write to Image ================================== - +exit #1. Write patch file echo "Writing Patch file" From da224c07d366c089b20b46433b166c5abfa09b8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Wed, 26 Jan 2022 11:32:32 +0100 Subject: [PATCH 12/32] Add password store and GPG --- rpi/update_image.sh | 89 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index e0dd95e..9c7fdae 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -6,6 +6,9 @@ SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) SCRIPT_OWNER=$(stat -c '%U' ${SCRIPT_PATH}) pushd ${SCRIPT_PATH} +#====================== Required Packages ======================== +#jq, ansible-vault (ansible), pass + #======================= Predefined settings ===================== CONFIG_FILE="riasc.edgeflex.yaml" @@ -25,6 +28,15 @@ GIT_PASS_REPO_NAME="PMU_pass" GIT_PASS_REPO_ID=67640 GIT_PASS_REPO="${GIT_SERVER}/acs/public/software/pmu/${GIT_PASS_REPO_NAME}.git" +PASS_GPG_DIR="gpg" +PASS_GPG_KEYRING="${PASS_GPG_DIR}/keyring.gpg" +PASS_GPG_OPTIONS="--no-default-keyring --keyring ${PASS_GPG_KEYRING} --homedir ${PASS_GPG_DIR}" + +#======================= Convenience Functions ==================== +function pass_cmd() { + PASSWORD_STORE_DIR=${GIT_PASS_REPO_NAME} PASSWORD_STORE_GPG_OPTS=${PASS_GPG_OPTIONS} pass $@ +} + #========================= Get User input ========================= usage(){ echo "Usage:" @@ -35,7 +47,7 @@ usage(){ echo " -y [Dont ask for confirmations]" echo "" echo "Credentials for ansible/pass repo" - echo " -U [${GIT_SERVER} username -U myName]" + echo " -U [${GIT_SERVER} username -U myName]" #Needed? echo " -P [${GIT_SERVER} pass/token -P Token]" exit } @@ -155,21 +167,43 @@ echo "Done" pushd ${NODENAME} #4. Make sure repos are here +echo "Cloning GIT repos" git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_PASS_REPO}" git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_ANSIBLE_REPO}" -b ${PMU_GIT_BRANCH:main} +echo "Done" + +#5. Import GPG keys +echo "Setting up GPG keyring / trustdb" +mkdir ${PASS_GPG_DIR} +touch ${PASS_GPG_KEYRING} +chmod 600 ${PASS_GPG_DIR}/* +chmod 700 ${PASS_GPG_DIR} + +gpg ${PASS_GPG_OPTIONS} --import $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.acs) +for keyfile in $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.acs| xargs basename -a -s .acs); do + echo "${keyfile}:6:" | gpg ${PASS_GPG_OPTIONS} --import-ownertrust; +done +echo "Done" -#5. Generate secrets and write to files +#6. Generate secrets and write to files echo "Generating secrets" #Vault Key -VAULT_KEY=$(openssl rand -hex 128) +#check if password repo allready contains +if [[ $(pass_cmd ${NODENAME}) != 0 ]]; then #No valid vault key was returned form password store + echo "Did not find existing vault key." + pass_cmd generate "${NODENAME}" +fi +VAULT_KEY=$(PASSWORD_STORE_DIR=${GIT_PASS_REPO_NAME} PASSWORD_STORE_GPG_OPTS="--homedir /home/${SCRIPT_OWNER}/.gnupg/" pass show ${NODENAME}) #Dont use GPG opts so the local secret key is used + + cat < vaultkey.secret #!/bin/bash echo "${VAULT_KEY}" EOF #Git token -#request git token from API TODO: check if old token exists and delte +#request git token from API TODO: check if old token exists and delete TOKEN_RESP_J=$(curl -s --request POST --header "PRIVATE-TOKEN: ${GIT_PASS}" --header "Content-Type:application/json" --data "{ \"name\":\"${NODENAME}\", \"scopes\":[\"read_repository\"]}" "${GIT_API_URL}/projects/${GIT_ANSIBLE_REPO_ID}/access_tokens") PMU_GIT_TOKEN=$(echo ${TOKEN_RESP_J} | jq -r '.token') echo ${PMU_GIT_TOKEN} @@ -215,23 +249,39 @@ echo "Done" #1. Encrypt with ansible #OpenVPN. -ansible-vault encrypt --vault-password-file ./vaultkey.secret acs-lab.conf - +ansible-vault encrypt --vault-password-file ./vaultkey.secret acs-lab.conf --output acs-lab.conf.secret #SNMP SNMP_PASS_VAULT=$(ansible-vault encrypt_string --vault-password-file ./vaultkey.secret --name SNMP_PASS ${SNMP_PASS}) -echo ${SNMP_PASS_VAULT} + #2. Write variables to ansible-repo #make sure host bin exists +HOST_BIN="./${GIT_ANSIBLE_REPO_NAME}/inventory/edgeflex/host_vars/${NODENAME}" +if ! [[ -d ${HOST_BIN} ]]; then + mkdir ${HOST_BIN} +fi + +#Replace SNMP Pass +cat < ./${GIT_ANSIBLE_REPO_NAME}/inventory/edgeflex/host_vars/${NODENAME}/snmp.yml +${SNMP_PASS_VAULT} +SNMP_USR: ${NODENAME} +EOF +#Replace openVPN config +cp ./acs-lab.conf.secret ${HOST_BIN} #3. Commit and push ansible-repo - -#4. Encrypt password with PGP key +pushd ${GIT_ANSIBLE_REPO_NAME} +git add . +git commit -m "Running update_image on $(date) for ${NODENAME}" +git push +popd #5. Push to pass repo +pushd ${GIT_PASS_REPO_NAME} +#pass_cmd push +popd #================================== Write to Image ================================== -exit #1. Write patch file echo "Writing Patch file" @@ -276,17 +326,20 @@ guestfish < edgeflex.fish #4. Clean up -if [[ DEBUG == false ]]; then -echo "removing stuff" -rm "acs-lab.conf" -rm "edgeflex.fish" -rm "riasc.yaml" -rm "user-data" -rm "*.secret" +if [[ ${DEBUG} == false ]]; then + echo "removing files" + rm "acs-lab.conf" + rm "edgeflex.fish" + rm "riasc.yaml" + rm "user-data" + rm *.secret + rm ${PASS_GPG_DIR} -r + rm ${GIT_PASS_REPO_NAME} -r + rm ${GIT_ANSIBLE_REPO_NAME} -r fi #4. Final outputs echo "Please write the new image to an SD card:" lsblk |grep sd -echo " dd bs=1M if=${NODE_IMAGE_FILE}.img of=/dev/sdX" +echo " dd bs=1M if=${NODE_IMAGE_FILE}.img of=/dev/sdX" \ No newline at end of file From e5ab67496ef5f0d3853253e3101d90a9e7b5ac03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Wed, 26 Jan 2022 11:36:21 +0100 Subject: [PATCH 13/32] replace tab with 2 spaces --- rpi/update_image.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 9c7fdae..61d6468 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -232,9 +232,9 @@ sed -i \ #Select branch if [[ -n ${PMU_GIT_BRANCH} ]]; then sed -i \ - -e "/url: /a\\\\tbranch: ${PMU_GIT_BRANCH}" riasc.yaml + -e "/url: /a\\ branch: ${PMU_GIT_BRANCH}" riasc.yaml fi - +exit #Git token sed -i -e "s/git.rwth-aachen/pmu-acs:${PMU_GIT_TOKEN}@git.rwth-aachen/g" riasc.yaml @@ -278,7 +278,7 @@ popd #5. Push to pass repo pushd ${GIT_PASS_REPO_NAME} -#pass_cmd push +pass_cmd push popd #================================== Write to Image ================================== From d3ed475f389d6e310b7741f1dedc7496ff38b496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Wed, 26 Jan 2022 14:34:05 +0100 Subject: [PATCH 14/32] changed how vault passwords are created/handeled --- rpi/update_image.sh | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 61d6468..65af5bf 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -34,7 +34,7 @@ PASS_GPG_OPTIONS="--no-default-keyring --keyring ${PASS_GPG_KEYRING} --homedir $ #======================= Convenience Functions ==================== function pass_cmd() { - PASSWORD_STORE_DIR=${GIT_PASS_REPO_NAME} PASSWORD_STORE_GPG_OPTS=${PASS_GPG_OPTIONS} pass $@ + PASSWORD_STORE_DIR="${SCRIPT_PATH}/${NODENAME}/${GIT_PASS_REPO_NAME}" PASSWORD_STORE_GPG_OPTS=${PASS_GPG_OPTIONS} PASSWORD_STORE_CLIP_TIME=1 pass $@ } #========================= Get User input ========================= @@ -45,6 +45,7 @@ usage(){ echo " -B [Git branch -B development]" echo " -S [Path to SSL Cert: -S /path/to/cert]" echo " -y [Dont ask for confirmations]" + echo " -d [Debug mode. Dont delete temp files. Dont push to Repos]" echo "" echo "Credentials for ansible/pass repo" echo " -U [${GIT_SERVER} username -U myName]" #Needed? @@ -60,10 +61,10 @@ do S) SSL_CERT_FILE=${OPTARG} ;; N) NODENAME=${OPTARG} ;; y) ASK_CONFIRM=false ;; + d) DEBUG=true ;; B) PMU_GIT_BRANCH=${OPTARG} ;; U) GIT_USERNAME=${OPTARG} ;; P) GIT_PASS=${OPTARG} ;; - d) DEBUG=true ;; *) echo "Unknown argument ${OPTARG}" usage ;; :) usage ;; @@ -77,7 +78,7 @@ done #fi #Ensure RIASC Image file is found -if ! [[ -r ${IMAGE_FILE} ]]; then #TODO: Check if this is a .zip file +if ! [[ -r ${IMAGE_FILE} ]]; then echo "Image file '${IMAGE_FILE}' does not exist" usage fi @@ -135,6 +136,10 @@ if [[ -n ${PMU_GIT_BRANCH} ]]; then echo "Branch: ${PMU_GIT_BRANCH}" fi +if [[ ${DEBUG} = true ]]; then + echo "Running in Debug mode" +fi + if [[ ${ASK_CONFIRM} == true ]]; then echo "Continue? (Y,n)" read inp @@ -143,7 +148,6 @@ if [[ ${ASK_CONFIRM} == true ]]; then fi fi - #============================== Setup to create patch ============================== #1. Create temp directory @@ -170,6 +174,7 @@ pushd ${NODENAME} echo "Cloning GIT repos" git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_PASS_REPO}" git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_ANSIBLE_REPO}" -b ${PMU_GIT_BRANCH:main} +pass_cmd git init #tell pass we want to create commits when working with passwords echo "Done" #5. Import GPG keys @@ -189,13 +194,10 @@ echo "Done" echo "Generating secrets" #Vault Key -#check if password repo allready contains -if [[ $(pass_cmd ${NODENAME}) != 0 ]]; then #No valid vault key was returned form password store - echo "Did not find existing vault key." - pass_cmd generate "${NODENAME}" -fi -VAULT_KEY=$(PASSWORD_STORE_DIR=${GIT_PASS_REPO_NAME} PASSWORD_STORE_GPG_OPTS="--homedir /home/${SCRIPT_OWNER}/.gnupg/" pass show ${NODENAME}) #Dont use GPG opts so the local secret key is used - +#backup existing password if exists +pass_cmd mv ${NODENAME} "old/${NODENAME}_$(date '+%Y-%m-%d_%H:%M:%S')" +pass_cmd generate ${NODENAME} -c -n 20 #TODO: this is not a pretty way to do this... +VAULT_KEY=$(xclip -out -select clipboard) cat < vaultkey.secret #!/bin/bash @@ -203,7 +205,7 @@ echo "${VAULT_KEY}" EOF #Git token -#request git token from API TODO: check if old token exists and delete +#request git token from API TOKEN_RESP_J=$(curl -s --request POST --header "PRIVATE-TOKEN: ${GIT_PASS}" --header "Content-Type:application/json" --data "{ \"name\":\"${NODENAME}\", \"scopes\":[\"read_repository\"]}" "${GIT_API_URL}/projects/${GIT_ANSIBLE_REPO_ID}/access_tokens") PMU_GIT_TOKEN=$(echo ${TOKEN_RESP_J} | jq -r '.token') echo ${PMU_GIT_TOKEN} @@ -212,11 +214,10 @@ if [ -z ${PMU_GIT_TOKEN} ]; then exit fi -echo ${PMU_GIT_TOKEN} > "git_token.secret" #TODO: braucht man das?? +echo ${PMU_GIT_TOKEN} > "git_token.secret" #SNMP pass SNMP_PASS=$(openssl rand -hex 10) -echo ${SNMP_PASS} > "snmp.secret" #TODO: braucht man das?? echo "Done" @@ -234,7 +235,7 @@ if [[ -n ${PMU_GIT_BRANCH} ]]; then sed -i \ -e "/url: /a\\ branch: ${PMU_GIT_BRANCH}" riasc.yaml fi -exit + #Git token sed -i -e "s/git.rwth-aachen/pmu-acs:${PMU_GIT_TOKEN}@git.rwth-aachen/g" riasc.yaml @@ -273,12 +274,16 @@ cp ./acs-lab.conf.secret ${HOST_BIN} pushd ${GIT_ANSIBLE_REPO_NAME} git add . git commit -m "Running update_image on $(date) for ${NODENAME}" -git push +if ! [ ${DEBUG} = true ]; then + git push +fi popd #5. Push to pass repo pushd ${GIT_PASS_REPO_NAME} -pass_cmd push +if ! [ ${DEBUG} = true ]; then + pass_cmd git push +fi popd #================================== Write to Image ================================== From 6512bf4438efeb6681ff7890d13b9e1d741e3735 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 31 Jan 2022 15:16:46 +0100 Subject: [PATCH 15/32] various fixes --- rpi/update_image.sh | 44 ++++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 65af5bf..11eee96 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -1,6 +1,5 @@ #!/bin/bash set -e -set -x SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) SCRIPT_OWNER=$(stat -c '%U' ${SCRIPT_PATH}) @@ -14,6 +13,7 @@ pushd ${SCRIPT_PATH} CONFIG_FILE="riasc.edgeflex.yaml" SSL_SEARCH_PATH="./ssl" ASK_CONFIRM=true +UPDATE=false DEBUG=false GIT_SERVER="git.rwth-aachen.de" @@ -21,7 +21,7 @@ GIT_USE_KEY=false GIT_MIN_ACCESS_LEVEL=40 GIT_ANSIBLE_REPO_NAME="pmu-ansible" -GIT_ANSIBLE_REPO_ID=67607 +GIT_ANSIBLE_REPO_ID=61980 GIT_ANSIBLE_REPO="${GIT_SERVER}/acs/public/software/pmu/${GIT_ANSIBLE_REPO_NAME}.git" GIT_PASS_REPO_NAME="PMU_pass" @@ -46,6 +46,7 @@ usage(){ echo " -S [Path to SSL Cert: -S /path/to/cert]" echo " -y [Dont ask for confirmations]" echo " -d [Debug mode. Dont delete temp files. Dont push to Repos]" + echo " -u [Update mode. Dont delete temp files but push to Repos]" echo "" echo "Credentials for ansible/pass repo" echo " -U [${GIT_SERVER} username -U myName]" #Needed? @@ -54,7 +55,7 @@ usage(){ } -while getopts ":I:N:B:S:U::P::yd" opt +while getopts ":I:N:B:S:U::P::ydu" opt do case "${opt}" in I) IMAGE_FILE=${OPTARG};; @@ -62,7 +63,8 @@ do N) NODENAME=${OPTARG} ;; y) ASK_CONFIRM=false ;; d) DEBUG=true ;; - B) PMU_GIT_BRANCH=${OPTARG} ;; + u) UPDATE=true ;; + B) PMU_GIT_BRANCH=${OPTARG} ;; U) GIT_USERNAME=${OPTARG} ;; P) GIT_PASS=${OPTARG} ;; *) echo "Unknown argument ${OPTARG}" @@ -104,6 +106,7 @@ else exit fi + #========================= Check if we can access Git repos ========================= GIT_API_URL="https://${GIT_SERVER}/api/v4" GIT_API_AUTH_HEADER="--header 'PRIVATE-TOKEN: ${GIT_PASS}'" @@ -140,6 +143,10 @@ if [[ ${DEBUG} = true ]]; then echo "Running in Debug mode" fi +if [[ ${UPDATE} = true ]]; then + echo "Running in Update mode" +fi + if [[ ${ASK_CONFIRM} == true ]]; then echo "Continue? (Y,n)" read inp @@ -195,19 +202,28 @@ echo "Generating secrets" #Vault Key #backup existing password if exists -pass_cmd mv ${NODENAME} "old/${NODENAME}_$(date '+%Y-%m-%d_%H:%M:%S')" -pass_cmd generate ${NODENAME} -c -n 20 #TODO: this is not a pretty way to do this... -VAULT_KEY=$(xclip -out -select clipboard) +if [[ -r ${GIT_PASS_REPO_NAME}/${NODENAME}.gpg ]]; then + echo "Backing old PW up" + pass_cmd mv ${NODENAME} "old/${NODENAME}_$(date '+%Y-%m-%d_%H:%M:%S')" +fi + +VAULT_KEY=$(pass_cmd generate ${NODENAME} -n 20 | tail -1) #TODO: this is not a pretty way to do this... cat < vaultkey.secret #!/bin/bash echo "${VAULT_KEY}" EOF +chmod +x vaultkey.secret #Git token #request git token from API -TOKEN_RESP_J=$(curl -s --request POST --header "PRIVATE-TOKEN: ${GIT_PASS}" --header "Content-Type:application/json" --data "{ \"name\":\"${NODENAME}\", \"scopes\":[\"read_repository\"]}" "${GIT_API_URL}/projects/${GIT_ANSIBLE_REPO_ID}/access_tokens") -PMU_GIT_TOKEN=$(echo ${TOKEN_RESP_J} | jq -r '.token') +if [[ ${DEBUG} = false ]]; then + TOKEN_RESP_J=$(curl -s --request POST --header "PRIVATE-TOKEN: ${GIT_PASS}" --header "Content-Type:application/json" --data "{ \"name\":\"${NODENAME}\", \"scopes\":[\"read_repository\"]}" "${GIT_API_URL}/projects/${GIT_ANSIBLE_REPO_ID}/access_tokens") + PMU_GIT_TOKEN=$(echo ${TOKEN_RESP_J} | jq -r '.token') +else + PMU_GIT_TOKEN="TEMPTOKEN" + echo "DID NOT GENERATE TOKEN DUE TO DEBUG MODE" +fi echo ${PMU_GIT_TOKEN} if [ -z ${PMU_GIT_TOKEN} ]; then echo "Error while creating git access token" @@ -273,9 +289,11 @@ cp ./acs-lab.conf.secret ${HOST_BIN} #3. Commit and push ansible-repo pushd ${GIT_ANSIBLE_REPO_NAME} git add . -git commit -m "Running update_image on $(date) for ${NODENAME}" +git commit -m "Running update_image on $(date) for ${NODENAME}" --author="Update Image Script" if ! [ ${DEBUG} = true ]; then git push +else + echo "Didnt push to ansible repo due to debug mode" fi popd @@ -283,6 +301,8 @@ popd pushd ${GIT_PASS_REPO_NAME} if ! [ ${DEBUG} = true ]; then pass_cmd git push +else + echo "Didnt push to pass repo due to debug mode" fi popd @@ -331,7 +351,7 @@ guestfish < edgeflex.fish #4. Clean up -if [[ ${DEBUG} == false ]]; then +if [[ ${DEBUG} == false ]] && [[ ${UPDATE} == false ]]; then echo "removing files" rm "acs-lab.conf" rm "edgeflex.fish" @@ -347,4 +367,4 @@ fi #4. Final outputs echo "Please write the new image to an SD card:" lsblk |grep sd -echo " dd bs=1M if=${NODE_IMAGE_FILE}.img of=/dev/sdX" \ No newline at end of file +echo " dd bs=1M if=${NODE_IMAGE_FILE}.img of=/dev/sdX" From 169b4e22b39a8937a1042ede3b2218e2d83d0c6f Mon Sep 17 00:00:00 2001 From: Vincent Bareiss Date: Tue, 1 Feb 2022 11:15:47 +0100 Subject: [PATCH 16/32] Update Readme --- README.md | 44 ++++++++++++++++++++++++++++++++++++++------ rpi/update_image.sh | 4 ++-- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f2bd9f0..59903da 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ ## Introduction +This fork of the RIasC Provisioning scripts is modified for the use with the edgePMU ## Documentation @@ -15,18 +16,49 @@ For further documentation, please consult: https://riasc.eu/docs/ The scripts have been tested with the following operating systems: -- Ubuntu 20.03 -- Raspbian Buster - -## Initial Config - -Update the content of riasc.yml and user-data on the boot partion of the sd card +- Ubuntu 20.01 +## Initial Setup +Before using this script, you will have to make sure that: +1. The referenced git repositories in `update_image.sh` exist and you have sufficient access rights. +2. Your ansible inventory is located at `{REPO}/inventory/edgeflex` +3. You have created the host_vars directory in your inventory +4. The password repository contains an initialized [*PASSWORD_STORE*](https://www.passwordstore.org/) with the subdirectories `keys` and `old` ## Usage +### 1. Creating an Image See: https://riasc.eu/docs/setup/agent/manual +### 2. Updating an Image +Before flashing the created image, the `update_image.sh` script will write and update the necessary configuration files to the boot partition of the image. + +Additionally, some of the configuration values are written into a git repository. + +To run the `update_image.sh` script, execute the script as **root** an follow the usage guide. + +i.e. +``` +sudo ./update_image.sh -I PATH_TO_IMAGE_FROM_CREATE_IMAGE.SH -N edgepmuXX -B main -S ../../SSL/CERT -U your_git_username -P your_git_access_token +``` + +After the script has finished, the Image can be flashed to the Raspberry PI. + +**Warning:** Running this script will override (and backup) old credentials, etc + +### 3. Updating an edgePMU that is already flashed + +To update an edgePMU that is already flashed, run the `update_image.sh` script with the *-u* option. This will lead to the configuration files getting written to the image and temporary files for you to manually copy to the device in question via SCP. + +You will need to copy: +1. The generated `vaultkey.secret` to `/boot/firmware/vaultkey.secret` +2. The updated `riasc.yaml` to `/boot/firmware/riasc.yaml` +3. The updated `user-data` to `/boot/firmware/user-data` +4. The `git token` to `boot/firmware/git_token.secret` +5. The updated `../common/riasc-update.sh` to `/usr/local/bin/riasc-update.sh` + +Other data such as new SNMP credentials or new vpn configuration files can be distributed via the pmu-ansible repo. + ## Credits - [Steffen Vogel](https://github.com/stv0g) [📧](mailto:post@steffenvogel.de), [Institute for Automation of Complex Power Systems](https://www.acs.eonerc.rwth-aachen.de), [RWTH Aachen University](https://www.rwth-aachen.de) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 11eee96..952405d 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -42,8 +42,8 @@ usage(){ echo "Usage:" echo " -I [Path to Image: -I /path/to/image/]" echo " -N [Hostname to use: -N name]" - echo " -B [Git branch -B development]" - echo " -S [Path to SSL Cert: -S /path/to/cert]" + echo " -B [Git branch for ansible repo -B development]" + echo " -S [Path to SSL Cert: -S /path/to/cert]" echo " -y [Dont ask for confirmations]" echo " -d [Debug mode. Dont delete temp files. Dont push to Repos]" echo " -u [Update mode. Dont delete temp files but push to Repos]" From 2e120226bd18788c308a066fc0015824cebefe09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Mon, 14 Feb 2022 09:03:00 +0100 Subject: [PATCH 17/32] Change acslab to openvpn and make ovpn+snmp opt --- rpi/update_image.sh | 74 +++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 952405d..7c4dbda 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -12,6 +12,8 @@ pushd ${SCRIPT_PATH} #======================= Predefined settings ===================== CONFIG_FILE="riasc.edgeflex.yaml" SSL_SEARCH_PATH="./ssl" +USE_SNMP=false +USE_OVPN=false ASK_CONFIRM=true UPDATE=false DEBUG=false @@ -43,7 +45,8 @@ usage(){ echo " -I [Path to Image: -I /path/to/image/]" echo " -N [Hostname to use: -N name]" echo " -B [Git branch for ansible repo -B development]" - echo " -S [Path to SSL Cert: -S /path/to/cert]" + echo " -C [Path to OpenVPN Cert: -C /path/to/cert]" + echo " -S [Create SNMP config]" echo " -y [Dont ask for confirmations]" echo " -d [Debug mode. Dont delete temp files. Dont push to Repos]" echo " -u [Update mode. Dont delete temp files but push to Repos]" @@ -55,11 +58,12 @@ usage(){ } -while getopts ":I:N:B:S:U::P::ydu" opt +while getopts ":I:N:B:C::SU::P::ydu" opt do case "${opt}" in I) IMAGE_FILE=${OPTARG};; - S) SSL_CERT_FILE=${OPTARG} ;; + C) OVPN_CERT_FILE=${OPTARG} ;; + S) ${USE_SNMP}=true ;; N) NODENAME=${OPTARG} ;; y) ASK_CONFIRM=false ;; d) DEBUG=true ;; @@ -91,10 +95,15 @@ if ! [[ -n ${NODENAME} ]]; then usage fi -#Ensure SSL cert file is found -if ! [[ -r ${SSL_CERT_FILE} ]]; then - echo "SSL cert file '${SSL_CERT_FILE}' does not exist" - usage +#Check if OVPN cert file has been supplied and is valid +if [ -z ${OVPN_CERT_FILE} ]; then + $USE_OVPN=false +else + $USE_OVPN=true + if ! [[ -r ${OVPN_CERT_FILE} ]]; then + echo "OVPN cert file '${OVPN_CERT_FILE}' does not exist" + usage + fi fi #Ensure default config is found @@ -131,7 +140,13 @@ fi echo "Gathered following configuration:" echo "Nodename: ${NODENAME}" echo "Image: ${IMAGE_FILE}" -echo "SSL_CERT: ${SSL_CERT_FILE}" + +if ! [ ${USE_OVPN} ]; then + echo "OpenVPN Cert: ${OVPN_CERT_FILE}" +else + echo "Not using OpenVPN" +fi + echo "Config: ${CONFIG_PATH}" echo "Git User: ${GIT_USERNAME}" @@ -169,7 +184,9 @@ mkdir ${NODENAME} echo "Copying files" NODE_IMAGE_FILE="${NODENAME}_IMAGE" cp ${IMAGE_FILE} "${NODENAME}/${NODE_IMAGE_FILE}.img" -cp ${SSL_CERT_FILE} "${NODENAME}/acs-lab.conf" +if [[ ${USE_OVPN} == true ]]; then + cp ${OVPN_CERT_FILE} "${NODENAME}/openvpn.conf" +fi cp ${CONFIG_PATH} "${NODENAME}/riasc.yaml" cp "user-data" "${NODENAME}/user-data" echo "Done" @@ -265,10 +282,14 @@ echo "Done" #============================== Edit Files in git repo ============================= #1. Encrypt with ansible -#OpenVPN. -ansible-vault encrypt --vault-password-file ./vaultkey.secret acs-lab.conf --output acs-lab.conf.secret +#OpenVPN +if [[ ${USE_OVPN} == true ]]; then + ansible-vault encrypt --vault-password-file ./vaultkey.secret openvpn.conf --output openvpn.conf.secret +fi #SNMP -SNMP_PASS_VAULT=$(ansible-vault encrypt_string --vault-password-file ./vaultkey.secret --name SNMP_PASS ${SNMP_PASS}) +if [[ ${USE_SNMP} == true ]]; then + SNMP_PASS_VAULT=$(ansible-vault encrypt_string --vault-password-file ./vaultkey.secret --name SNMP_PASS ${SNMP_PASS}) +fi #2. Write variables to ansible-repo #make sure host bin exists @@ -278,13 +299,22 @@ if ! [[ -d ${HOST_BIN} ]]; then fi #Replace SNMP Pass -cat < ./${GIT_ANSIBLE_REPO_NAME}/inventory/edgeflex/host_vars/${NODENAME}/snmp.yml -${SNMP_PASS_VAULT} -SNMP_USR: ${NODENAME} +if [[ ${USE_OVPN} == true ]]; then + cat < ./${GIT_ANSIBLE_REPO_NAME}/inventory/edgeflex/host_vars/${NODENAME}/snmp.yml + ${SNMP_PASS_VAULT} + SNMP_USR: ${NODENAME} EOF +else + #Loesche alte SNMP config if it exists to not confuse ansible + rm ${HOST_BIN}/snmp.yml +fi #Replace openVPN config -cp ./acs-lab.conf.secret ${HOST_BIN} +if [[ ${USE_OVPN} == true ]]; then + cp ./openvpn.conf.secret ${HOST_BIN} +else + rm ${HOST_BIN}/openvpn.conf.secret +fi #3. Commit and push ansible-repo pushd ${GIT_ANSIBLE_REPO_NAME} @@ -332,10 +362,14 @@ copy-in riasc.yaml /boot copy-in user-data /boot copy-in vaultkey.secret /boot copy-in git_token.secret /boot +EOF -mkdir /boot/openvpn/ -copy-in acs-lab.conf /boot/openvpn +if [[ ${USE_OVPN} == true ]]; then + cat < edgeflex.fish + mkdir /boot/openvpn/ + copy-in openvpn.conf /boot/openvpn EOF +fi #2. Write patch to image echo "Patching image with guestfish..." @@ -353,7 +387,9 @@ guestfish < edgeflex.fish #4. Clean up if [[ ${DEBUG} == false ]] && [[ ${UPDATE} == false ]]; then echo "removing files" - rm "acs-lab.conf" + if [[ ${USE_OVPN} == true ]]; then + rm "openvpn.conf" + fi rm "edgeflex.fish" rm "riasc.yaml" rm "user-data" From b8c36a202bb11837c558ec72af7afd49ebdb3591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Mon, 14 Feb 2022 09:05:46 +0100 Subject: [PATCH 18/32] Add dependency checks --- rpi/update_image.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 7c4dbda..1c88c93 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -6,8 +6,21 @@ SCRIPT_OWNER=$(stat -c '%U' ${SCRIPT_PATH}) pushd ${SCRIPT_PATH} #====================== Required Packages ======================== -#jq, ansible-vault (ansible), pass +#jq, ansible-vault (ansible), pass, gpg, curl, guestfish, git +function check_command() { + if ! command -v $1 &> /dev/null; then + echo "$1 could not be found" + exit + fi +} +check_command jq +check_command ansible-vault +check_command pass +check_command curl +check_command gpg +check_command git +check_command guestfish #======================= Predefined settings ===================== CONFIG_FILE="riasc.edgeflex.yaml" From fadb7b1aeb4398ede31d5157ab018faa2f13085f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Mon, 14 Feb 2022 14:18:50 +0100 Subject: [PATCH 19/32] Fix wrong setting dialog --- rpi/update_image.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 1c88c93..68aba3b 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -76,7 +76,7 @@ do case "${opt}" in I) IMAGE_FILE=${OPTARG};; C) OVPN_CERT_FILE=${OPTARG} ;; - S) ${USE_SNMP}=true ;; + S) USE_SNMP=true ;; N) NODENAME=${OPTARG} ;; y) ASK_CONFIRM=false ;; d) DEBUG=true ;; @@ -110,9 +110,9 @@ fi #Check if OVPN cert file has been supplied and is valid if [ -z ${OVPN_CERT_FILE} ]; then - $USE_OVPN=false + USE_OVPN=false else - $USE_OVPN=true + USE_OVPN=true if ! [[ -r ${OVPN_CERT_FILE} ]]; then echo "OVPN cert file '${OVPN_CERT_FILE}' does not exist" usage @@ -154,15 +154,21 @@ echo "Gathered following configuration:" echo "Nodename: ${NODENAME}" echo "Image: ${IMAGE_FILE}" -if ! [ ${USE_OVPN} ]; then +if [[ ${USE_OVPN} == true ]]; then echo "OpenVPN Cert: ${OVPN_CERT_FILE}" else echo "Not using OpenVPN" fi echo "Config: ${CONFIG_PATH}" -echo "Git User: ${GIT_USERNAME}" +if [[ ${USE_SNMP} == true ]]; then + echo "Creating SNMP files" +else + echo "No SNMP" +fi + +echo "Git User: ${GIT_USERNAME}" if [[ -n ${PMU_GIT_BRANCH} ]]; then echo "Branch: ${PMU_GIT_BRANCH}" fi From 3bde8a1c9c9248494304e368c22982ec96ba7c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Mon, 14 Feb 2022 14:29:14 +0100 Subject: [PATCH 20/32] fix small errors --- rpi/update_image.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 68aba3b..437c98f 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -318,21 +318,21 @@ if ! [[ -d ${HOST_BIN} ]]; then fi #Replace SNMP Pass -if [[ ${USE_OVPN} == true ]]; then +if [[ ${USE_SNMP} == true ]]; then cat < ./${GIT_ANSIBLE_REPO_NAME}/inventory/edgeflex/host_vars/${NODENAME}/snmp.yml ${SNMP_PASS_VAULT} SNMP_USR: ${NODENAME} EOF else #Loesche alte SNMP config if it exists to not confuse ansible - rm ${HOST_BIN}/snmp.yml + rm -f ${HOST_BIN}/snmp.yml fi #Replace openVPN config if [[ ${USE_OVPN} == true ]]; then cp ./openvpn.conf.secret ${HOST_BIN} else - rm ${HOST_BIN}/openvpn.conf.secret + rm -f ${HOST_BIN}/openvpn.conf.secret fi #3. Commit and push ansible-repo @@ -384,7 +384,7 @@ copy-in git_token.secret /boot EOF if [[ ${USE_OVPN} == true ]]; then - cat < edgeflex.fish + cat <> edgeflex.fish mkdir /boot/openvpn/ copy-in openvpn.conf /boot/openvpn EOF From f0fc2ebe392aca426484624b193b518c65b2a5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Tue, 15 Feb 2022 13:41:37 +0100 Subject: [PATCH 21/32] add docker functionality --- .gitignore | 1 + docker/Dockerfile | 17 +++++++++++++++++ docker/docker-compose.yml | 11 +++++++++++ 3 files changed, 29 insertions(+) create mode 100644 .gitignore create mode 100644 docker/Dockerfile create mode 100644 docker/docker-compose.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58f8ea3 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +docker/ssl* \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..9608348 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,17 @@ +from ubuntu:latest + +#Get dependencies +RUN apt-get update +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata +RUN apt-get install -y jq ansible pass curl gpg git wget unzip zip +ENV LIBGUESTFS_BACKEND=direct + +WORKDIR edgepmu +RUN apt-get update && apt-get install --no-install-recommends -y libguestfs-tools qemu-utils linux-image-generic +RUN git clone https://github.com/cl0-de/riasc-provisioning.git -b development +ENV FLAVOR=edgeflex +ENV LIBGUESTFS_DEBUG=1 +ENV LIBGUESTFS_TRACE=1 +RUN ./riasc-provisioning/rpi/create_image.sh + +CMD /bin/bash \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..fd80b03 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,11 @@ +version: "3.9" +services: + img_create: + test: "Hallo" + build: . + tty: true + volumes: + - ./ssl:/edgepmu/ssl + - ./output:/edgepmu/output + command: + /bin/bash ${test}" From dd75d85f3298c9c2007f130a6da83175740c19a1 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 16 Feb 2022 14:07:18 +0100 Subject: [PATCH 22/32] small fixes --- common/riasc-update.sh | 2 +- rpi/update_image.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common/riasc-update.sh b/common/riasc-update.sh index 53f8e05..7a6ac43 100755 --- a/common/riasc-update.sh +++ b/common/riasc-update.sh @@ -156,7 +156,7 @@ if [ $(config '.ansible.verify_commit') == "true" ]; then ANSIBLE_OPTS+="--verify-commit" fi -if ! [ -z $(config '.ansible.branch') ]; then +if ! [ $(config '.ansible.branch') = null ]; then ANSIBLE_OPTS+=" --checkout $(config '.ansible.branch')" fi diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 437c98f..5420bf0 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +set -x SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) SCRIPT_OWNER=$(stat -c '%U' ${SCRIPT_PATH}) @@ -216,7 +217,8 @@ pushd ${NODENAME} #4. Make sure repos are here echo "Cloning GIT repos" git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_PASS_REPO}" -git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_ANSIBLE_REPO}" -b ${PMU_GIT_BRANCH:main} +echo ${PMU_GIT_BRANCH} +git clone "https://${GIT_USERNAME}:${GIT_PASS}@${GIT_ANSIBLE_REPO}" -b ${PMU_GIT_BRANCH:-master} pass_cmd git init #tell pass we want to create commits when working with passwords echo "Done" From e40be718d0827eeb2c91df9aa8f678180ed884e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Sun, 10 Apr 2022 19:00:56 +0200 Subject: [PATCH 23/32] strip color bytes from pass output --- rpi/update_image.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 5420bf0..862ad09 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -set -x + SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) SCRIPT_OWNER=$(stat -c '%U' ${SCRIPT_PATH}) @@ -245,7 +245,8 @@ if [[ -r ${GIT_PASS_REPO_NAME}/${NODENAME}.gpg ]]; then pass_cmd mv ${NODENAME} "old/${NODENAME}_$(date '+%Y-%m-%d_%H:%M:%S')" fi -VAULT_KEY=$(pass_cmd generate ${NODENAME} -n 20 | tail -1) #TODO: this is not a pretty way to do this... +VAULT_KEY=$(pass_cmd generate ${NODENAME} -n 20 | tail -1 | sed -r "s/\x1B\[([0-9]{1,3}((;[0-9]{1,3})*)?)?[m|K]//g" ) #TODO: this is not a pretty way to do this... +#SED curtesy of: https://gist.github.com/stevenh512/2245881 cat < vaultkey.secret #!/bin/bash From 42e57646a73a62c6ca7eee0e3044c520556ed251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Barei=C3=9F?= Date: Tue, 19 Jul 2022 15:28:20 +0200 Subject: [PATCH 24/32] Changes to reflect new structure of pass repo --- rpi/update_image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 862ad09..3dff9df 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -34,7 +34,7 @@ DEBUG=false GIT_SERVER="git.rwth-aachen.de" GIT_USE_KEY=false -GIT_MIN_ACCESS_LEVEL=40 +GIT_MIN_ACCESS_LEVEL=30 GIT_ANSIBLE_REPO_NAME="pmu-ansible" GIT_ANSIBLE_REPO_ID=61980 @@ -229,7 +229,7 @@ touch ${PASS_GPG_KEYRING} chmod 600 ${PASS_GPG_DIR}/* chmod 700 ${PASS_GPG_DIR} -gpg ${PASS_GPG_OPTIONS} --import $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.acs) +gpg ${PASS_GPG_OPTIONS} --import $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.asc) for keyfile in $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.acs| xargs basename -a -s .acs); do echo "${keyfile}:6:" | gpg ${PASS_GPG_OPTIONS} --import-ownertrust; done From 1167b2f01425b923541c8a0adb5559b056df1daa Mon Sep 17 00:00:00 2001 From: KarlBarx Date: Thu, 15 Sep 2022 11:57:16 +0200 Subject: [PATCH 25/32] fix spelling mistake in gpg key import --- rpi/update_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpi/update_image.sh b/rpi/update_image.sh index 3dff9df..fba5052 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -230,7 +230,7 @@ chmod 600 ${PASS_GPG_DIR}/* chmod 700 ${PASS_GPG_DIR} gpg ${PASS_GPG_OPTIONS} --import $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.asc) -for keyfile in $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.acs| xargs basename -a -s .acs); do +for keyfile in $(ls -1 ${GIT_PASS_REPO_NAME}/keys/*.asc| xargs basename -a -s .asc); do echo "${keyfile}:6:" | gpg ${PASS_GPG_OPTIONS} --import-ownertrust; done echo "Done" From 1d23764b5f3e04a2ba39c63daaf8fa594d70ae7d Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Thu, 14 Mar 2024 17:54:07 +0100 Subject: [PATCH 26/32] update for docker compose --- .gitignore | 4 +- common/riasc.edgeflex.yaml | 37 --------- common/{riasc.yaml => riasc.raspios.yaml} | 0 common/riasc.ubuntu.yaml | 40 ++++++++++ docker/Dockerfile | 17 +++-- docker/docker-compose.yml | 18 +++-- rpi/create_image.sh | 91 +++++++++++++++-------- 7 files changed, 121 insertions(+), 86 deletions(-) delete mode 100644 common/riasc.edgeflex.yaml rename common/{riasc.yaml => riasc.raspios.yaml} (100%) create mode 100644 common/riasc.ubuntu.yaml diff --git a/.gitignore b/.gitignore index 58f8ea3..43c7c1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -docker/ssl* \ No newline at end of file +docker/download/* +docker/images/* +docker/output/* \ No newline at end of file diff --git a/common/riasc.edgeflex.yaml b/common/riasc.edgeflex.yaml deleted file mode 100644 index 0e5db46..0000000 --- a/common/riasc.edgeflex.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -# RIasC configuration file -# See also: https://erigrid2.github.io/riasc/docs/setup/config - -# A unique hostname to identify the node -hostname: edgepmu - -ansible: - # List of PGP keys which are used to verify the commits in the Ansible repo - keys: [] - - keyserver: keys.openpgp.org - - # A repository containing ansible playbooks which will be fetched via ansible-pull - url: https://git.rwth-aachen.de/acs/public/software/pmu/pmu-ansible.git - - verify_commit: false - - # The playbook which should be provision the node - playbook: playbook.yml - - # A path to the Ansible inventory within the repo from above - inventory: inventory/edgeflex/hosts.yml - - # extra_args: - # - --only-if-changed - - # Additional variables which are passed to the Ansible playbook for provisioning - variables: - - # A list of SSH keys which will be added to the 'pi' user - #additional_ssh_keys: - #- "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDOmn+2RUo3XrHmm7h3w647+f6DlV5wXzBJSYLa7NBVlEasBd6Gxp4HDh1iKmNgZrneqXWOAH19P41k1qiwAx8/gUYDlnoah30RWp3qlXXIN+RUCUyUx34tatGTSDynuAyYKAOg0CawK066pSaRsMan2JdhL+r0YAKDXswMU8NVdc32AFTJLrZyGzlrFn4y7hSgHCtOy2RvoULWcjVtcF0GLuSr3WCLUwm1Qy83tft++5FJCcNtg986or5OOeWtZxFU035Q+2Khd2JqyweFTmCMRXEXAzUMv6Lqxuw19qpK4eOOU59oybXlga5yLFZY0CVBK8XVVnA0pkCnabQPHvVJWL7tcDVvzfnd0JdHpQ6oMsYFM7MdncKbHay5/g/tUz34WXPPAPWvciyLkI1NMV++JpaPeMEkAvrsUUZMko+4EQ75ZErUOY3PAJG+2Jilb79QTRMzpCVWfQ0A/Zx1O7Y0sLks+f5VDJaPnHl0vXetFIkEpv9XjomNZgp81m3XRItJh0+BwGqoYtc+MTLelAu0oap1LyHSbpBWGzKxi2tW9VOrX7/pI18f0AhhCc/BG/6UFjMhyYzGj9wg6HWAo1N8hlhdeuNic004hbADX7qzXJMyVEol5AL0rpu3j3j5zTeULY3HxiE+FC6yKg1DKvcxHkmwXoWJYwW9yAgYDyjhbw==" # Manuel Pitz - # Set this to true if you want to login via SSH keys only. - # If you dont have an SSH key, set this to false. - # Important: Dont forget to change the default password after your first login! - disable_password_login: false diff --git a/common/riasc.yaml b/common/riasc.raspios.yaml similarity index 100% rename from common/riasc.yaml rename to common/riasc.raspios.yaml diff --git a/common/riasc.ubuntu.yaml b/common/riasc.ubuntu.yaml new file mode 100644 index 0000000..111ee10 --- /dev/null +++ b/common/riasc.ubuntu.yaml @@ -0,0 +1,40 @@ +--- +# RIasC configuration file +# See also: https://erigrid2.github.io/riasc/docs/setup/config + +# A unique hostname to identify the node +hostname: edgepmu + +ansible: + # List of PGP keys which are used to verify the commits in the Ansible repo + keys: [] + + keyserver: keys.openpgp.org + + # A repository containing ansible playbooks which will be fetched via ansible-pull + url: https://git.rwth-aachen.de/acs/public/software/pmu/pmu-ansible.git + + verify_commit: false + + # The playbook which should be provision the node + playbook: playbook.yml + + # A path to the Ansible inventory within the repo from above + inventory: inventory/edgeflex/hosts.yml + + # extra_args: + # - --only-if-changed + + # Additional variables which are passed to the Ansible playbook for provisioning + variables: + + # A list of SSH keys which will be added to the 'pi' user + #additional_ssh_keys: + #- "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDOmn+2RUo3XrHmm7h3w647+f6DlV5wXzBJSYLa7NBVlEasBd6Gxp4HDh1iKmNgZrneqXWOAH19P41k1qiwAx8/gUYDlnoah30RWp3qlXXIN+RUCUyUx34tatGTSDynuAyYKAOg0CawK066pSaRsMan2JdhL+r0YAKDXswMU8NVdc32AFTJLrZyGzlrFn4y7hSgHCtOy2RvoULWcjVtcF0GLuSr3WCLUwm1Qy83tft++5FJCcNtg986or5OOeWtZxFU035Q+2Khd2JqyweFTmCMRXEXAzUMv6Lqxuw19qpK4eOOU59oybXlga5yLFZY0CVBK8XVVnA0pkCnabQPHvVJWL7tcDVvzfnd0JdHpQ6oMsYFM7MdncKbHay5/g/tUz34WXPPAPWvciyLkI1NMV++JpaPeMEkAvrsUUZMko+4EQ75ZErUOY3PAJG+2Jilb79QTRMzpCVWfQ0A/Zx1O7Y0sLks+f5VDJaPnHl0vXetFIkEpv9XjomNZgp81m3XRItJh0+BwGqoYtc+MTLelAu0oap1LyHSbpBWGzKxi2tW9VOrX7/pI18f0AhhCc/BG/6UFjMhyYzGj9wg6HWAo1N8hlhdeuNic004hbADX7qzXJMyVEol5AL0rpu3j3j5zTeULY3HxiE+FC6yKg1DKvcxHkmwXoWJYwW9yAgYDyjhbw==" # Manuel Pitz + # Set this to true if you want to login via SSH keys only. + # If you dont have an SSH key, set this to false. + # Important: Dont forget to change the default password after your first login! + disable_password_login: false + + # Replace this token with the token provided by your RIasC provider + token: XXXXX # changeme! diff --git a/docker/Dockerfile b/docker/Dockerfile index 9608348..13472ba 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -from ubuntu:latest +from ubuntu:22.04 #Get dependencies RUN apt-get update @@ -6,12 +6,13 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata RUN apt-get install -y jq ansible pass curl gpg git wget unzip zip ENV LIBGUESTFS_BACKEND=direct -WORKDIR edgepmu +WORKDIR /tmp RUN apt-get update && apt-get install --no-install-recommends -y libguestfs-tools qemu-utils linux-image-generic -RUN git clone https://github.com/cl0-de/riasc-provisioning.git -b development -ENV FLAVOR=edgeflex -ENV LIBGUESTFS_DEBUG=1 -ENV LIBGUESTFS_TRACE=1 -RUN ./riasc-provisioning/rpi/create_image.sh +# RUN git clone https://github.com/cl0-de/riasc-provisioning.git -b development +ENV FLAVOR=raspios +ENV REPOFOLDER=/tmp +# ENV LIBGUESTFS_DEBUG=1 +# ENV LIBGUESTFS_TRACE=1 +# RUN ./riasc-provisioning/rpi/create_image.sh -CMD /bin/bash \ No newline at end of file +CMD ${REPOFOLDER}/rpi/create_image.sh \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index fd80b03..dbd84ed 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,11 +1,15 @@ -version: "3.9" +version: "3.7" + services: img_create: - test: "Hallo" + environment: + - FLAVOR=ubuntu22.04 + - REPOFOLDER=/tmp/blubber/ build: . - tty: true + tty: true volumes: - - ./ssl:/edgepmu/ssl - - ./output:/edgepmu/output - command: - /bin/bash ${test}" + - ./download:/tmp/download + - ../rpi:/tmp/blubber/rpi + - ../common:/tmp/blubber/common + - ./output:/tmp/output + - ./images:/tmp/images diff --git a/rpi/create_image.sh b/rpi/create_image.sh index b3a16e6..7e242f5 100755 --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -3,31 +3,49 @@ set -e SCRIPT_PATH=$(dirname $(realpath "${BASH_SOURCE[0]}")) -pushd ${SCRIPT_PATH} +cd "${SCRIPT_PATH}" # Settings NODENAME="${NODENAME:-riasc-agent}" TOKEN="${TOKEN:-XXXXX}" -FLAVOR=${FLAVOR:-erigrid} +DOWNLOAD_FOLDER="/tmp/download/" +OUTPUT_FOLDER="/tmp/output/" +IMG_FOLDER="/tmp/images/" +WORKDIR="/tmp/" +REPOFOLDER="/tmp/blubber/" + +FLAVOR=${FLAVOR:-raspios} case ${FLAVOR} in - edgeflex) + ubuntu22.04) OS="ubuntu" ;; - - erigrid) + ubuntu20.04) + OS="ubuntu" + ;; + raspios) OS="raspios" ;; + *) + echo "Flavor $FLAVOR not known!" + exit 0 + ;; esac -case ${OS} in - ubuntu) +case ${FLAVOR} in + ubuntu20.04) IMAGE_FILE="ubuntu-20.04.2-preinstalled-server-arm64+raspi" IMAGE_SUFFIX="img.xz" IMAGE_URL="https://cdimage.ubuntu.com/releases/20.04.2/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" ;; + ubuntu22.04) + IMAGE_FILE="ubuntu-22.04.4-preinstalled-server-arm64+raspi" + IMAGE_SUFFIX="img.xz" + IMAGE_URL="https://cdimage.ubuntu.com/releases/22.04/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" + ;; + raspios) IMAGE_FILE="2021-05-07-raspios-buster-armhf-lite" IMAGE_SUFFIX="zip" @@ -38,7 +56,7 @@ esac RIASC_IMAGE_FILE="$(date +%Y-%m-%d)-riasc-${OS}" function check_command() { - if ! command -v $1 &> /dev/null; then + if ! command -v "$1" &> /dev/null; then echo "$1 could not be found" exit fi @@ -59,37 +77,44 @@ check_command xz # apt-get install libguestfs-tools wget unzip zip xz-utils # Download image -if [ ! -f ${IMAGE_FILE}.${IMAGE_SUFFIX} ]; then +cd ${DOWNLOAD_FOLDER} +if [ ! -f "${IMAGE_FILE}"."${IMAGE_SUFFIX}" ]; then echo "Downloading image..." - wget ${IMAGE_URL} + wget "${IMAGE_URL}" +else + echo "${IMAGE_FILE}.${IMAGE_SUFFIX} exists skipping download" fi + # Unzip image -if [ ! -f ${IMAGE_FILE}.img ]; then +cd ${IMG_FOLDER} + +if [ ! -f "${IMAGE_FILE}".img ]; then echo "Unzipping image..." case ${IMAGE_SUFFIX} in img.xz) - unxz --keep --threads=0 ${IMAGE_FILE}.${IMAGE_SUFFIX} + unxz --keep --threads=0 ${DOWNLOAD_FOLDER}/"${IMAGE_FILE}"."${IMAGE_SUFFIX}" + mv ${DOWNLOAD_FOLDER}/"${IMAGE_FILE}".img ./ ;; zip) - unzip ${IMAGE_FILE}.${IMAGE_SUFFIX} + unzip "${DOWNLOAD_FOLDER}"/"${IMAGE_FILE}"."${IMAGE_SUFFIX}" ;; esac +else + echo "${IMAGE_FILE}.img exists skipping unpack" fi echo "Copying image..." -cp ${IMAGE_FILE}.img ${RIASC_IMAGE_FILE}.img +cd ${WORKDIR} + +cp ${IMG_FOLDER}/"${IMAGE_FILE}".img "${RIASC_IMAGE_FILE}".img # Prepare config -case ${FLAVOR} in - erigrid) - CONFIG_FILE="riasc.yaml" - ;; - *) - CONFIG_FILE="riasc.${FLAVOR}.yaml" - ;; -esac -cp ../common/${CONFIG_FILE} riasc.yaml + + +CONFIG_FILE="riasc.${OS}.yaml" + +cp ${REPOFOLDER}/common/${CONFIG_FILE} riasc.yaml # Patch config sed -i \ @@ -106,7 +131,7 @@ EOF # Download PGP keys for verifying Ansible Git commits echo "Download PGP keys..." mkdir -p keys -wget -O keys/steffen-vogel.asc https://keys.openpgp.org/vks/v1/by-fingerprint/09BE3BAE8D55D4CD8579285A9675EAC34897E6E2 # Steffen Vogel (RWTH) +#wget -O keys/xxx.asc https://xxx # Patching image cat < patch.fish @@ -127,17 +152,17 @@ echo "Available space:" df-h echo "Copy files into image..." -copy-in rootfs/etc/ / -copy-in riasc.yaml /boot +copy-in ${REPOFOLDER}/rpi/rootfs/etc/ / +copy-in ${WORKDIR}/riasc.yaml /boot mkdir-p /etc/systemd/timesyncd.conf.d/ -copy-in fallback-ntp.conf /etc/systemd/timesyncd.conf.d/ +copy-in ${REPOFOLDER}/rpi/fallback-ntp.conf /etc/systemd/timesyncd.conf.d/ mkdir-p /usr/local/bin -copy-in ../common/riasc-update.sh ../common/riasc-set-hostname.sh /usr/local/bin/ +copy-in ${REPOFOLDER}/common/riasc-update.sh ${REPOFOLDER}/common/riasc-set-hostname.sh /usr/local/bin/ glob chmod 755 /usr/local/bin/riasc-*.sh -copy-in keys/ /boot/ +copy-in ${REPOFOLDER}/rpi/keys/ /boot/ echo "Disable daily APT timers" rm /etc/systemd/system/timers.target.wants/apt-daily-upgrade.timer @@ -152,7 +177,7 @@ EOF case ${OS} in ubuntu) cat <> patch.fish -copy-in user-data /boot +copy-in ${REPOFOLDER}/rpi/user-data /boot EOF ;; *) @@ -170,7 +195,7 @@ EOF ;; esac -if [ "${FLAVOR}" = "edgeflex" -a "${OS}" = "ubuntu" ]; then +if [ "${OS}" = "ubuntu" ]; then cat <> patch.fish echo "Disable Grub boot" write-append /boot/usercfg.txt "[all]\ninitramfs initrd.img followkernel\nkernel=vmlinuz\n" @@ -183,8 +208,8 @@ guestfish < patch.fish # Zip image echo "Zipping image..." -rm -f ${RIASC_IMAGE_FILE}.zip -zip ${RIASC_IMAGE_FILE}.zip ${RIASC_IMAGE_FILE}.img +rm -f "${RIASC_IMAGE_FILE}".zip +zip ${OUTPUT_FOLDER}/"${RIASC_IMAGE_FILE}".zip "${RIASC_IMAGE_FILE}".img echo "Please write the new image to an SD card:" echo " dd bs=1M if=${RIASC_IMAGE_FILE}.img of=/dev/sdX" \ No newline at end of file From cc52e9b8df7e02284d9b2b224e20f1406672e251 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Tue, 16 Apr 2024 22:46:29 +0200 Subject: [PATCH 27/32] added env to create script --- .gitignore | 5 ++--- docker/Dockerfile => Dockerfile | 8 +++----- README.md | 36 +++++++++++++++++++++++++++++++++ common/riasc-update.sh | 5 ++++- common/riasc.ubuntu.yaml | 4 ++-- common/villas_docker.yaml | 3 +++ docker/docker-compose.yml | 15 -------------- rpi/create_image.sh | 32 ++++++++++++++++++++++++----- rpi/update_image.sh | 2 +- 9 files changed, 78 insertions(+), 32 deletions(-) rename docker/Dockerfile => Dockerfile (52%) create mode 100644 common/villas_docker.yaml delete mode 100644 docker/docker-compose.yml diff --git a/.gitignore b/.gitignore index 43c7c1d..b678442 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -docker/download/* -docker/images/* -docker/output/* \ No newline at end of file +out/* +env \ No newline at end of file diff --git a/docker/Dockerfile b/Dockerfile similarity index 52% rename from docker/Dockerfile rename to Dockerfile index 13472ba..ed54471 100644 --- a/docker/Dockerfile +++ b/Dockerfile @@ -1,13 +1,11 @@ -from ubuntu:22.04 +FROM ubuntu:22.04 #Get dependencies -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt-get install -y jq ansible pass curl gpg git wget unzip zip +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt update && apt-get -y install tzdata ENV LIBGUESTFS_BACKEND=direct WORKDIR /tmp -RUN apt-get update && apt-get install --no-install-recommends -y libguestfs-tools qemu-utils linux-image-generic +RUN apt-get install --no-install-recommends --no-install-suggests -y libguestfs-tools qemu-utils linux-image-generic wget unzip zip # RUN git clone https://github.com/cl0-de/riasc-provisioning.git -b development ENV FLAVOR=raspios ENV REPOFOLDER=/tmp diff --git a/README.md b/README.md index 59903da..8d9de87 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,41 @@ # RIasC Provisioning Scripts + +## Usage + +switch to directory docker +``` +docker build --tag "imagebuilder" . +``` +run docker run +``` +docker run \ +--volume ./rpi:/tmp/rpi \ +--volume ./common:/tmp/common \ +--volume ./out/output:/tmp/output \ +--volume ./out/images:/tmp/images \ +--volume ./out/download:/tmp/download \ +--env-file ./env \ +imagebuilder +``` +Create `env` file: +``` +GIT_URL=https://mygiturl +FLAVOR=ubuntu22.04 +REPOFOLDER=/tmp/ +GIT_BRANCH=mybranch +GIT_TOKEN=mytoken +NODENAME=myhost +``` + +List of available variables + +Optional varaibles + +Help +mount ubuntu-22.04.4-preinstalled-server-arm64+raspi.img -o loop,offset=$(( 512 * 526336)) /mnt/ +fdisk -lu ubuntu-22.04.4-preinstalled-server-arm64+raspi.img + [![GitHub](https://img.shields.io/github/license/ERIGrid2/riasc-provisioning)](https://github.com/ERIGrid2/riasc-provisioning/blob/master/LICENSE) - **Based on:** diff --git a/common/riasc-update.sh b/common/riasc-update.sh index 7a6ac43..674de80 100755 --- a/common/riasc-update.sh +++ b/common/riasc-update.sh @@ -150,7 +150,10 @@ ANSIBLE_EXTRA_VARS="$(config --tojson --indent 0 .ansible.variables)" ANSIBLE_OPTS=" --url $(config .ansible.url)" ANSIBLE_OPTS+=" --inventory $(config .ansible.inventory)" ANSIBLE_OPTS+=" $(config '.ansible.extra_args // [ ] | join(" ")')" -ANSIBLE_OPTS+=" --vault-password-file /boot/firmware/vaultkey.secret" +if [ -f /boot/firmware/vaultkey.secret ]; then + ANSIBLE_OPTS+=" --vault-password-file /boot/firmware/vaultkey.secret" +fi + if [ $(config '.ansible.verify_commit') == "true" ]; then ANSIBLE_OPTS+="--verify-commit" diff --git a/common/riasc.ubuntu.yaml b/common/riasc.ubuntu.yaml index 111ee10..96b0e80 100644 --- a/common/riasc.ubuntu.yaml +++ b/common/riasc.ubuntu.yaml @@ -3,7 +3,7 @@ # See also: https://erigrid2.github.io/riasc/docs/setup/config # A unique hostname to identify the node -hostname: edgepmu +hostname: dummyHostname ansible: # List of PGP keys which are used to verify the commits in the Ansible repo @@ -12,7 +12,7 @@ ansible: keyserver: keys.openpgp.org # A repository containing ansible playbooks which will be fetched via ansible-pull - url: https://git.rwth-aachen.de/acs/public/software/pmu/pmu-ansible.git + url: dummyGitUrl verify_commit: false diff --git a/common/villas_docker.yaml b/common/villas_docker.yaml new file mode 100644 index 0000000..aa98faa --- /dev/null +++ b/common/villas_docker.yaml @@ -0,0 +1,3 @@ +hostname: villas-box + +ansible: \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index dbd84ed..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: "3.7" - -services: - img_create: - environment: - - FLAVOR=ubuntu22.04 - - REPOFOLDER=/tmp/blubber/ - build: . - tty: true - volumes: - - ./download:/tmp/download - - ../rpi:/tmp/blubber/rpi - - ../common:/tmp/blubber/common - - ./output:/tmp/output - - ./images:/tmp/images diff --git a/rpi/create_image.sh b/rpi/create_image.sh index 7e242f5..36f28bf 100755 --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -13,7 +13,7 @@ DOWNLOAD_FOLDER="/tmp/download/" OUTPUT_FOLDER="/tmp/output/" IMG_FOLDER="/tmp/images/" WORKDIR="/tmp/" -REPOFOLDER="/tmp/blubber/" +REPOFOLDER="${REPOFOLDER:-/tmp/}" FLAVOR=${FLAVOR:-raspios} @@ -65,6 +65,11 @@ function check_command() { # Show config echo "Using hostname: ${NODENAME}" echo "Using token: ${TOKEN}" +echo "Using flavor: ${FLAVOR}" +echo "Using temp folder: ${REPOFOLDER}" +echo "Using repo: ${GIT_URL}" +echo "Using branch: ${GIT_BRANCH}" +echo "Using token: ${GIT_TOKEN}" # Check that required commands exist echo "Check if required commands are installed..." @@ -80,7 +85,9 @@ check_command xz cd ${DOWNLOAD_FOLDER} if [ ! -f "${IMAGE_FILE}"."${IMAGE_SUFFIX}" ]; then echo "Downloading image..." - wget "${IMAGE_URL}" + wget \ + --progress=bar:force \ + "${IMAGE_URL}" else echo "${IMAGE_FILE}.${IMAGE_SUFFIX} exists skipping download" fi @@ -114,14 +121,28 @@ cp ${IMG_FOLDER}/"${IMAGE_FILE}".img "${RIASC_IMAGE_FILE}".img CONFIG_FILE="riasc.${OS}.yaml" -cp ${REPOFOLDER}/common/${CONFIG_FILE} riasc.yaml +cp "${REPOFOLDER}"/common/${CONFIG_FILE} riasc.yaml # Patch config sed -i \ -e "s/XXXXX/${TOKEN}/g" \ - -e "s/riasc-agent/${NODENAME}/g" \ + -e "s/dummyHostname/${NODENAME}/g" \ riasc.yaml +#Select branch +if [[ -n ${GIT_BRANCH} ]]; then + sed -i \ + -e "/url: /a\\ branch: ${GIT_BRANCH}" \ + riasc.yaml +fi + +#Git token +if [[ -n ${GIT_TOKEN} ]]; then + sed -i \ + -e "s,dummyGitUrl,riasc:${GIT_TOKEN}@${GIT_URL},g" \ + riasc.yaml +fi + # Prepare systemd-timesyncd config cat > fallback-ntp.conf < patch.fish echo "Loading image..." @@ -210,6 +231,7 @@ guestfish < patch.fish echo "Zipping image..." rm -f "${RIASC_IMAGE_FILE}".zip zip ${OUTPUT_FOLDER}/"${RIASC_IMAGE_FILE}".zip "${RIASC_IMAGE_FILE}".img +chmod o+w ${OUTPUT_FOLDER}/"${RIASC_IMAGE_FILE}".zip echo "Please write the new image to an SD card:" echo " dd bs=1M if=${RIASC_IMAGE_FILE}.img of=/dev/sdX" \ No newline at end of file diff --git a/rpi/update_image.sh b/rpi/update_image.sh index fba5052..6bd2d67 100755 --- a/rpi/update_image.sh +++ b/rpi/update_image.sh @@ -282,7 +282,7 @@ echo "Done" echo "Writing configuration" sed -i \ - -e "s/edgepmu/${NODENAME}/g" \ + -e "s/exampleHostname/${NODENAME}/g" \ riasc.yaml #Select branch From b577c356d9cc1580fa61ad3294a9ff98dac748d3 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 17 Apr 2024 20:17:57 +0200 Subject: [PATCH 28/32] add ansible vault secret generation, fix hostname setting, update readme --- Dockerfile | 2 +- README.md | 114 +++++++++++++++++++------------------------- rpi/create_image.sh | 51 ++++++++++++++++---- rpi/user-data | 2 +- 4 files changed, 94 insertions(+), 75 deletions(-) diff --git a/Dockerfile b/Dockerfile index ed54471..1cc5603 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,4 +13,4 @@ ENV REPOFOLDER=/tmp # ENV LIBGUESTFS_TRACE=1 # RUN ./riasc-provisioning/rpi/create_image.sh -CMD ${REPOFOLDER}/rpi/create_image.sh \ No newline at end of file +CMD ${REPOFOLDER}/riasc/rpi/create_image.sh \ No newline at end of file diff --git a/README.md b/README.md index 8d9de87..787fd90 100644 --- a/README.md +++ b/README.md @@ -1,103 +1,87 @@ -# RIasC Provisioning Scripts +# Raspberry PI Image generation + +This project generates and customizes a Raspberry Pi image, either for Ubuntu or Raspberry Pi OS. The script is build for running within a docker container. + +The customizations include: +* Generating or adding a `vaultkey.secret` file +* Updating Ansible configuration in `/boot/firmware/riasc.yaml` +* Updating cloud-init file in `/boot/firmware/user-data` +* Setting GIT Ansible repor in `/boot/firmware/riasc.yaml` +* Adding and enabling GIT based Ansible updates on reboot ## Usage -switch to directory docker +1) Switch to direcory with Dockerfile to build image ``` docker build --tag "imagebuilder" . ``` -run docker run -``` -docker run \ ---volume ./rpi:/tmp/rpi \ ---volume ./common:/tmp/common \ ---volume ./out/output:/tmp/output \ ---volume ./out/images:/tmp/images \ ---volume ./out/download:/tmp/download \ ---env-file ./env \ -imagebuilder -``` -Create `env` file: +2) Create `env` file: ``` GIT_URL=https://mygiturl FLAVOR=ubuntu22.04 -REPOFOLDER=/tmp/ GIT_BRANCH=mybranch GIT_TOKEN=mytoken NODENAME=myhost +TAG=test ``` +3) Run docker container to generate image +``` +docker run \ +--volume ./:/tmp/riasc \ +--volume ./out/:/tmp/data \ +--env-file ./env \ +imagebuilder +``` +4) Image is placed in ´out/output´ folder +5) Copy image to SD card. Either using dd or the Raspberry Pi Imiger -List of available variables - -Optional varaibles - -Help -mount ubuntu-22.04.4-preinstalled-server-arm64+raspi.img -o loop,offset=$(( 512 * 526336)) /mnt/ -fdisk -lu ubuntu-22.04.4-preinstalled-server-arm64+raspi.img - -[![GitHub](https://img.shields.io/github/license/ERIGrid2/riasc-provisioning)](https://github.com/ERIGrid2/riasc-provisioning/blob/master/LICENSE) - -- **Based on:** - -## Introduction - -This fork of the RIasC Provisioning scripts is modified for the use with the edgePMU - -## Documentation - -For further documentation, please consult: https://riasc.eu/docs/ -## System requirements +## List of available variables +| Variable | Info | +| - | - | +|GIT_URL | URL to ansible git repository| +|FLAVOR | Falvor of os. See list of flavors| +|GIT_BRANCH | Branch used in ansible git pull| +|GIT_TOKEN | Token unsed in ansible git pull| +|NODENAME | The hostname of the device| +|TAG | A tag that is added to the name| +|RAW_OUTPOUT | Set to yes to get the .img file as output| +|TOKEN | A token used by Ansible| -The scripts have been tested with the following operating systems: +### List of flavors -- Ubuntu 20.01 +ubuntu22.04 -## Initial Setup -Before using this script, you will have to make sure that: -1. The referenced git repositories in `update_image.sh` exist and you have sufficient access rights. -2. Your ansible inventory is located at `{REPO}/inventory/edgeflex` -3. You have created the host_vars directory in your inventory -4. The password repository contains an initialized [*PASSWORD_STORE*](https://www.passwordstore.org/) with the subdirectories `keys` and `old` +ubuntu20.04 -## Usage +raspios -### 1. Creating an Image -See: https://riasc.eu/docs/setup/agent/manual -### 2. Updating an Image -Before flashing the created image, the `update_image.sh` script will write and update the necessary configuration files to the boot partition of the image. +# Help -Additionally, some of the configuration values are written into a git repository. +## How to mount the generated image to check the content? -To run the `update_image.sh` script, execute the script as **root** an follow the usage guide. +Check for the partitions in the image file: -i.e. -``` -sudo ./update_image.sh -I PATH_TO_IMAGE_FROM_CREATE_IMAGE.SH -N edgepmuXX -B main -S ../../SSL/CERT -U your_git_username -P your_git_access_token -``` +`fdisk -lu ubuntu-22.04.4-preinstalled-server-arm64+raspi.img` -After the script has finished, the Image can be flashed to the Raspberry PI. +Run mount command. Make sure to update the offset (526336) for the correct value -**Warning:** Running this script will override (and backup) old credentials, etc +`mount ubuntu-22.04.4-preinstalled-server-arm64+raspi.img -o loop,offset=$(( 512 * 526336)) /mnt/` -### 3. Updating an edgePMU that is already flashed +## How to add my custom secrets file for ansible vaults? +Copy the file in `out/output`. and make sure that the name is NODENAME-vaultkey.secret -To update an edgePMU that is already flashed, run the `update_image.sh` script with the *-u* option. This will lead to the configuration files getting written to the image and temporary files for you to manually copy to the device in question via SCP. -You will need to copy: -1. The generated `vaultkey.secret` to `/boot/firmware/vaultkey.secret` -2. The updated `riasc.yaml` to `/boot/firmware/riasc.yaml` -3. The updated `user-data` to `/boot/firmware/user-data` -4. The `git token` to `boot/firmware/git_token.secret` -5. The updated `../common/riasc-update.sh` to `/usr/local/bin/riasc-update.sh` +[![GitHub](https://img.shields.io/github/license/ERIGrid2/riasc-provisioning)](https://github.com/ERIGrid2/riasc-provisioning/blob/master/LICENSE) -Other data such as new SNMP credentials or new vpn configuration files can be distributed via the pmu-ansible repo. ## Credits - [Steffen Vogel](https://github.com/stv0g) [📧](mailto:post@steffenvogel.de), [Institute for Automation of Complex Power Systems](https://www.acs.eonerc.rwth-aachen.de), [RWTH Aachen University](https://www.rwth-aachen.de) +- [Vincent Bareiß]() [📧](mailto:), [Institute for Automation of Complex Power Systems](https://www.acs.eonerc.rwth-aachen.de), [RWTH Aachen University](https://www.rwth-aachen.de) +- [Manuel Pitz](https://https://github.com/windrad6) [📧](mailto:post@cl0.de), [Institute for Automation of Complex Power Systems](https://www.acs.eonerc.rwth-aachen.de), [RWTH Aachen University](https://www.rwth-aachen.de) ### Funding acknowledment diff --git a/rpi/create_image.sh b/rpi/create_image.sh index 36f28bf..832fcfc 100755 --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -9,14 +9,23 @@ cd "${SCRIPT_PATH}" NODENAME="${NODENAME:-riasc-agent}" TOKEN="${TOKEN:-XXXXX}" -DOWNLOAD_FOLDER="/tmp/download/" -OUTPUT_FOLDER="/tmp/output/" -IMG_FOLDER="/tmp/images/" +DOWNLOAD_FOLDER="/tmp/data/download/" +OUTPUT_FOLDER="/tmp/data/output/" +IMG_FOLDER="/tmp/data/images/" WORKDIR="/tmp/" -REPOFOLDER="${REPOFOLDER:-/tmp/}" - +REPOFOLDER="/tmp/riasc/" FLAVOR=${FLAVOR:-raspios} +if [ ! -d "$DOWNLOAD_FOLDER" ]; then + mkdir ${DOWNLOAD_FOLDER} +fi +if [ ! -d "$OUTPUT_FOLDER" ]; then + mkdir ${OUTPUT_FOLDER} +fi +if [ ! -d "$IMG_FOLDER" ]; then + mkdir ${IMG_FOLDER} +fi + case ${FLAVOR} in ubuntu22.04) OS="ubuntu" @@ -53,7 +62,7 @@ case ${FLAVOR} in ;; esac -RIASC_IMAGE_FILE="$(date +%Y-%m-%d)-riasc-${OS}" +RIASC_IMAGE_FILE="$(date +%Y-%m-%d)-riasc-${NODENAME}${TAG}" function check_command() { if ! command -v "$1" &> /dev/null; then @@ -66,7 +75,6 @@ function check_command() { echo "Using hostname: ${NODENAME}" echo "Using token: ${TOKEN}" echo "Using flavor: ${FLAVOR}" -echo "Using temp folder: ${REPOFOLDER}" echo "Using repo: ${GIT_URL}" echo "Using branch: ${GIT_BRANCH}" echo "Using token: ${GIT_TOKEN}" @@ -129,6 +137,13 @@ sed -i \ -e "s/dummyHostname/${NODENAME}/g" \ riasc.yaml +if [ "${OS}" = "ubuntu" ]; then + cp "${REPOFOLDER}"/rpi/user-data ${WORKDIR}/ + sed -i \ + -e "s/dummyHostname/${NODENAME}/g" \ + user-data +fi + #Select branch if [[ -n ${GIT_BRANCH} ]]; then sed -i \ @@ -143,6 +158,21 @@ if [[ -n ${GIT_TOKEN} ]]; then riasc.yaml fi +#Generate ansible secret +if [ ! -f ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret ]; then + echo "Generate ansible secret" + VAULT_KEY=$(tr -dc A-Za-z0-9 ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret +#!/bin/bash +echo "${VAULT_KEY}" +EOF + chmod +x ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret +else + echo "Skip ansible secret generation use existing key" +fi +cp ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret ${WORKDIR}/vaultkey.secret + + # Prepare systemd-timesyncd config cat > fallback-ntp.conf <> patch.fish -copy-in ${REPOFOLDER}/rpi/user-data /boot +copy-in ${WORKDIR}/user-data /boot EOF ;; *) @@ -226,6 +257,10 @@ fi echo "Patching image with guestfish..." guestfish < patch.fish +if [ "${RAW_OUTPOUT}" = "yes" ]; then + echo "Copy raw image..." + cp "${RIASC_IMAGE_FILE}".img ${OUTPUT_FOLDER}/ +fi # Zip image echo "Zipping image..." diff --git a/rpi/user-data b/rpi/user-data index f61a044..bf161f2 100644 --- a/rpi/user-data +++ b/rpi/user-data @@ -22,7 +22,7 @@ ssh_pwauth: true locale: C.UTF-8 timezone: Europe/Berlin -hostname: exampleHost +hostname: dummyHostname ## Run arbitrary commands at rc.local like time runcmd: From b1614b3625c6aecbadeca269bb3496a9ca0e6b73 Mon Sep 17 00:00:00 2001 From: Manuel Pitz Date: Wed, 17 Apr 2024 20:46:22 +0200 Subject: [PATCH 29/32] remove git token. Now part of the git url --- README.md | 2 -- rpi/create_image.sh | 11 ++++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 787fd90..294e72e 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ docker build --tag "imagebuilder" . GIT_URL=https://mygiturl FLAVOR=ubuntu22.04 GIT_BRANCH=mybranch -GIT_TOKEN=mytoken NODENAME=myhost TAG=test ``` @@ -43,7 +42,6 @@ imagebuilder |GIT_URL | URL to ansible git repository| |FLAVOR | Falvor of os. See list of flavors| |GIT_BRANCH | Branch used in ansible git pull| -|GIT_TOKEN | Token unsed in ansible git pull| |NODENAME | The hostname of the device| |TAG | A tag that is added to the name| |RAW_OUTPOUT | Set to yes to get the .img file as output| diff --git a/rpi/create_image.sh b/rpi/create_image.sh index 832fcfc..8213988 100755 --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -77,7 +77,6 @@ echo "Using token: ${TOKEN}" echo "Using flavor: ${FLAVOR}" echo "Using repo: ${GIT_URL}" echo "Using branch: ${GIT_BRANCH}" -echo "Using token: ${GIT_TOKEN}" # Check that required commands exist echo "Check if required commands are installed..." @@ -151,12 +150,10 @@ if [[ -n ${GIT_BRANCH} ]]; then riasc.yaml fi -#Git token -if [[ -n ${GIT_TOKEN} ]]; then - sed -i \ - -e "s,dummyGitUrl,riasc:${GIT_TOKEN}@${GIT_URL},g" \ - riasc.yaml -fi +#Git url +sed -i \ + -e "s,dummyGitUrl,${GIT_URL},g" \ + riasc.yaml #Generate ansible secret if [ ! -f ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret ]; then From 69d3da0bc222c2e11921d931d5a6c39864e3f5a7 Mon Sep 17 00:00:00 2001 From: Calvin Katt Date: Tue, 30 Apr 2024 18:12:17 +0200 Subject: [PATCH 30/32] Allow generation of images based on ubuntu 24.04 --- rpi/create_image.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rpi/create_image.sh b/rpi/create_image.sh index 8213988..6c2b814 100755 --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -27,6 +27,9 @@ if [ ! -d "$IMG_FOLDER" ]; then fi case ${FLAVOR} in + ubuntu24.04) + OS="ubuntu" + ;; ubuntu22.04) OS="ubuntu" ;; @@ -55,6 +58,12 @@ case ${FLAVOR} in IMAGE_URL="https://cdimage.ubuntu.com/releases/22.04/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" ;; + ubuntu24.04) + IMAGE_FILE="ubuntu-24.04-preinstalled-server-arm64+raspi" + IMAGE_SUFFIX="img.xz" + IMAGE_URL="https://cdimage.ubuntu.com/releases/24.04/release/${IMAGE_FILE}.${IMAGE_SUFFIX}" + ;; + raspios) IMAGE_FILE="2021-05-07-raspios-buster-armhf-lite" IMAGE_SUFFIX="zip" From 0535c6bed70003335a8fed3bcf2880d6d6ca78cf Mon Sep 17 00:00:00 2001 From: Calvin Katt Date: Mon, 27 May 2024 15:28:06 +0200 Subject: [PATCH 31/32] Added support for a VAULT_KEY env variable --- rpi/create_image.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rpi/create_image.sh b/rpi/create_image.sh index 6c2b814..8c94d5c 100755 --- a/rpi/create_image.sh +++ b/rpi/create_image.sh @@ -86,6 +86,9 @@ echo "Using token: ${TOKEN}" echo "Using flavor: ${FLAVOR}" echo "Using repo: ${GIT_URL}" echo "Using branch: ${GIT_BRANCH}" +if [! -z "$VAULT_KEY" ]; then + echo "Using ansible secret ${VAULT_KEY}" +fi # Check that required commands exist echo "Check if required commands are installed..." @@ -166,8 +169,10 @@ sed -i \ #Generate ansible secret if [ ! -f ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret ]; then - echo "Generate ansible secret" - VAULT_KEY=$(tr -dc A-Za-z0-9 ${OUTPUT_FOLDER}/"${NODENAME}"-vaultkey.secret #!/bin/bash echo "${VAULT_KEY}" From b684ce2adbbae61cd77b7b0d71fd6cc8b68d452d Mon Sep 17 00:00:00 2001 From: Calvin Katt Date: Mon, 27 May 2024 15:43:15 +0200 Subject: [PATCH 32/32] Added info about VAULT_KEY variable to README.md and fixed the flavor list --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 294e72e..b76c9a5 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,12 @@ imagebuilder |TAG | A tag that is added to the name| |RAW_OUTPOUT | Set to yes to get the .img file as output| |TOKEN | A token used by Ansible| +|VAULT_KEY | Key to use in the vaultkey.secret file| ### List of flavors +ubuntu24.04 + ubuntu22.04 ubuntu20.04 @@ -69,8 +72,8 @@ Run mount command. Make sure to update the offset (526336) for the correct value `mount ubuntu-22.04.4-preinstalled-server-arm64+raspi.img -o loop,offset=$(( 512 * 526336)) /mnt/` ## How to add my custom secrets file for ansible vaults? -Copy the file in `out/output`. and make sure that the name is NODENAME-vaultkey.secret +To use a custom secret the VAULT_KEY variable can be set. If a vaultkey file of the name NODENAME-vaultkey.secret already exists the variable will be ignored. [![GitHub](https://img.shields.io/github/license/ERIGrid2/riasc-provisioning)](https://github.com/ERIGrid2/riasc-provisioning/blob/master/LICENSE)