Update dependencies and merge from upstream

.gitignore

@@ -6,7 +6,6 @@ SKIP
 SKIP_IMAGES
 .pc
 *-pc
-apt-cacher-ng/
 
 
 #

.gitlab-ci.yml

@@ -0,0 +1,4 @@
+include:
+  - project: serge/pi-gen
+    ref: ci
+    file: 'pi-gen.yml'

Dockerfile

@@ -7,8 +7,8 @@ RUN apt-get -y update && \
     apt-get -y install --no-install-recommends \
         git vim parted \
         quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
-        libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\
-        binfmt-support ca-certificates qemu-utils kpartx fdisk gpg pigz\
+        libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc \
+        binfmt-support ca-certificates fdisk gpg pigz arch-test \
     && rm -rf /var/lib/apt/lists/*
 
 COPY . /pi-gen/

GL-README.md

@@ -13,7 +13,7 @@ There are several different images available, depending on your needs, from smal
 
 ## Source Code
 
-This build system is based on [pi-gen](https://github.com/RPi-Distro/pi-gen).  Refer to its [original README](/README.md) for how everything works.  The (`gl-config`)[gl-config] file is the key source of control.  (What is called "config" in the original.)
+This build system is based on [pi-gen](https://github.com/RPi-Distro/pi-gen).  Refer to its [original README](/README.md) for how everything works.  The [`gl-config`](gl-config) file is the key source of control.  (What is called "config" in the original.)
 
 Note that we're tracking the `arm64` branch, not main.  (If we build off the main branch, we hit [an issue with missing `arm/v8` docker images](https://github.com/groundlight/monitoring-notification-server/issues/39) and likely others, because we make these funky machines with a 64-bit kernel, but 32-bit applications.)
 

build-docker.sh

@@ -141,9 +141,6 @@ time ${DOCKER} run \
   $DOCKER_CMDLINE_PRE \
   --name "${DOCKER_CMDLINE_NAME}" \
   --privileged \
-  --cap-add=ALL \
-  -v /dev:/dev \
-  -v /lib/modules:/lib/modules \
   ${PIGEN_DOCKER_OPTS} \
   --volume "${CONFIG_FILE}":/config:ro \
   -e "GIT_HASH=${GIT_HASH}" \

build.sh

@@ -24,11 +24,6 @@ EOF
 				on_chroot << EOF
 apt-get -o Acquire::Retries=3 install --no-install-recommends -y $PACKAGES
 EOF
-				if [ "${USE_QCOW2}" = "1" ]; then
-					on_chroot << EOF
-apt-get clean
-EOF
-				fi
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages-nr"
 		fi
@@ -39,11 +34,6 @@ EOF
 				on_chroot << EOF
 apt-get -o Acquire::Retries=3 install -y $PACKAGES
 EOF
-				if [ "${USE_QCOW2}" = "1" ]; then
-					on_chroot << EOF
-apt-get clean
-EOF
-				fi
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages"
 		fi
@@ -102,24 +92,15 @@ run_stage(){
 	STAGE_WORK_DIR="${WORK_DIR}/${STAGE}"
 	ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs
 
-	if [ "${USE_QCOW2}" = "1" ]; then
-		if [ ! -f SKIP ]; then
-			load_qimage
-		fi
-	else
-		# make sure we are not umounting during export-image stage
-		if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
-			unmount "${WORK_DIR}/${STAGE}"
-		fi
-	fi
+	unmount "${WORK_DIR}/${STAGE}"
 
 	if [ ! -f SKIP_IMAGES ]; then
 		if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then
 			EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
 		fi
 	fi
 	if [ ! -f SKIP ]; then
-		if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then
+		if [ "${CLEAN}" = "1" ]; then
 			if [ -d "${ROOTFS_DIR}" ]; then
 				rm -rf "${ROOTFS_DIR}"
 			fi
@@ -136,14 +117,7 @@ run_stage(){
 		done
 	fi
 
-	if [ "${USE_QCOW2}" = "1" ]; then
-		unload_qimage
-	else
-		# make sure we are not umounting during export-image stage
-		if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
-			unmount "${WORK_DIR}/${STAGE}"
-		fi
-	fi
+	unmount "${WORK_DIR}/${STAGE}"
 
 	PREV_STAGE="${STAGE}"
 	PREV_STAGE_DIR="${STAGE_DIR}"
@@ -187,9 +161,16 @@ do
 done
 
 term() {
-	if [ "${USE_QCOW2}" = "1" ]; then
-		log "Unloading image"
-		unload_qimage
+	if [ "$?" -ne 0 ]; then
+		log "Build failed"
+	else
+		log "Build finished"
+	fi
+	unmount "${STAGE_WORK_DIR}"
+	if [ "$STAGE" = "export-image" ]; then
+		for img in "${STAGE_WORK_DIR}/"*.img; do
+			unmount_image "$img"
+		done
 	fi
 }
 
@@ -199,10 +180,9 @@ export PI_GEN=${PI_GEN:-pi-gen}
 export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen}
 export PI_GEN_RELEASE=${PI_GEN_RELEASE:-Raspberry Pi reference}
 
-if [ -z "${IMG_NAME}" ]; then
-	echo "IMG_NAME not set" 1>&2
-	exit 1
-fi
+export ARCH=arm64
+export RELEASE=${RELEASE:-bookworm} # Don't forget to update stage0/prerun.sh
+export IMG_NAME="${IMG_NAME:-raspios-$RELEASE-$ARCH}"
 
 export USE_QEMU="${USE_QEMU:-0}"
 export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}"
@@ -230,7 +210,6 @@ export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi}
 export FIRST_USER_NAME=${FIRST_USER_NAME:-pi}
 export FIRST_USER_PASS
 export DISABLE_FIRST_BOOT_USER_RENAME=${DISABLE_FIRST_BOOT_USER_RENAME:-0}
-export RELEASE=${RELEASE:-bookworm} # Don't forget to update stage0/prerun.sh
 export WPA_COUNTRY
 export ENABLE_SSH="${ENABLE_SSH:-0}"
 export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}"
@@ -247,7 +226,6 @@ export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"}
 export PUBKEY_SSH_FIRST_USER
 
 export CLEAN
-export IMG_NAME
 export APT_PROXY
 
 export STAGE
@@ -273,24 +251,21 @@ source "${SCRIPT_DIR}/common"
 # shellcheck source=scripts/dependencies_check
 source "${SCRIPT_DIR}/dependencies_check"
 
-export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
-export USE_QCOW2="${USE_QCOW2:-0}"
-export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G}
-source "${SCRIPT_DIR}/qcow2_handling"
-if [ "${USE_QCOW2}" = "1" ]; then
-	NO_PRERUN_QCOW2=1
-else
-	NO_PRERUN_QCOW2=0
-fi
-
-export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
-
 if [ "$SETFCAP" != "1" ]; then
 	export CAPSH_ARG="--drop=cap_setfcap"
 fi
 
 dependencies_check "${BASE_DIR}/depends"
 
+echo "Checking native $ARCH executable support..."
+if ! arch-test -n "$ARCH"; then
+	echo "WARNING: Only a native build environment is supported. Checking emulated support..."
+	if ! arch-test "$ARCH"; then
+		echo "No fallback mechanism found. Ensure your OS has binfmt_misc support enabled and configured."
+		exit 1
+	fi
+fi
+
 #check username is valid
 if [[ ! "$FIRST_USER_NAME" =~ ^[a-z][-a-z0-9_]*$ ]]; then
 	echo "Invalid FIRST_USER_NAME: $FIRST_USER_NAME"
@@ -327,6 +302,7 @@ mkdir -p "${WORK_DIR}"
 log "Begin ${BASE_DIR}"
 
 STAGE_LIST=${STAGE_LIST:-${BASE_DIR}/stage*}
+export STAGE_LIST
 
 for STAGE_DIR in $STAGE_LIST; do
 	STAGE_DIR=$(realpath "${STAGE_DIR}")
@@ -339,98 +315,22 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
 	# shellcheck source=/dev/null
 	source "${EXPORT_DIR}/EXPORT_IMAGE"
 	EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs
-	if [ "${USE_QCOW2}" = "1" ]; then
-		USE_QCOW2=0
-		EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}"
-		echo "------------------------------------------------------------------------"
-		echo "Running export stage for ${EXPORT_NAME}"
-		rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true
-		rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true
-		rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true
-		rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true
-		EXPORT_STAGE=$(basename "${EXPORT_DIR}")
-		for s in $STAGE_LIST; do
-			TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}")
-		done
-		FIRST_STAGE=${TMP_LIST%% *}
-		FIRST_IMAGE="image-${FIRST_STAGE}.qcow2"
-
-		pushd "${WORK_DIR}" > /dev/null
-		echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}"
-		cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2"
-
-		ARR=($TMP_LIST)
-		# rebase stage images to new export base
-		for CURR_STAGE in "${ARR[@]}"; do
-			if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
-				PREV_IMG="${EXPORT_NAME}"
-				continue
-			fi
-		echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
-			qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
-			if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
-				break
-			fi
-			PREV_IMG="image-${CURR_STAGE}"
-		done
-
-		# commit current export stage into base export image
-		echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2"
-		qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2
-
-		# rebase stage images back to original first stage for easy re-run
-		for CURR_STAGE in "${ARR[@]}"; do
-			if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
-				PREV_IMG="image-${CURR_STAGE}"
-				continue
-			fi
-		echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
-			qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
-			if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
-				break
-			fi
-			PREV_IMG="image-${CURR_STAGE}"
-		done
-		popd > /dev/null
-
-		mkdir -p "${WORK_DIR}/export-image/rootfs"
-		mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/"
-		echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs"
-		mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs"
-
-		CLEAN=0
-		run_stage
-		CLEAN=1
-		USE_QCOW2=1
-
-	else
-		run_stage
-	fi
+	run_stage
 	if [ "${USE_QEMU}" != "1" ]; then
 		if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then
 			# shellcheck source=/dev/null
 			source "${EXPORT_DIR}/EXPORT_NOOBS"
 			STAGE_DIR="${BASE_DIR}/export-noobs"
-			if [ "${USE_QCOW2}" = "1" ]; then
-				USE_QCOW2=0
-				run_stage
-				USE_QCOW2=1
-			else
-				run_stage
-			fi
+			run_stage
 		fi
 	fi
 done
 
-if [ -x postrun.sh ]; then
+if [ -x "${BASE_DIR}/postrun.sh" ]; then
 	log "Begin postrun.sh"
 	cd "${BASE_DIR}"
 	./postrun.sh
 	log "End postrun.sh"
 fi
 
-if [ "${USE_QCOW2}" = "1" ]; then
-	unload_qimage
-fi
-
 log "End ${BASE_DIR}"

depends

@@ -17,7 +17,6 @@ file
 git
 lsmod:kmod
 bc
-qemu-nbd:qemu-utils
-kpartx
 gpg
 pigz
+arch-test

export-image/04-set-partuuid/00-run.sh

@@ -1,17 +1,13 @@
 #!/bin/bash -e
 
-if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
+IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 
-	IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
+IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
 
-	IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
+BOOT_PARTUUID="${IMGID}-01"
+ROOT_PARTUUID="${IMGID}-02"
 
-	BOOT_PARTUUID="${IMGID}-01"
-	ROOT_PARTUUID="${IMGID}-02"
-
-	sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
-	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
-
-	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/firmware/cmdline.txt"
-fi
+sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 
+sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/firmware/cmdline.txt"

export-image/05-finalise/01-run.sh

@@ -83,24 +83,17 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE"
 	dpkg -l --root "$ROOTFS_DIR"
 } >> "$INFO_FILE"
 
-mkdir -p "${DEPLOY_DIR}"
+ROOT_DEV="$(awk "\$2 == \"${ROOTFS_DIR}\" {print \$1}" /etc/mtab)"
 
-rm -f "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.*"
-rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
+unmount "${ROOTFS_DIR}"
+zerofree "${ROOT_DEV}"
 
-mv "$INFO_FILE" "$DEPLOY_DIR/"
+unmount_image "${IMG_FILE}"
 
-if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
-	ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')"
-
-	unmount "${ROOTFS_DIR}"
-	zerofree "${ROOT_DEV}"
+mkdir -p "${DEPLOY_DIR}"
 
-	unmount_image "${IMG_FILE}"
-else
-	unload_qimage
-	make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE"
-fi
+rm -f "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.*"
+rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 
 case "${DEPLOY_COMPRESSION}" in
 zip)
@@ -121,3 +114,5 @@ none | *)
 	cp "$IMG_FILE" "$DEPLOY_DIR/"
 ;;
 esac
+
+cp "$INFO_FILE" "$DEPLOY_DIR/"