Skip to content

Commit

Permalink
Merge from upstream 2024-07
Browse files Browse the repository at this point in the history
  • Loading branch information
f-wright authored and robotrapta committed Jul 18, 2024
2 parents dc4042c + 48efb5f commit c53e72e
Show file tree
Hide file tree
Showing 27 changed files with 277 additions and 775 deletions.
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ SKIP
SKIP_IMAGES
.pc
*-pc
apt-cacher-ng/


#
Expand Down
4 changes: 4 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
include:
- project: serge/pi-gen
ref: ci
file: 'pi-gen.yml'
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ RUN apt-get -y update && \
apt-get -y install --no-install-recommends \
git vim parted \
quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\
binfmt-support ca-certificates qemu-utils kpartx fdisk gpg pigz\
libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc \
binfmt-support ca-certificates fdisk gpg pigz arch-test \
&& rm -rf /var/lib/apt/lists/*

COPY . /pi-gen/
Expand Down
190 changes: 46 additions & 144 deletions README.md

Large diffs are not rendered by default.

3 changes: 0 additions & 3 deletions build-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -141,9 +141,6 @@ time ${DOCKER} run \
$DOCKER_CMDLINE_PRE \
--name "${DOCKER_CMDLINE_NAME}" \
--privileged \
--cap-add=ALL \
-v /dev:/dev \
-v /lib/modules:/lib/modules \
${PIGEN_DOCKER_OPTS} \
--volume "${CONFIG_FILE}":/config:ro \
-e "GIT_HASH=${GIT_HASH}" \
Expand Down
158 changes: 29 additions & 129 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,6 @@ EOF
on_chroot << EOF
apt-get -o Acquire::Retries=3 install --no-install-recommends -y $PACKAGES
EOF
if [ "${USE_QCOW2}" = "1" ]; then
on_chroot << EOF
apt-get clean
EOF
fi
fi
log "End ${SUB_STAGE_DIR}/${i}-packages-nr"
fi
Expand All @@ -39,11 +34,6 @@ EOF
on_chroot << EOF
apt-get -o Acquire::Retries=3 install -y $PACKAGES
EOF
if [ "${USE_QCOW2}" = "1" ]; then
on_chroot << EOF
apt-get clean
EOF
fi
fi
log "End ${SUB_STAGE_DIR}/${i}-packages"
fi
Expand Down Expand Up @@ -102,24 +92,15 @@ run_stage(){
STAGE_WORK_DIR="${WORK_DIR}/${STAGE}"
ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs

if [ "${USE_QCOW2}" = "1" ]; then
if [ ! -f SKIP ]; then
load_qimage
fi
else
# make sure we are not umounting during export-image stage
if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
unmount "${WORK_DIR}/${STAGE}"
fi
fi
unmount "${WORK_DIR}/${STAGE}"

if [ ! -f SKIP_IMAGES ]; then
if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then
EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
fi
fi
if [ ! -f SKIP ]; then
if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then
if [ "${CLEAN}" = "1" ]; then
if [ -d "${ROOTFS_DIR}" ]; then
rm -rf "${ROOTFS_DIR}"
fi
Expand All @@ -136,14 +117,7 @@ run_stage(){
done
fi

if [ "${USE_QCOW2}" = "1" ]; then
unload_qimage
else
# make sure we are not umounting during export-image stage
if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
unmount "${WORK_DIR}/${STAGE}"
fi
fi
unmount "${WORK_DIR}/${STAGE}"

PREV_STAGE="${STAGE}"
PREV_STAGE_DIR="${STAGE_DIR}"
Expand Down Expand Up @@ -187,9 +161,16 @@ do
done

term() {
if [ "${USE_QCOW2}" = "1" ]; then
log "Unloading image"
unload_qimage
if [ "$?" -ne 0 ]; then
log "Build failed"
else
log "Build finished"
fi
unmount "${STAGE_WORK_DIR}"
if [ "$STAGE" = "export-image" ]; then
for img in "${STAGE_WORK_DIR}/"*.img; do
unmount_image "$img"
done
fi
}

Expand All @@ -199,10 +180,9 @@ export PI_GEN=${PI_GEN:-pi-gen}
export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen}
export PI_GEN_RELEASE=${PI_GEN_RELEASE:-Raspberry Pi reference}

if [ -z "${IMG_NAME}" ]; then
echo "IMG_NAME not set" 1>&2
exit 1
fi
export ARCH=arm64
export RELEASE=${RELEASE:-bookworm} # Don't forget to update stage0/prerun.sh
export IMG_NAME="${IMG_NAME:-raspios-$RELEASE-$ARCH}"

export USE_QEMU="${USE_QEMU:-0}"
export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}"
Expand Down Expand Up @@ -230,7 +210,6 @@ export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi}
export FIRST_USER_NAME=${FIRST_USER_NAME:-pi}
export FIRST_USER_PASS
export DISABLE_FIRST_BOOT_USER_RENAME=${DISABLE_FIRST_BOOT_USER_RENAME:-0}
export RELEASE=${RELEASE:-bookworm} # Don't forget to update stage0/prerun.sh
export WPA_COUNTRY
export ENABLE_SSH="${ENABLE_SSH:-0}"
export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}"
Expand All @@ -247,7 +226,6 @@ export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"}
export PUBKEY_SSH_FIRST_USER

export CLEAN
export IMG_NAME
export APT_PROXY

export STAGE
Expand All @@ -273,24 +251,21 @@ source "${SCRIPT_DIR}/common"
# shellcheck source=scripts/dependencies_check
source "${SCRIPT_DIR}/dependencies_check"

export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
export USE_QCOW2="${USE_QCOW2:-0}"
export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G}
source "${SCRIPT_DIR}/qcow2_handling"
if [ "${USE_QCOW2}" = "1" ]; then
NO_PRERUN_QCOW2=1
else
NO_PRERUN_QCOW2=0
fi

export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"

if [ "$SETFCAP" != "1" ]; then
export CAPSH_ARG="--drop=cap_setfcap"
fi

dependencies_check "${BASE_DIR}/depends"

echo "Checking native $ARCH executable support..."
if ! arch-test -n "$ARCH"; then
echo "WARNING: Only a native build environment is supported. Checking emulated support..."
if ! arch-test "$ARCH"; then
echo "No fallback mechanism found. Ensure your OS has binfmt_misc support enabled and configured."
exit 1
fi
fi

#check username is valid
if [[ ! "$FIRST_USER_NAME" =~ ^[a-z][-a-z0-9_]*$ ]]; then
echo "Invalid FIRST_USER_NAME: $FIRST_USER_NAME"
Expand Down Expand Up @@ -327,6 +302,7 @@ mkdir -p "${WORK_DIR}"
log "Begin ${BASE_DIR}"

STAGE_LIST=${STAGE_LIST:-${BASE_DIR}/stage*}
export STAGE_LIST

for STAGE_DIR in $STAGE_LIST; do
STAGE_DIR=$(realpath "${STAGE_DIR}")
Expand All @@ -339,98 +315,22 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
# shellcheck source=/dev/null
source "${EXPORT_DIR}/EXPORT_IMAGE"
EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs
if [ "${USE_QCOW2}" = "1" ]; then
USE_QCOW2=0
EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}"
echo "------------------------------------------------------------------------"
echo "Running export stage for ${EXPORT_NAME}"
rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true
rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true
rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true
rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true
EXPORT_STAGE=$(basename "${EXPORT_DIR}")
for s in $STAGE_LIST; do
TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}")
done
FIRST_STAGE=${TMP_LIST%% *}
FIRST_IMAGE="image-${FIRST_STAGE}.qcow2"

pushd "${WORK_DIR}" > /dev/null
echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}"
cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2"

ARR=($TMP_LIST)
# rebase stage images to new export base
for CURR_STAGE in "${ARR[@]}"; do
if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
PREV_IMG="${EXPORT_NAME}"
continue
fi
echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
break
fi
PREV_IMG="image-${CURR_STAGE}"
done

# commit current export stage into base export image
echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2"
qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2

# rebase stage images back to original first stage for easy re-run
for CURR_STAGE in "${ARR[@]}"; do
if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
PREV_IMG="image-${CURR_STAGE}"
continue
fi
echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
break
fi
PREV_IMG="image-${CURR_STAGE}"
done
popd > /dev/null

mkdir -p "${WORK_DIR}/export-image/rootfs"
mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/"
echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs"
mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs"

CLEAN=0
run_stage
CLEAN=1
USE_QCOW2=1

else
run_stage
fi
run_stage
if [ "${USE_QEMU}" != "1" ]; then
if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then
# shellcheck source=/dev/null
source "${EXPORT_DIR}/EXPORT_NOOBS"
STAGE_DIR="${BASE_DIR}/export-noobs"
if [ "${USE_QCOW2}" = "1" ]; then
USE_QCOW2=0
run_stage
USE_QCOW2=1
else
run_stage
fi
run_stage
fi
fi
done

if [ -x postrun.sh ]; then
if [ -x "${BASE_DIR}/postrun.sh" ]; then
log "Begin postrun.sh"
cd "${BASE_DIR}"
./postrun.sh
log "End postrun.sh"
fi

if [ "${USE_QCOW2}" = "1" ]; then
unload_qimage
fi

log "End ${BASE_DIR}"
3 changes: 1 addition & 2 deletions depends
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ file
git
lsmod:kmod
bc
qemu-nbd:qemu-utils
kpartx
gpg
pigz
arch-test
10 changes: 0 additions & 10 deletions docker-compose.yml

This file was deleted.

18 changes: 7 additions & 11 deletions export-image/04-set-partuuid/00-run.sh
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
#!/bin/bash -e

if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"

IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"

IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
BOOT_PARTUUID="${IMGID}-01"
ROOT_PARTUUID="${IMGID}-02"

BOOT_PARTUUID="${IMGID}-01"
ROOT_PARTUUID="${IMGID}-02"

sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"

sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/firmware/cmdline.txt"
fi
sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"

sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/firmware/cmdline.txt"
23 changes: 9 additions & 14 deletions export-image/05-finalise/01-run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,24 +83,17 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE"
dpkg -l --root "$ROOTFS_DIR"
} >> "$INFO_FILE"

mkdir -p "${DEPLOY_DIR}"
ROOT_DEV="$(awk "\$2 == \"${ROOTFS_DIR}\" {print \$1}" /etc/mtab)"

rm -f "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.*"
rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
unmount "${ROOTFS_DIR}"
zerofree "${ROOT_DEV}"

mv "$INFO_FILE" "$DEPLOY_DIR/"
unmount_image "${IMG_FILE}"

if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')"

unmount "${ROOTFS_DIR}"
zerofree "${ROOT_DEV}"
mkdir -p "${DEPLOY_DIR}"

unmount_image "${IMG_FILE}"
else
unload_qimage
make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE"
fi
rm -f "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.*"
rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"

case "${DEPLOY_COMPRESSION}" in
zip)
Expand All @@ -121,3 +114,5 @@ none | *)
cp "$IMG_FILE" "$DEPLOY_DIR/"
;;
esac

cp "$INFO_FILE" "$DEPLOY_DIR/"
Loading

0 comments on commit c53e72e

Please sign in to comment.