diff --git a/.github/workflows/integration_delivery.yml b/.github/workflows/integration_delivery.yml index 4e0ff8de..2d606f2b 100644 --- a/.github/workflows/integration_delivery.yml +++ b/.github/workflows/integration_delivery.yml @@ -167,8 +167,8 @@ jobs: - dependencies runs-on: ubuntu-latest outputs: - version: ${{ steps.extract_version.outputs.VERSION }} - name: ${{ steps.extract_version.outputs.NAME }} + version: ${{ steps.extract-version.outputs.VERSION }} + name: ${{ steps.extract-version.outputs.NAME }} steps: - uses: actions/checkout@v4 name: Checkout @@ -191,7 +191,7 @@ jobs: key: poetry-${{ hashFiles('poetry.lock') }}-${{ env.PYTHON_VERSION}} - name: Extract Version - id: extract_version + id: extract-version run: | echo "VERSION=$(poetry version --short)" >> "$GITHUB_OUTPUT" echo "NAME=$(poetry version | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" @@ -199,11 +199,10 @@ jobs: echo "NAME=$(poetry version | cut -d' ' -f1)" - name: Extract Version from CHANGELOG.md - id: extract_changelog_version run: | VERSION_CHANGELOG=$(sed -n '3 s/## Version //p' CHANGELOG.md) echo "VERSION_CHANGELOG=$VERSION_CHANGELOG" - if [ "${{ steps.extract_version.outputs.VERSION }}" != "$VERSION_CHANGELOG" ]; then + if [ "${{ steps.extract-version.outputs.VERSION }}" != "$VERSION_CHANGELOG" ]; then echo "Error: Version extracted from CHANGELOG.md does not match the version in pyproject.toml" exit 1 else @@ -212,11 +211,10 @@ jobs: - name: Extract Version from Tag if: startsWith(github.ref, 'refs/tags/v') - id: extract_tag_version run: | VERSION_TAG=$(sed 's/^v//' <<< ${{ github.ref_name }}) echo "VERSION_TAG=$VERSION_TAG" - if [ "${{ steps.extract_version.outputs.VERSION }}" != "$VERSION_TAG" ]; then + if [ "${{ steps.extract-version.outputs.VERSION }}" != "$VERSION_TAG" ]; then echo "Error: Version extracted from tag does not match the version in pyproject.toml" exit 1 else @@ -228,7 +226,7 @@ jobs: echo "SENTRY_DSN=${{ secrets.SENTRY_DSN }}" >> ubo_app/.env # conditionally set it based on whether it's a tag or not using github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') if [ "${{ github.event_name }}" == "push" ] && [ "$(echo ${{ github.ref }} | grep -c 'refs/tags/v')" -eq 1 ]; then - echo "SENTRY_RELEASE=ubo-app@${{ steps.extract_version.outputs.VERSION }}" >> ubo_app/.env + echo "SENTRY_RELEASE=ubo-app@${{ steps.extract-version.outputs.VERSION }}" >> ubo_app/.env else echo "SENTRY_RELEASE=ubo-app@${{ github.sha }}" >> ubo_app/.env fi @@ -260,9 +258,7 @@ jobs: - build runs-on: ubuntu-latest container: - image: ghcr.io/solo-io/packer-plugin-arm-image - env: - PACKER_CACHE_DIR: /build/packer_cache + image: mkaczanowski/packer-builder-arm volumes: - /dev:/dev options: --rm --privileged @@ -287,7 +283,7 @@ jobs: path: /build/dist - name: Generate Image URL and Checksum - id: generate_image_url + id: generate-image-url run: | if [ -n "${{ matrix.suffix }}" ]; then SUFFIX="_${{ matrix.suffix }}" @@ -296,46 +292,52 @@ jobs: fi DASHED_SUFFIX=$(echo $SUFFIX | sed 's/_/-/g') IMAGE_URL="https://downloads.raspberrypi.com/raspios${SUFFIX}_arm64/images/raspios${SUFFIX}_arm64-2024-03-15/2024-03-15-raspios-bookworm-arm64${DASHED_SUFFIX}.img.xz" - IMAGE_CHECKSUM=$(curl -s "${IMAGE_URL}.sha256" | awk '{print $1}') + CHECKSUM_URL="${IMAGE_URL}.sha256" echo "suffix=$SUFFIX" >> "$GITHUB_OUTPUT" echo "dashed_suffix=$DASHED_SUFFIX" >> "$GITHUB_OUTPUT" echo "image_url=$IMAGE_URL" >> "$GITHUB_OUTPUT" - echo "image_checksum=$IMAGE_CHECKSUM" >> "$GITHUB_OUTPUT" + echo "image_checksum_url=$CHECKSUM_URL" >> "$GITHUB_OUTPUT" + IMAGE_SIZE_GB=${{ matrix.suffix == 'lite' && '3.75' || matrix.suffix == '' && '6' || '13' }} + IMAGE_SIZE=$(awk -v IMAGE_SIZE_GB=$IMAGE_SIZE_GB 'BEGIN {printf "%.0f", IMAGE_SIZE_GB * 1024 ^ 3}') + echo "image_size=$IMAGE_SIZE" >> "$GITHUB_OUTPUT" - name: Build Artifact env: PKR_VAR_ubo_app_version: ${{ needs.build.outputs.version }} - PKR_VAR_image_url: ${{ steps.generate_image_url.outputs.image_url }} - PKR_VAR_image_checksum: - ${{ steps.generate_image_url.outputs.image_checksum }} + PKR_VAR_image_url: ${{ steps.generate-image-url.outputs.image_url }} + PKR_VAR_image_checksum_url: + ${{ steps.generate-image-url.outputs.image_checksum_url }} PKR_VAR_target_image_size: - ${{ matrix.suffix == 'lite' && '4026531840' || matrix.suffix == '' - && '6174015488' || '0' }} + ${{ steps.generate-image-url.outputs.image_size}} run: | - /entrypoint.sh validate scripts/packer/image.pkr.hcl - /entrypoint.sh build scripts/packer/image.pkr.hcl + /entrypoint.sh init scripts/packer/image.pkr.hcl + SETUP_QEMU=false /entrypoint.sh build scripts/packer/image.pkr.hcl + ls -lh + mv image.img /build + ls -lh /build - - name: Write Zeroes to Free Space + - name: Fill Free Space with Zeros run: | - losetup -P /dev/loop3 /build/image.img - apt-get -y update + apt-get update apt-get install -y zerofree - zerofree /dev/loop3p2 - losetup -d /dev/loop3 + LOOP_DEV=$(losetup -f) + losetup -P $LOOP_DEV /build/image.img + zerofree -v "${LOOP_DEV}p2" + losetup -d $LOOP_DEV - name: Compress File with Gzip run: | - scripts/consume.sh /build/image.img | gzip -9 > /ubo_app-${{ needs.build.outputs.version }}-bookworm${{ steps.generate_image_url.outputs.dashed_suffix }}.img.gz + scripts/consume.sh /build/image.img | gzip -9 > /ubo_app-${{ needs.build.outputs.version }}-bookworm${{ steps.generate-image-url.outputs.dashed_suffix }}.img.gz - name: Upload Image uses: actions/upload-artifact@v4 with: name: ubo_app-${{ needs.build.outputs.version }}-bookworm${{ - steps.generate_image_url.outputs.dashed_suffix}}-arm64.img.gz + steps.generate-image-url.outputs.dashed_suffix}}-arm64.img.gz path: /ubo_app-${{ needs.build.outputs.version }}-bookworm${{ - steps.generate_image_url.outputs.dashed_suffix }}.img.gz + steps.generate-image-url.outputs.dashed_suffix }}.img.gz if-no-files-found: error pypi-publish: diff --git a/CHANGELOG.md b/CHANGELOG.md index ff58dcab..4051a037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ the audio work #53 - build(packer): mount the first partition of the image in `/boot/firmware` instead of `/boot` to be compatible with the new linux kernel +- ci(github): download and cache images as it is the slowest part of the build ## Version 0.12.6 diff --git a/scripts/packer/image-local.pkr.hcl b/scripts/packer/image-local.pkr.hcl index ec3c697d..e84ae46f 100644 --- a/scripts/packer/image-local.pkr.hcl +++ b/scripts/packer/image-local.pkr.hcl @@ -18,7 +18,7 @@ build { provisioner "shell" { inline = [ "chmod +x /install.sh", - "/install.sh --for-packer --source /build/wheel/ubo_app-${var.ubo_app_version}-py3-none-any.whl", + "/install.sh --in-packer --source /build/wheel/ubo_app-${var.ubo_app_version}-py3-none-any.whl", "rm /install.sh" ] } diff --git a/scripts/packer/image.pkr.hcl b/scripts/packer/image.pkr.hcl index 914f456d..554d38f4 100644 --- a/scripts/packer/image.pkr.hcl +++ b/scripts/packer/image.pkr.hcl @@ -6,25 +6,56 @@ variable "image_url" { type = string } -variable "image_checksum" { +variable "image_checksum_url" { type = string } variable "target_image_size" { - type = number - default = 0 + type = string +} + +packer { + required_plugins { + git = { + version = ">=v0.3.2" + source = "github.com/ethanmdavidson/git" + } + } } -source "arm-image" "raspberry_pi_os" { - iso_url = var.image_url - iso_checksum = var.image_checksum - output_filename = "/build/image.img" - target_image_size = var.target_image_size - image_mounts = ["/boot/firmware", "/"] +source "arm" "raspios" { + file_urls = [var.image_url] + file_checksum_url = var.image_checksum_url + file_checksum_type = "sha256" + file_target_extension = "xz" + file_unarchive_cmd = ["xz", "--decompress", "$ARCHIVE_PATH"] + image_build_method = "resize" + image_path = "image.img" + image_size = var.target_image_size + image_type = "dos" + image_partitions { + name = "boot" + type = "c" + start_sector = "8192" + filesystem = "fat" + size = "512MB" + mountpoint = "/boot/firmware" + } + image_partitions { + name = "root" + type = "83" + start_sector = "1056768" + filesystem = "ext4" + size = "0" + mountpoint = "/" + } + image_chroot_env = ["PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin"] + qemu_binary_source_path = "/usr/bin/qemu-aarch64-static" + qemu_binary_destination_path = "/usr/bin/qemu-aarch64-static" } build { - sources = ["source.arm-image.raspberry_pi_os"] + sources = ["source.arm.raspios"] provisioner "file" { source = "ubo_app/system/install.sh" @@ -38,13 +69,11 @@ build { provisioner "shell" { inline = [ - "echo 'ls -l /boot'; ls -l /boot", "chmod +x /install.sh", - "/install.sh --for-packer --with-docker --source=/ubo_app-${var.ubo_app_version}-py3-none-any.whl", + "/install.sh --in-packer --with-docker --source=/ubo_app-${var.ubo_app_version}-py3-none-any.whl", "rm /install.sh /ubo_app-${var.ubo_app_version}-py3-none-any.whl", "/usr/bin/env systemctl disable userconfig || true", - "/usr/bin/env systemctl disable lightdm || true", - "apt clean", + "apt-get clean -y", "echo DF; df -h" ] } diff --git a/ubo_app/bootstrap.py b/ubo_app/bootstrap.py index a347dd09..3fd7632b 100644 --- a/ubo_app/bootstrap.py +++ b/ubo_app/bootstrap.py @@ -9,7 +9,7 @@ def main() -> None: """Run the bootstrap script.""" bootstrap( with_docker='--with-docker' in sys.argv, - for_packer='--for-packer' in sys.argv, + in_packer='--in-packer' in sys.argv, ) sys.stdout.write('Bootstrap completed.\n') sys.stdout.flush() diff --git a/ubo_app/system/bootstrap.py b/ubo_app/system/bootstrap.py index 1f4bc982..568e0012 100644 --- a/ubo_app/system/bootstrap.py +++ b/ubo_app/system/bootstrap.py @@ -163,6 +163,11 @@ def enable_services() -> None: def configure_fan() -> None: """Configure the behavior of the fan.""" + if ( + 'dtoverlay=gpio-fan,gpiopin=22,temp=60000' + in Path('/boot/firmware/config.txt').read_text() + ): + return with Path('/boot/firmware/config.txt').open('a') as config_file: config_file.write('dtoverlay=gpio-fan,gpiopin=22,temp=60000\n') @@ -208,28 +213,22 @@ def install_docker() -> None: stdout.flush() -def install_audio_driver(*, for_packer: bool) -> None: +def install_audio_driver(*, in_packer: bool) -> None: """Install the audio driver.""" - if for_packer: - # Convincing the installer that we are on a Raspberry Pi as /proc is not created - # in packer build - Path('/proc/device-tree').mkdir(parents=True, exist_ok=True) - Path('/proc/device-tree/model').write_text('Raspberry Pi') stdout.write('Installing wm8960...\n') stdout.flush() subprocess.run( - [Path(__file__).parent.joinpath('install_wm8960.sh').as_posix()], # noqa: S603 + [ # noqa: S603 + Path(__file__).parent.joinpath('install_wm8960.sh').as_posix(), + ] + + (['--in-packer'] if in_packer else []), check=True, ) stdout.write('Done installing wm8960\n') stdout.flush() - if for_packer: - Path('/proc/device-tree/model').unlink() - Path('/proc/device-tree').rmdir() - Path('/proc').rmdir() -def bootstrap(*, with_docker: bool = False, for_packer: bool = False) -> None: +def bootstrap(*, with_docker: bool = False, in_packer: bool = False) -> None: """Create the service files and enable the services.""" # Ensure we have the required permissions if os.geteuid() != 0: @@ -241,7 +240,7 @@ def bootstrap(*, with_docker: bool = False, for_packer: bool = False) -> None: for service in SERVICES: create_service_file(service) - if for_packer: + if in_packer: Path('/var/lib/systemd/linger').mkdir(exist_ok=True, parents=True) Path(f'/var/lib/systemd/linger/{USERNAME}').touch(mode=0o644, exist_ok=True) else: @@ -264,4 +263,4 @@ def bootstrap(*, with_docker: bool = False, for_packer: bool = False) -> None: stdout.write('Done installing docker\n') stdout.flush() - install_audio_driver(for_packer=for_packer) + install_audio_driver(in_packer=in_packer) diff --git a/ubo_app/system/install.sh b/ubo_app/system/install.sh index 8ffb1e7c..abae221d 100755 --- a/ubo_app/system/install.sh +++ b/ubo_app/system/install.sh @@ -9,7 +9,7 @@ USERNAME=${USERNAME:-"ubo"} UPDATE=${UPDATE:-false} ALPHA=${ALPHA:-false} WITH_DOCKER=${WITH_DOCKER:-false} -FOR_PACKER=false +IN_PACKER=false SOURCE=${SOURCE:-"ubo-app"} export DEBIAN_FRONTEND=noninteractive @@ -30,9 +30,9 @@ do WITH_DOCKER=true shift # Remove --with-docker from processing ;; - --for-packer) - FOR_PACKER=true - shift # Remove --for-packer from processing + --in-packer) + IN_PACKER=true + shift # Remove --in-packer from processing ;; --source=*) SOURCE="${arg#*=}" @@ -124,7 +124,10 @@ chown -R $USERNAME:$USERNAME "$INSTALLATION_PATH" chmod -R 700 "$INSTALLATION_PATH" # Bootstrap the application -UBO_LOG_LEVEL=INFO "$INSTALLATION_PATH/env/bin/bootstrap"${WITH_DOCKER:+ --with-docker}${FOR_PACKER:+ --for-packer} +ls -l / +ls -l /proc +mount +UBO_LOG_LEVEL=INFO "$INSTALLATION_PATH/env/bin/bootstrap"${WITH_DOCKER:+ --with-docker}${IN_PACKER:+ --in-packer} echo "Bootstrapping completed" if [ "$UPDATE" = true ]; then @@ -132,7 +135,7 @@ if [ "$UPDATE" = true ]; then rm -rf "$INSTALLATION_PATH/_update" fi -if [ "$FOR_PACKER" = true ]; then +if [ "$IN_PACKER" = true ]; then exit 0 else # The audio driver needs a reboot to work diff --git a/ubo_app/system/install_wm8960.sh b/ubo_app/system/install_wm8960.sh index a527908d..284efede 100755 --- a/ubo_app/system/install_wm8960.sh +++ b/ubo_app/system/install_wm8960.sh @@ -5,10 +5,108 @@ set -o errexit set -o pipefail set -o nounset +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root (use sudo)" 1>&2 + exit 1 +fi + +if [ ! -f /etc/rpi-issue ]; then + echo "Sorry, this drivers only works on raspberry pi" + exit 1 +fi + + +#download the archive git clone https://github.com/waveshare/WM8960-Audio-HAT cd WM8960-Audio-HAT -./install.sh +apt-get -y update +apt-get -y install raspberrypi-kernel-headers #raspberrypi-kernel +apt-get -y install dkms git i2c-tools libasound2-plugins + +# locate currently installed kernels (may be different to running kernel if +# it's just been updated) +kernels=$(ls /lib/modules) + +function install_module { + ver="1.0" + # we create a dir with this version to ensure that 'dkms remove' won't delete + # the sources during kernel updates + marker="0.0.0" + + src=$1 + mod=$2 + + if [[ -d /var/lib/dkms/$mod/$ver/$marker ]]; then + rmdir /var/lib/dkms/$mod/$ver/$marker + fi + + if [[ -e /usr/src/$mod-$ver || -e /var/lib/dkms/$mod/$ver ]]; then + dkms remove --force -m $mod -v $ver --all + rm -rf /usr/src/$mod-$ver + fi + mkdir -p /usr/src/$mod-$ver + cp -a $src/* /usr/src/$mod-$ver/ + dkms add -m $mod -v $ver + for kernel in $kernels + do + # It works for kernels greater than or equal 6.5 + if [ $(echo "$kernel 6.5" | awk '{if ($1 >= $2) print 1; else print 0}') -eq 0 ]; then + continue + fi + dkms build "$kernel" -k "$kernel" --kernelsourcedir "/lib/modules/$kernel/build" -m $mod -v $ver && + dkms install --force "$kernel" -k "$kernel" -m $mod -v $ver + done + + mkdir -p /var/lib/dkms/$mod/$ver/$marker +} +install_module "./" "wm8960-soundcard" + +# install dtbos +cp wm8960-soundcard.dtbo /boot/overlays + + +#set kernel modules +grep -q "^i2c-dev$" /etc/modules || \ + echo "i2c-dev" >> /etc/modules +grep -q "^snd-soc-wm8960$" /etc/modules || \ + echo "snd-soc-wm8960" >> /etc/modules +grep -q "^snd-soc-wm8960-soundcard$" /etc/modules || \ + echo "snd-soc-wm8960-soundcard" >> /etc/modules + +# set modprobe blacklist +grep -q "^blacklist snd_bcm2835$" /etc/modprobe.d/raspi-blacklist.conf || \ + echo "blacklist snd_bcm2835" >> /etc/modprobe.d/raspi-blacklist.conf + +#set dtoverlays +sed -i -e 's:#dtparam=i2s=on:dtparam=i2s=on:g' /boot/firmware/config.txt || true +sed -i -e 's:#dtparam=i2c_arm=on:dtparam=i2c_arm=on:g' /boot/firmware/config.txt || true +grep -q "^dtoverlay=i2s-mmap$" /boot/firmware/config.txt || \ + echo "dtoverlay=i2s-mmap" >> /boot/firmware/config.txt + +grep -q "^dtparam=i2s=on$" /boot/firmware/config.txt || \ + echo "dtparam=i2s=on" >> /boot/firmware/config.txt + +grep -q "^dtoverlay=wm8960-soundcard$" /boot/firmware/config.txt || \ + echo "dtoverlay=wm8960-soundcard" >> /boot/firmware/config.txt + +#install config files +mkdir -p /etc/wm8960-soundcard +cp *.conf /etc/wm8960-soundcard +cp *.state /etc/wm8960-soundcard + +#set service +cp wm8960-soundcard /usr/bin/ +chmod -x wm8960-soundcard.service +cp wm8960-soundcard.service /lib/systemd/system/ +systemctl enable wm8960-soundcard.service + +#cleanup cd .. rm -rf WM8960-Audio-HAT + +echo "------------------------------------------------------" +echo "Please reboot your raspberry pi to apply all settings" +echo "Enjoy!" +echo "------------------------------------------------------"