CI/CD

fix(core): use fasteners read-write lock implementation for the persi… #545

Workflow file for this run

.github/workflows/integration_delivery.yml at 9cb0bc3

	name: CI/CD

	on:
	push:
	pull_request:
	workflow_dispatch:

	env:
	PYTHON_VERSION: '3.11.9'

	jobs:
	dependencies:
	name: Install Dependencies
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	name: Checkout

	- name: System Dependencies
	run: \|
	sudo apt-get update -y
	sudo apt-get install -y portaudio19-dev

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	architecture: x64

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	virtualenvs-create: true

	- name: Check Cached Poetry
	id: cached-poetry
	uses: actions/cache/restore@v4
	with:
	lookup-only: true
	path: \|
	~/.cache
	~/.local
	key:
	poetry-ubuntu-latest-python${{ env.PYTHON_VERSION
	}}-${{hashFiles('poetry.lock') }}

	- name: Install dependencies
	if: steps.cached-poetry.outputs.cache-hit != 'true'
	run: poetry install --no-interaction --no-root --extras=dev --with dev

	- name: Save Cached Poetry
	if: steps.cached-poetry.outputs.cache-hit != 'true'
	uses: actions/cache/save@v4
	with:
	path: \|
	~/.cache
	~/.local
	key:
	poetry-ubuntu-latest-python${{ env.PYTHON_VERSION
	}}-${{hashFiles('poetry.lock') }}

	ubo-pod-dependencies:
	name: Install Ubo Pod Dependencies
	runs-on: ubo-pod
	outputs:
	python-version: ${{ steps.get-python-version.outputs.PYTHON_VERSION }}
	steps:
	- uses: actions/checkout@v4
	name: Checkout

	- name: Extract Python Version
	id: get-python-version
	run: \|
	echo "PYTHON_VERSION=$(python --version \| cut -d' ' -f2)" >> $GITHUB_OUTPUT
	echo "PYTHON_VERSION=$(python --version \| cut -d' ' -f2)"

	- name: Check Cached Poetry
	id: cached-poetry
	uses: actions/cache/restore@v4
	with:
	lookup-only: true
	path: \|
	~/.cache
	~/.local
	key:
	poetry-ubo-pod-python${{
	steps.get-python-version.outputs.PYTHON_VERSION }}-${{
	hashFiles('poetry.lock') }}

	- name: Install dependencies
	if: steps.cached-poetry.outputs.cache-hit != 'true'
	run: \|
	poetry env remove 3.11 \|\| true
	poetry config virtualenvs.options.system-site-packages true --local
	poetry env use 3.11
	poetry install --no-interaction --no-root --extras=dev --with dev

	- name: Save Cached Poetry
	if: steps.cached-poetry.outputs.cache-hit != 'true'
	uses: actions/cache/save@v4
	with:
	path: \|
	~/.cache
	~/.local
	key:
	poetry-ubo-pod-python${{
	steps.get-python-version.outputs.PYTHON_VERSION }}-${{
	hashFiles('poetry.lock') }}

	type-check:
	name: Type Check
	needs:
	- ubo-pod-dependencies
	runs-on: ubo-pod
	steps:
	- uses: actions/checkout@v4
	name: Checkout

	- name: Load Cached Poetry
	id: cached-poetry
	uses: actions/cache/restore@v4
	with:
	path: \|
	~/.cache
	~/.local
	key:
	poetry-ubo-pod-python${{
	steps.get-python-version.outputs.PYTHON_VERSION }}-${{
	hashFiles('poetry.lock') }}

	- name: Create stub files
	run: \|
	poetry run pyright --createstub kivy

	- name: Type Check
	run: \|
	poetry run poe typecheck

	lint:
	name: Lint
	needs:
	- dependencies
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	name: Checkout

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	architecture: x64

	- name: Load Cached Poetry
	id: cached-poetry
	uses: actions/cache/restore@v4
	with:
	path: \|
	~/.cache
	~/.local
	key:
	poetry-ubuntu-latest-python${{ env.PYTHON_VERSION
	}}-${{hashFiles('poetry.lock') }}

	- name: Lint
	run: poetry run poe lint

	test:
	name: Test
	needs:
	- dependencies
	- ubo-pod-dependencies
	strategy:
	matrix:
	runner: ['ubo-pod', 'ubuntu-latest']
	runs-on: ${{ matrix.runner }}
	environment:
	name: test
	url: https://app.codecov.io/gh/${{ github.repository }}/
	steps:
	- name: System Dependencies
	if: matrix.runner != 'ubo-pod'
	run: \|
	sudo apt-get update -y
	sudo apt-get install -y libegl1 libgl1 libmtdev1 libzbar0

	- uses: actions/checkout@v4
	name: Checkout

	- name: Setup Python
	if: matrix.runner != 'ubo-pod'
	uses: actions/setup-python@v5
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	architecture: x64

	- name: Load Cached Poetry
	id: cached-poetry
	uses: actions/cache/restore@v4
	with:
	path: \|
	~/.cache
	~/.local
	key:
	poetry-${{ matrix.runner }}-python${{ matrix.runner == 'ubo-pod' &&
	needs.dependencies.outputs.python-version \|\| env.PYTHON_VERSION
	}}-${{hashFiles('poetry.lock') }}

	- name: Run Tests
	run: \|
	poetry run poe test --make-screenshots --cov-report=xml --cov-report=html --log-level=DEBUG -n ${{ matrix.runner == 'ubo-pod' && '1' \|\| 'auto' }}

	- name: Collect Window Screenshots
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: screenshots-${{ matrix.runner }}
	path: tests//results//*.png

	- name: Collect Store Snapshots
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: snapshots-${{ matrix.runner }}
	path: tests//results//*.jsonc

	- name: Collect HTML Coverage Report
	uses: actions/upload-artifact@v4
	with:
	name: coverage-report-${{ matrix.runner }}
	path: htmlcov

	- name: Upload Coverage to Codecov
	uses: codecov/codecov-action@v4
	with:
	file: ./coverage.xml
	flags: ${{ matrix.runner }}
	fail_ci_if_error: false
	token: ${{ secrets.CODECOV_TOKEN }}

	build:
	name: Build
	needs:
	- dependencies
	runs-on: ubuntu-latest
	outputs:
	version: ${{ steps.extract-version.outputs.VERSION }}
	name: ${{ steps.extract-version.outputs.NAME }}
	steps:
	- uses: actions/checkout@v4
	name: Checkout
	# with: TODO: this, it consumes lfs bandwidth and should be used only for releases
	# lfs: true

	- uses: actions/setup-python@v5
	name: Setup Python
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	architecture: x64

	- name: Load Cached Poetry
	id: cached-poetry
	uses: actions/cache/restore@v4
	with:
	path: \|
	~/.cache
	~/.local
	key:
	poetry-ubuntu-latest-python${{ env.PYTHON_VERSION
	}}-${{hashFiles('poetry.lock') }}

	- name: Extract Version
	id: extract-version
	run: \|
	echo "VERSION=$(poetry version --short)" >> "$GITHUB_OUTPUT"
	echo "VERSION=$(poetry version --short)"
	echo "NAME=$(poetry version \| cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
	echo "NAME=$(poetry version \| cut -d' ' -f1)"

	- name: Extract Version from CHANGELOG.md
	run: \|
	VERSION_CHANGELOG=$(sed -n '3 s/## Version //p' CHANGELOG.md)
	echo "VERSION_CHANGELOG=$VERSION_CHANGELOG"
	if [ "${{ steps.extract-version.outputs.VERSION }}" != "$VERSION_CHANGELOG" ]; then
	echo "Error: Version extracted from CHANGELOG.md does not match the version in pyproject.toml"
	exit 1
	else
	echo "Versions are consistent."
	fi

	- name: Extract Version from Tag
	if: startsWith(github.ref, 'refs/tags/v')
	run: \|
	VERSION_TAG=$(sed 's/^v//' <<< ${{ github.ref_name }})
	echo "VERSION_TAG=$VERSION_TAG"
	if [ "${{ steps.extract-version.outputs.VERSION }}" != "$VERSION_TAG" ]; then
	echo "Error: Version extracted from tag does not match the version in pyproject.toml"
	exit 1
	else
	echo "Versions are consistent."
	fi

	- name: Configure Sentry
	run: \|
	echo "SENTRY_DSN=${{ secrets.SENTRY_DSN }}" >> ubo_app/.env
	# conditionally set it based on whether it's a tag or not using github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
	if [ "${{ github.event_name }}" == "push" ] && [ "$(echo ${{ github.ref }} \| grep -c 'refs/tags/v')" -eq 1 ]; then
	echo "SENTRY_RELEASE=ubo-app@${{ steps.extract-version.outputs.VERSION }}" >> ubo_app/.env
	else
	echo "SENTRY_RELEASE=ubo-app@${{ github.sha }}" >> ubo_app/.env
	fi
	cat ubo_app/.env

	- name: Build
	run: poetry build

	- name: Upload wheel
	uses: actions/upload-artifact@v4
	with:
	name: wheel
	path: dist/*.whl
	if-no-files-found: error

	- name: Upload binary
	uses: actions/upload-artifact@v4
	with:
	name: binary
	path: dist/*.tar.gz
	if-no-files-found: error

	images:
	name: Create Images
	needs:
	- type-check
	- lint
	- test
	- build
	runs-on: ubuntu-latest
	container:
	image: mkaczanowski/packer-builder-arm
	volumes:
	- /dev:/dev
	options: --rm --privileged
	strategy:
	fail-fast: false
	matrix:
	suffix: ['lite', '']
	steps:
	- run: echo Building amd64-${{ matrix.suffix }} image

	- name: Checkout
	uses: actions/checkout@v4

	- uses: actions/download-artifact@v4
	with:
	name: wheel
	path: /build/dist

	- uses: actions/download-artifact@v4
	with:
	name: binary
	path: /build/dist

	- name: Generate Image URL and Checksum
	id: generate-image-url
	run: \|
	if [ -n "${{ matrix.suffix }}" ]; then
	SUFFIX="_${{ matrix.suffix }}"
	else
	SUFFIX="${{ matrix.suffix }}"
	fi
	DASHED_SUFFIX=$(echo $SUFFIX \| sed 's/_/-/g')
	IMAGE_URL="https://downloads.raspberrypi.com/raspios${SUFFIX}_arm64/images/raspios${SUFFIX}_arm64-2024-07-04/2024-07-04-raspios-bookworm-arm64${DASHED_SUFFIX}.img.xz"
	CHECKSUM_URL="${IMAGE_URL}.sha256"
	echo "suffix=$SUFFIX" >> "$GITHUB_OUTPUT"
	echo "dashed_suffix=$DASHED_SUFFIX" >> "$GITHUB_OUTPUT"
	echo "image_url=$IMAGE_URL" >> "$GITHUB_OUTPUT"
	echo "image_checksum_url=$CHECKSUM_URL" >> "$GITHUB_OUTPUT"
	IMAGE_SIZE_GB=${{ matrix.suffix == 'lite' && '4.25' \|\| matrix.suffix == '' && '6.5' \|\| '13' }}
	IMAGE_SIZE=$(awk -v IMAGE_SIZE_GB=$IMAGE_SIZE_GB 'BEGIN {printf "%.0f", IMAGE_SIZE_GB * 1024 ^ 3}')
	echo "image_size=$IMAGE_SIZE" >> "$GITHUB_OUTPUT"

	- name: Build Artifact
	env:
	PKR_VAR_ubo_app_version: ${{ needs.build.outputs.version }}
	PKR_VAR_image_url: ${{ steps.generate-image-url.outputs.image_url }}
	PKR_VAR_image_checksum_url:
	${{ steps.generate-image-url.outputs.image_checksum_url }}
	PKR_VAR_target_image_size:
	${{ steps.generate-image-url.outputs.image_size}}
	run: \|
	/entrypoint.sh init scripts/packer/image.pkr.hcl
	SETUP_QEMU=false /entrypoint.sh build scripts/packer/image.pkr.hcl
	ls -lh
	mv image.img /build
	ls -lh /build

	- name: Fill Free Space with Zeros
	run: \|
	apt-get update
	apt-get install -y zerofree
	LOOP_DEV=$(losetup -f)
	losetup -P $LOOP_DEV /build/image.img
	zerofree -v "${LOOP_DEV}p2"
	losetup -d $LOOP_DEV

	- name: Compress File with Gzip
	run: \|
	scripts/consume.sh /build/image.img \| gzip -9 > /ubo_app-${{ needs.build.outputs.version }}-bookworm${{ steps.generate-image-url.outputs.dashed_suffix }}.img.gz

	- name: Upload Image
	uses: actions/upload-artifact@v4
	with:
	name:
	ubo_app-${{ needs.build.outputs.version }}-bookworm${{
	steps.generate-image-url.outputs.dashed_suffix}}-arm64.img.gz
	path:
	/ubo_app-${{ needs.build.outputs.version }}-bookworm${{
	steps.generate-image-url.outputs.dashed_suffix }}.img.gz
	if-no-files-found: error

	pypi-publish:
	name: Publish to PyPI
	if: >-
	github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
	needs:
	- type-check
	- lint
	- test
	- build
	- images
	runs-on: ubuntu-latest
	environment:
	name: pypi
	url: https://pypi.org/p/${{ needs.build.outputs.name }}
	permissions:
	id-token: write
	steps:
	- uses: actions/download-artifact@v4
	with:
	name: wheel
	path: dist

	- uses: actions/download-artifact@v4
	with:
	name: binary
	path: dist

	- name: Publish package distributions to PyPI
	uses: pypa/gh-action-pypi-publish@release/v1
	with:
	packages-dir: dist
	verbose: true

	release:
	if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
	name: Release
	needs:
	- type-check
	- lint
	- test
	- build
	- pypi-publish
	- images
	runs-on: ubuntu-latest
	environment:
	name: release
	url:
	https://github.com/${{ github.repository }}/releases/tag/v${{
	needs.build.outputs.version }}
	permissions:
	contents: write
	steps:
	- uses: actions/checkout@v4
	name: Checkout

	- name: Procure Lite Image
	uses: actions/download-artifact@v4
	with:
	name:
	ubo_app-${{ needs.build.outputs.version
	}}-bookworm-lite-arm64.img.gz
	path: artifacts

	- name: Procure Default Image
	uses: actions/download-artifact@v4
	with:
	name: ubo_app-${{ needs.build.outputs.version }}-bookworm-arm64.img.gz
	path: artifacts

	- name: Procure Wheel
	uses: actions/download-artifact@v4
	with:
	name: wheel
	path: artifacts

	- name: Procure Binary
	uses: actions/download-artifact@v4
	with:
	name: binary
	path: artifacts

	- name:
	Split Large Files into 2GB chunks in a for loop only if they are
	bigger than 2GB
	run: \|
	for file in artifacts/*; do
	if [ $(stat -c%s "$file") -gt 2147000000 ]; then
	split -b 2147000000 "$file" "$file"_
	rm "$file"
	fi
	done

	- name: Extract Changelog
	id: changelog
	run: \|
	perl -0777 -ne 'while (/## Version ${{ needs.build.outputs.version }}\n(\s\n)(.?)(\s\n)*## Version \d+\.\d+\.\d+\n/sg) {print "$2\n"}' CHANGELOG.md > CURRENT_CHANGELOG.md
	{
	echo "CONTENT<<EOF"
	cat CURRENT_CHANGELOG.md
	echo "EOF"
	} >> "$GITHUB_OUTPUT"

	- name: Release
	uses: softprops/action-gh-release@v2
	with:
	files: artifacts/*
	tag_name: v${{ needs.build.outputs.version }}
	body: \|
	PyPI package: https://pypi.org/project/${{ needs.build.outputs.name }}/${{ needs.build.outputs.version }}

	# Changes:
	${{ steps.changelog.outputs.CONTENT }}

	---

	Note than GitHub doesn't allow assets bigger than 2GB in a release. Due to this, the files bigger than 2GB have been split into 2GB chunks. You can join them using the following command:

	```bash
	cat [[filename]]_* > [[filename]]
	```
	prerelease: false
	draft: false

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix(core): use fasteners read-write lock implementation for the persi… #545

Workflow file

fix(core): use fasteners read-write lock implementation for the persi… #545

Jobs

Run details

Workflow file for this run