chore(test): set up testing framework with initial examples

chore(test): set up snapshot test helper to compare screenshots of different stages of tests with previous successful tests using hashes

.github/workflows/integration_delivery.yml

@@ -96,6 +96,59 @@ jobs:
       - name: Lint
         run: poetry run poe lint
 
+  test:
+    name: Test
+    needs:
+      - dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - name: System Dependencies
+        run: sudo apt-get install -y libegl1 libgl1 libmtdev1 libzbar0
+
+      - uses: actions/checkout@v4
+        name: Checkout
+
+      - uses: actions/setup-python@v5
+        name: Setup Python
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          architecture: x64
+
+      - name: Load Cached Poetry
+        id: cached-poetry
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cache
+            ~/.local
+          key: poetry-${{ hashFiles('poetry.lock') }}
+
+      - name: Test
+        run: poetry run poe test --cov-report=xml --cov-report=html
+
+      - name: Collect Mismatching Screenshots
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: mismatching-screenshots
+          path: tests/**/*.mismatch.png
+
+      - name: Collect HTML Coverage Report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: htmlcov
+
+      - name: Upload Coverage to Codecov
+        if: always()
+        uses: codecov/[email protected]
+        with:
+          file: ./coverage.xml
+          flags: integration
+          fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+
   build:
     name: Build
     needs:
@@ -155,6 +208,7 @@ jobs:
     needs:
       - type-check
       - lint
+      - test
       - build
     runs-on: ubuntu-latest
     environment:
@@ -183,6 +237,9 @@ jobs:
   images:
     name: Create Images
     needs:
+      - type-check
+      - lint
+      - test
       - build
     runs-on: ubuntu-latest
     container:
@@ -271,6 +328,7 @@ jobs:
     needs:
       - type-check
       - lint
+      - test
       - build
       - pypi-publish
       - images

.gitignore

@@ -52,6 +52,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 cover/
+tests/**/results/*png
 
 # Translations
 *.mo

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Version 0.11.1
+
+- chore(test): set up testing framework with initial examples
+- chore(test): set up snapshot test helper to compare screenshots of different stages
+  of tests with previous successful tests using hashes
+
 ## Version 0.11.0
 
 - feat: add ollama and open-webui docker images

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ubo-app"
-version = "0.11.0"
+version = "0.11.1"
 description = "Ubo main app, running on device initialization. A platform for running other apps."
 authors = ["Sassan Haradji <[email protected]>"]
 license = "Apache-2.0"
@@ -15,21 +15,22 @@ priority = "primary"
 
 
 [tool.poetry.dependencies]
+
 python = "^3.11"
 psutil = "^5.9.8"
 ubo-gui = [
-  { version = "^0.9.7", markers = "extra=='default'", extras = [
+  { version = "^0.9.9", markers = "extra=='default'", extras = [
     'default',
   ] },
-  { version = "^0.9.7", markers = "extra=='dev'", extras = [
+  { version = "^0.9.9", markers = "extra=='dev'", extras = [
     'dev',
   ] },
 ]
-python-redux = "^0.10.7"
+python-redux = "^0.11.0"
 pyzbar = "^0.1.9"
 sdbus-networkmanager = { version = "^2.0.0", markers = "platform_machine=='aarch64'" }
 rpi_ws281x = { version = "^5.0.0", markers = "platform_machine=='aarch64'" }
-python-debouncer = "^0.1.3"
+python-debouncer = "^0.1.4"
 adafruit-circuitpython-neopixel = "^6.3.11"
 pulsectl = "^23.5.2"
 aiohttp = "^3.9.1"
@@ -43,9 +44,15 @@ python-dotenv = "^1.0.1"
 optional = true
 
 [tool.poetry.group.dev.dependencies]
+poethepoet = "^0.24.4"
 pyright = "^1.1.349"
 ruff = "^0.3.2"
 toml = "^0.10.2"
+pytest = "^8.0.0"
+pytest-cov = "^4.1.0"
+pytest-xdist = "^3.5.0"
+tenacity = "^8.2.3"
+pytest-asyncio = "^0.23.5.post1"
 
 [tool.poetry.extras]
 default = ['ubo-gui']
@@ -61,7 +68,8 @@ build-backend = "poetry.core.masonry.api"
 [tool.poe.tasks]
 lint = "ruff check . --unsafe-fixes"
 typecheck = "pyright -p pyproject.toml ."
-sanity = ["typecheck", "lint"]
+test = "pytest --cov=ubo_app --cov-report=term-missing"
+sanity = ["typecheck", "lint", "test"]
 
 [tool.poe.tasks.deploy_to_device]
 args = [
@@ -85,6 +93,9 @@ docstring-quotes = "double"
 inline-quotes = "single"
 multiline-quotes = "double"
 
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S101"]
+
 [tool.ruff.format]
 quote-style = 'single'
 
@@ -93,3 +104,9 @@ profile = "black"
 
 [tool.pyright]
 exclude = ['typings']
+
+[tool.pytest.ini_options]
+asyncio_mode = 'auto'
+filterwarnings = "ignore:'imghdr' is deprecated:DeprecationWarning"
+log_cli = 1
+log_cli_level = 'ERROR'

tests/conftest.py

@@ -0,0 +1,173 @@
+"""Pytest configuration file for the tests."""
+from __future__ import annotations
+
+import asyncio
+import atexit
+import datetime
+import gc
+import sys
+import threading
+import weakref
+from typing import TYPE_CHECKING, AsyncGenerator, Callable, Generator
+
+import pytest
+from redux import FinishAction
+from snapshot import snapshot
+from tenacity import retry, stop_after_delay, wait_exponential
+
+from ubo_app.utils.garbage_collection import examine
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+    from ubo_app.menu import MenuApp
+
+__all__ = ('app_context', 'snapshot')
+
+
+@pytest.fixture(autouse=True, name='monkeypatch_atexit')
+def _(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(atexit, 'register', lambda _: None)
+
+
+modules_snapshot = set(sys.modules)
+
+
+@pytest.fixture(autouse=True)
+def _(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Mock external resources."""
+    monkeypatch.setattr('psutil.cpu_percent', lambda **_: 50)
+    monkeypatch.setattr(
+        'psutil.virtual_memory',
+        lambda *_: type('', (object,), {'percent': 50}),
+    )
+
+    class DateTime(datetime.datetime):
+        @classmethod
+        def now(cls: type[DateTime], tz: datetime.tzinfo | None = None) -> DateTime:
+            _ = tz
+            return DateTime(2023, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc)
+
+    monkeypatch.setattr(datetime, 'datetime', DateTime)
+
+
+@pytest.fixture()
+def logger() -> Logger:
+    import logging
+
+    import ubo_app.logging
+    from ubo_app.constants import LOG_LEVEL
+
+    level = (
+        getattr(
+            ubo_app.logging,
+            LOG_LEVEL,
+            getattr(logging, LOG_LEVEL, logging.DEBUG),
+        )
+        if LOG_LEVEL
+        else logging.DEBUG
+    )
+
+    logger = ubo_app.logging.get_logger('test')
+    logger.setLevel(level)
+
+    return logger
+
+
+class AppContext:
+    """Context object for tests running a menu application."""
+
+    def set_app(self: AppContext, app: MenuApp) -> None:
+        """Set the application."""
+        self.app = app
+        loop = asyncio.get_event_loop()
+        self.task = loop.create_task(self.app.async_run(async_lib='asyncio'))
+
+
+@pytest.fixture()
+async def app_context(logger: Logger) -> AsyncGenerator[AppContext, None]:
+    """Create the application."""
+    import os
+
+    os.environ['KIVY_NO_FILELOG'] = '1'
+    os.environ['KIVY_NO_CONSOLELOG'] = '1'
+
+    import headless_kivy_pi.config
+
+    headless_kivy_pi.config.setup_headless_kivy({'automatic_fps': True})
+
+    context = AppContext()
+
+    yield context
+
+    assert context.task is not None, 'App not set for test'
+
+    await context.task
+
+    app_ref = weakref.ref(context.app)
+    context.app.root.clear_widgets()
+
+    del context.app
+    del context.task
+
+    gc.collect()
+    app = app_ref()
+
+    if app is not None:
+        logger.debug(
+            'Memory leak: failed to release app for test.',
+            extra={
+                'referrers': gc.get_referrers(app),
+                'referents': gc.get_referents(app),
+                'refcount': sys.getrefcount(app),
+                'ref': app,
+            },
+        )
+        gc.collect()
+        for cell in gc.get_referrers(app):
+            if type(cell).__name__ == 'cell':
+                logger.debug('CELL EXAMINATION', extra={'cell': cell})
+                examine(cell, depth_limit=2)
+    assert app is None, 'Memory leak: failed to release app for test'
+
+    from kivy.core.window import Window
+
+    Window.close()
+
+    for module in set(sys.modules) - modules_snapshot:
+        if module != 'objc' and 'numpy' not in module and 'cache' not in module:
+            del sys.modules[module]
+    gc.collect()
+
+
+@pytest.fixture()
+def needs_finish() -> Generator:
+    yield None
+
+    from ubo_app.store import dispatch
+
+    dispatch(FinishAction())
+
+
+class WaitFor(threading.Thread):
+    def __call__(
+        self: WaitFor,
+        satisfaction: Callable[[], None],
+        *,
+        timeout: float = 1,
+    ) -> None:
+        self.retry = retry(
+            stop=stop_after_delay(timeout),
+            wait=wait_exponential(multiplier=0.5),
+        )(satisfaction)
+        self.start()
+
+    def run(self: WaitFor) -> None:
+        self.retry()
+
+
+@pytest.fixture()
+def wait_for() -> Generator[WaitFor, None, None]:
+    context = WaitFor()
+    yield context
+    context.join()

tests/results/test_all_services_register-000.hash

@@ -0,0 +1 @@
+a4173e6761d21df059d9865602afbe8482c959c68a8a9426e3c3270374d77471