From d9059d57b8edf465b7e595cbc490b54fb39cc4c9 Mon Sep 17 00:00:00 2001 From: Sassan Haradji Date: Sat, 17 Aug 2024 11:53:46 +0400 Subject: [PATCH] refactor(core): general housekeeping, improve resource management in runtime and test environment, minor bug fixes --- .github/workflows/integration_delivery.yml | 2 +- CHANGELOG.md | 1 + poetry.lock | 124 +++++++++--------- pyproject.toml | 21 ++- scripts/deploy.sh | 38 +++--- scripts/test_on_device.sh | 37 +++--- tests/conftest.py | 4 +- tests/fixtures/app.py | 115 ++++++++-------- tests/fixtures/load_services.py | 2 +- tests/fixtures/menu.py | 13 +- tests/fixtures/mock_camera.py | 2 +- tests/fixtures/stability.py | 7 +- tests/fixtures/store.py | 2 +- tests/flows/conftest.py | 21 +++ tests/flows/test_wireless.py | 16 +-- .../all_services_register/store-rpi-000.jsonc | 4 +- tests/integration/test_services.py | 2 +- tests/monkeypatch.py | 2 +- tests/setup.sh | 4 +- ubo_app/load_services.py | 29 ++-- ubo_app/logging.py | 23 +++- ubo_app/main.py | 5 +- ubo_app/menu_app/menu_central.py | 19 +-- ubo_app/service.py | 50 +++---- ubo_app/services/000-audio/setup.py | 1 - ubo_app/services/030-wifi/setup.py | 2 +- ubo_app/setup.py | 20 ++- ubo_app/system/bootstrap.py | 10 ++ ubo_app/utils/async_.py | 60 +-------- ubo_app/utils/bus_provider.py | 24 +++- ubo_app/utils/persistent_store.py | 6 +- 31 files changed, 371 insertions(+), 295 deletions(-) create mode 100644 tests/flows/conftest.py diff --git a/.github/workflows/integration_delivery.yml b/.github/workflows/integration_delivery.yml index 4acb5179..46a68726 100644 --- a/.github/workflows/integration_delivery.yml +++ b/.github/workflows/integration_delivery.yml @@ -208,7 +208,7 @@ jobs: - name: Run Tests run: | - poetry run poe test --make-screenshots --cov-report=xml --cov-report=html --log-level=DEBUG -n ${{ matrix.runner == 'ubo-pod' && '1' || 'auto' }} + poetry run poe test --verbosity=2 --capture=no --make-screenshots --cov-report=xml --cov-report=html --log-level=DEBUG -n ${{ matrix.runner == 'ubo-pod' && '1' || 'auto' }} - name: Collect Window Screenshots uses: actions/upload-artifact@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index ff4b0987..2aea8859 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - feat(core): add signal management for ubo_app process - closes #156 - fix(core): use fasteners read-write lock implementation for the persistent store - closes #158 - feat(core): improve the user experience of update-manager by making it more verbose about the current state of the update progress - closes #153 +- refactor(core): general housekeeping, improve resource management in runtime and test environment, minor bug fixes ## Version 0.15.5 diff --git a/poetry.lock b/poetry.lock index 4a5cfa23..83bd4ccd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -216,13 +216,13 @@ files = [ [[package]] name = "aiohappyeyeballs" -version = "2.3.5" +version = "2.3.6" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.8" files = [ - {file = "aiohappyeyeballs-2.3.5-py3-none-any.whl", hash = "sha256:4d6dea59215537dbc746e93e779caea8178c866856a721c9c660d7a5a7b8be03"}, - {file = "aiohappyeyeballs-2.3.5.tar.gz", hash = "sha256:6fa48b9f1317254f122a07a131a86b71ca6946ca989ce6326fff54a99a920105"}, + {file = "aiohappyeyeballs-2.3.6-py3-none-any.whl", hash = "sha256:15dca2611fa78442f1cb54cf07ffb998573f2b4fbeab45ca8554c045665c896b"}, + {file = "aiohappyeyeballs-2.3.6.tar.gz", hash = "sha256:88211068d2a40e0436033956d7de3926ff36d54776f8b1022d6b21320cadae79"}, ] [[package]] @@ -1260,42 +1260,42 @@ files = [ [[package]] name = "onnxruntime" -version = "1.18.1" +version = "1.19.0" description = "ONNX Runtime is a runtime accelerator for Machine Learning models" optional = false python-versions = "*" files = [ - {file = "onnxruntime-1.18.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:29ef7683312393d4ba04252f1b287d964bd67d5e6048b94d2da3643986c74d80"}, - {file = "onnxruntime-1.18.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc706eb1df06ddf55776e15a30519fb15dda7697f987a2bbda4962845e3cec05"}, - {file = "onnxruntime-1.18.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7de69f5ced2a263531923fa68bbec52a56e793b802fcd81a03487b5e292bc3a"}, - {file = "onnxruntime-1.18.1-cp310-cp310-win32.whl", hash = "sha256:221e5b16173926e6c7de2cd437764492aa12b6811f45abd37024e7cf2ae5d7e3"}, - {file = "onnxruntime-1.18.1-cp310-cp310-win_amd64.whl", hash = "sha256:75211b619275199c861ee94d317243b8a0fcde6032e5a80e1aa9ded8ab4c6060"}, - {file = "onnxruntime-1.18.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:f26582882f2dc581b809cfa41a125ba71ad9e715738ec6402418df356969774a"}, - {file = "onnxruntime-1.18.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef36f3a8b768506d02be349ac303fd95d92813ba3ba70304d40c3cd5c25d6a4c"}, - {file = "onnxruntime-1.18.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:170e711393e0618efa8ed27b59b9de0ee2383bd2a1f93622a97006a5ad48e434"}, - {file = "onnxruntime-1.18.1-cp311-cp311-win32.whl", hash = "sha256:9b6a33419b6949ea34e0dc009bc4470e550155b6da644571ecace4b198b0d88f"}, - {file = "onnxruntime-1.18.1-cp311-cp311-win_amd64.whl", hash = "sha256:5c1380a9f1b7788da742c759b6a02ba771fe1ce620519b2b07309decbd1a2fe1"}, - {file = "onnxruntime-1.18.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:31bd57a55e3f983b598675dfc7e5d6f0877b70ec9864b3cc3c3e1923d0a01919"}, - {file = "onnxruntime-1.18.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b9e03c4ba9f734500691a4d7d5b381cd71ee2f3ce80a1154ac8f7aed99d1ecaa"}, - {file = "onnxruntime-1.18.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:781aa9873640f5df24524f96f6070b8c550c66cb6af35710fd9f92a20b4bfbf6"}, - {file = "onnxruntime-1.18.1-cp312-cp312-win32.whl", hash = "sha256:3a2d9ab6254ca62adbb448222e630dc6883210f718065063518c8f93a32432be"}, - {file = "onnxruntime-1.18.1-cp312-cp312-win_amd64.whl", hash = "sha256:ad93c560b1c38c27c0275ffd15cd7f45b3ad3fc96653c09ce2931179982ff204"}, - {file = "onnxruntime-1.18.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:3b55dc9d3c67626388958a3eb7ad87eb7c70f75cb0f7ff4908d27b8b42f2475c"}, - {file = "onnxruntime-1.18.1-cp38-cp38-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f80dbcfb6763cc0177a31168b29b4bd7662545b99a19e211de8c734b657e0669"}, - {file = "onnxruntime-1.18.1-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f1ff2c61a16d6c8631796c54139bafea41ee7736077a0fc64ee8ae59432f5c58"}, - {file = "onnxruntime-1.18.1-cp38-cp38-win32.whl", hash = "sha256:219855bd272fe0c667b850bf1a1a5a02499269a70d59c48e6f27f9c8bcb25d02"}, - {file = "onnxruntime-1.18.1-cp38-cp38-win_amd64.whl", hash = "sha256:afdf16aa607eb9a2c60d5ca2d5abf9f448e90c345b6b94c3ed14f4fb7e6a2d07"}, - {file = "onnxruntime-1.18.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:128df253ade673e60cea0955ec9d0e89617443a6d9ce47c2d79eb3f72a3be3de"}, - {file = "onnxruntime-1.18.1-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9839491e77e5c5a175cab3621e184d5a88925ee297ff4c311b68897197f4cde9"}, - {file = "onnxruntime-1.18.1-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad3187c1faff3ac15f7f0e7373ef4788c582cafa655a80fdbb33eaec88976c66"}, - {file = "onnxruntime-1.18.1-cp39-cp39-win32.whl", hash = "sha256:34657c78aa4e0b5145f9188b550ded3af626651b15017bf43d280d7e23dbf195"}, - {file = "onnxruntime-1.18.1-cp39-cp39-win_amd64.whl", hash = "sha256:9c14fd97c3ddfa97da5feef595e2c73f14c2d0ec1d4ecbea99c8d96603c89589"}, + {file = "onnxruntime-1.19.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6ce22a98dfec7b646ae305f52d0ce14a189a758b02ea501860ca719f4b0ae04b"}, + {file = "onnxruntime-1.19.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19019c72873f26927aa322c54cf2bf7312b23451b27451f39b88f57016c94f8b"}, + {file = "onnxruntime-1.19.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8eaa16df99171dc636e30108d15597aed8c4c2dd9dbfdd07cc464d57d73fb275"}, + {file = "onnxruntime-1.19.0-cp310-cp310-win32.whl", hash = "sha256:0eb0f8dbe596fd0f4737fe511fdbb17603853a7d204c5b2ca38d3c7808fc556b"}, + {file = "onnxruntime-1.19.0-cp310-cp310-win_amd64.whl", hash = "sha256:616092d54ba8023b7bc0a5f6d900a07a37cc1cfcc631873c15f8c1d6e9e184d4"}, + {file = "onnxruntime-1.19.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:a2b53b3c287cd933e5eb597273926e899082d8c84ab96e1b34035764a1627e17"}, + {file = "onnxruntime-1.19.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e94984663963e74fbb468bde9ec6f19dcf890b594b35e249c4dc8789d08993c5"}, + {file = "onnxruntime-1.19.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f379d1f050cfb55ce015d53727b78ee362febc065c38eed81512b22b757da73"}, + {file = "onnxruntime-1.19.0-cp311-cp311-win32.whl", hash = "sha256:4ccb48faea02503275ae7e79e351434fc43c294c4cb5c4d8bcb7479061396614"}, + {file = "onnxruntime-1.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:9cdc8d311289a84e77722de68bd22b8adfb94eea26f4be6f9e017350faac8b18"}, + {file = "onnxruntime-1.19.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1b59eaec1be9a8613c5fdeaafe67f73a062edce3ac03bbbdc9e2d98b58a30617"}, + {file = "onnxruntime-1.19.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be4144d014a4b25184e63ce7a463a2e7796e2f3df931fccc6a6aefa6f1365dc5"}, + {file = "onnxruntime-1.19.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10d7e7d4ca7021ce7f29a66dbc6071addf2de5839135339bd855c6d9c2bba371"}, + {file = "onnxruntime-1.19.0-cp312-cp312-win32.whl", hash = "sha256:87f2c58b577a1fb31dc5d92b647ecc588fd5f1ea0c3ad4526f5f80a113357c8d"}, + {file = "onnxruntime-1.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:8a1f50d49676d7b69566536ff039d9e4e95fc482a55673719f46528218ecbb94"}, + {file = "onnxruntime-1.19.0-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:71423c8c4b2d7a58956271534302ec72721c62a41efd0c4896343249b8399ab0"}, + {file = "onnxruntime-1.19.0-cp38-cp38-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d63630d45e9498f96e75bbeb7fd4a56acb10155de0de4d0e18d1b6cbb0b358a"}, + {file = "onnxruntime-1.19.0-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3bfd15db1e8794d379a86c1a9116889f47f2cca40cc82208fc4f7e8c38e8522"}, + {file = "onnxruntime-1.19.0-cp38-cp38-win32.whl", hash = "sha256:3b098003b6b4cb37cc84942e5f1fe27f945dd857cbd2829c824c26b0ba4a247e"}, + {file = "onnxruntime-1.19.0-cp38-cp38-win_amd64.whl", hash = "sha256:cea067a6541d6787d903ee6843401c5b1332a266585160d9700f9f0939443886"}, + {file = "onnxruntime-1.19.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:c4fcff12dc5ca963c5f76b9822bb404578fa4a98c281e8c666b429192799a099"}, + {file = "onnxruntime-1.19.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6dcad8a4db908fbe70b98c79cea1c8b6ac3316adf4ce93453136e33a524ac59"}, + {file = "onnxruntime-1.19.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bc449907c6e8d99eee5ae5cc9c8fdef273d801dcd195393d3f9ab8ad3f49522"}, + {file = "onnxruntime-1.19.0-cp39-cp39-win32.whl", hash = "sha256:947febd48405afcf526e45ccff97ff23b15e530434705f734870d22ae7fcf236"}, + {file = "onnxruntime-1.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:f60be47eff5ee77fd28a466b0fd41d7debc42a32179d1ddb21e05d6067d7b48b"}, ] [package.dependencies] coloredlogs = "*" flatbuffers = "*" -numpy = ">=1.21.6,<2.0" +numpy = ">=1.21.6" packaging = "*" protobuf = "*" sympy = "*" @@ -1534,13 +1534,13 @@ tests = ["pytest"] [[package]] name = "pvorca" -version = "0.2.3" +version = "0.2.4" description = "Orca Streaming Text-to-Speech Engine" optional = false python-versions = ">=3.8" files = [ - {file = "pvorca-0.2.3-py3-none-any.whl", hash = "sha256:c6f82cd8543031770ec60d04e2bebee567b976ff5e2f5ba2f7ce6cc5a6ff11cc"}, - {file = "pvorca-0.2.3.tar.gz", hash = "sha256:c75b74ec32615cbf27a8900c4ed447295891e63561beb1002c2c3812a133a616"}, + {file = "pvorca-0.2.4-py3-none-any.whl", hash = "sha256:dc6578300c791e0d2e9a1e0e7c91aa1635ce459eefb6f87ea86033bcdb42d656"}, + {file = "pvorca-0.2.4.tar.gz", hash = "sha256:719a7aeae1736a19750919dbc1117e01d194a553bb6edc0e7dae9d717b2c7ce2"}, ] [[package]] @@ -1554,9 +1554,9 @@ develop = false [package.source] type = "git" -url = "https://github.com/sassanh/pyfakefs.git" -reference = "skip_names" -resolved_reference = "2e8fcfb07496cef91483fa939b3f6caf4f5617aa" +url = "https://github.com/pytest-dev/pyfakefs.git" +reference = "HEAD" +resolved_reference = "c9549665f7b860aac43bb8229c1f6626afcd23df" [[package]] name = "pyftdi" @@ -1614,13 +1614,13 @@ files = [ [[package]] name = "pyright" -version = "1.1.375" +version = "1.1.376" description = "Command line wrapper for pyright" optional = false python-versions = ">=3.7" files = [ - {file = "pyright-1.1.375-py3-none-any.whl", hash = "sha256:4c5e27eddeaee8b41cc3120736a1dda6ae120edf8523bb2446b6073a52f286e3"}, - {file = "pyright-1.1.375.tar.gz", hash = "sha256:7765557b0d6782b2fadabff455da2014476404c9e9214f49977a4e49dec19a0f"}, + {file = "pyright-1.1.376-py3-none-any.whl", hash = "sha256:0f2473b12c15c46b3207f0eec224c3cea2bdc07cd45dd4a037687cbbca0fbeff"}, + {file = "pyright-1.1.376.tar.gz", hash = "sha256:bffd63b197cd0810395bb3245c06b01f95a85ddf6bfa0e5644ed69c841e954dd"}, ] [package.dependencies] @@ -1806,13 +1806,13 @@ typing-extensions = ">=4.10.0,<5.0.0" [[package]] name = "python-redux" -version = "0.15.10" +version = "0.16.0" description = "Redux implementation for Python" optional = false python-versions = "<4.0,>=3.11" files = [ - {file = "python_redux-0.15.10-py3-none-any.whl", hash = "sha256:dbd4b65a26fe7504f94df5fb55df9b94fe8bef3ff42948cffdb0f7818007cf05"}, - {file = "python_redux-0.15.10.tar.gz", hash = "sha256:eb72e931896632e2cc1922f704434f6a52cce9133992be655af6d7e3939ee4e4"}, + {file = "python_redux-0.16.0-py3-none-any.whl", hash = "sha256:9f5bbf1bdf0f8494de809047f056ccbe0f1548ba3125e9d96e90026e0c193390"}, + {file = "python_redux-0.16.0.tar.gz", hash = "sha256:5165ca8dacaca4ba15d62042b67f6c3f1aeb09c446349cadc5c1969ba3bd3d16"}, ] [package.dependencies] @@ -1953,29 +1953,29 @@ files = [ [[package]] name = "ruff" -version = "0.5.7" +version = "0.6.1" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.5.7-py3-none-linux_armv6l.whl", hash = "sha256:548992d342fc404ee2e15a242cdbea4f8e39a52f2e7752d0e4cbe88d2d2f416a"}, - {file = "ruff-0.5.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00cc8872331055ee017c4f1071a8a31ca0809ccc0657da1d154a1d2abac5c0be"}, - {file = "ruff-0.5.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:eaf3d86a1fdac1aec8a3417a63587d93f906c678bb9ed0b796da7b59c1114a1e"}, - {file = "ruff-0.5.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a01c34400097b06cf8a6e61b35d6d456d5bd1ae6961542de18ec81eaf33b4cb8"}, - {file = "ruff-0.5.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcc8054f1a717e2213500edaddcf1dbb0abad40d98e1bd9d0ad364f75c763eea"}, - {file = "ruff-0.5.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f70284e73f36558ef51602254451e50dd6cc479f8b6f8413a95fcb5db4a55fc"}, - {file = "ruff-0.5.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a78ad870ae3c460394fc95437d43deb5c04b5c29297815a2a1de028903f19692"}, - {file = "ruff-0.5.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ccd078c66a8e419475174bfe60a69adb36ce04f8d4e91b006f1329d5cd44bcf"}, - {file = "ruff-0.5.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e31c9bad4ebf8fdb77b59cae75814440731060a09a0e0077d559a556453acbb"}, - {file = "ruff-0.5.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d796327eed8e168164346b769dd9a27a70e0298d667b4ecee6877ce8095ec8e"}, - {file = "ruff-0.5.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4a09ea2c3f7778cc635e7f6edf57d566a8ee8f485f3c4454db7771efb692c499"}, - {file = "ruff-0.5.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a36d8dcf55b3a3bc353270d544fb170d75d2dff41eba5df57b4e0b67a95bb64e"}, - {file = "ruff-0.5.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9369c218f789eefbd1b8d82a8cf25017b523ac47d96b2f531eba73770971c9e5"}, - {file = "ruff-0.5.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b88ca3db7eb377eb24fb7c82840546fb7acef75af4a74bd36e9ceb37a890257e"}, - {file = "ruff-0.5.7-py3-none-win32.whl", hash = "sha256:33d61fc0e902198a3e55719f4be6b375b28f860b09c281e4bdbf783c0566576a"}, - {file = "ruff-0.5.7-py3-none-win_amd64.whl", hash = "sha256:083bbcbe6fadb93cd86709037acc510f86eed5a314203079df174c40bbbca6b3"}, - {file = "ruff-0.5.7-py3-none-win_arm64.whl", hash = "sha256:2dca26154ff9571995107221d0aeaad0e75a77b5a682d6236cf89a58c70b76f4"}, - {file = "ruff-0.5.7.tar.gz", hash = "sha256:8dfc0a458797f5d9fb622dd0efc52d796f23f0a1493a9527f4e49a550ae9a7e5"}, + {file = "ruff-0.6.1-py3-none-linux_armv6l.whl", hash = "sha256:b4bb7de6a24169dc023f992718a9417380301b0c2da0fe85919f47264fb8add9"}, + {file = "ruff-0.6.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:45efaae53b360c81043e311cdec8a7696420b3d3e8935202c2846e7a97d4edae"}, + {file = "ruff-0.6.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:bc60c7d71b732c8fa73cf995efc0c836a2fd8b9810e115be8babb24ae87e0850"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c7477c3b9da822e2db0b4e0b59e61b8a23e87886e727b327e7dcaf06213c5cf"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a0af7ab3f86e3dc9f157a928e08e26c4b40707d0612b01cd577cc84b8905cc9"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:392688dbb50fecf1bf7126731c90c11a9df1c3a4cdc3f481b53e851da5634fa5"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5278d3e095ccc8c30430bcc9bc550f778790acc211865520f3041910a28d0024"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe6d5f65d6f276ee7a0fc50a0cecaccb362d30ef98a110f99cac1c7872df2f18"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2e0dd11e2ae553ee5c92a81731d88a9883af8db7408db47fc81887c1f8b672e"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d812615525a34ecfc07fd93f906ef5b93656be01dfae9a819e31caa6cfe758a1"}, + {file = "ruff-0.6.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:faaa4060f4064c3b7aaaa27328080c932fa142786f8142aff095b42b6a2eb631"}, + {file = "ruff-0.6.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:99d7ae0df47c62729d58765c593ea54c2546d5de213f2af2a19442d50a10cec9"}, + {file = "ruff-0.6.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9eb18dfd7b613eec000e3738b3f0e4398bf0153cb80bfa3e351b3c1c2f6d7b15"}, + {file = "ruff-0.6.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c62bc04c6723a81e25e71715aa59489f15034d69bf641df88cb38bdc32fd1dbb"}, + {file = "ruff-0.6.1-py3-none-win32.whl", hash = "sha256:9fb4c4e8b83f19c9477a8745e56d2eeef07a7ff50b68a6998f7d9e2e3887bdc4"}, + {file = "ruff-0.6.1-py3-none-win_amd64.whl", hash = "sha256:c2ebfc8f51ef4aca05dad4552bbcf6fe8d1f75b2f6af546cc47cc1c1ca916b5b"}, + {file = "ruff-0.6.1-py3-none-win_arm64.whl", hash = "sha256:3bc81074971b0ffad1bd0c52284b22411f02a11a012082a76ac6da153536e014"}, + {file = "ruff-0.6.1.tar.gz", hash = "sha256:af3ffd8c6563acb8848d33cd19a69b9bfe943667f0419ca083f8ebe4224a3436"}, ] [[package]] @@ -2355,4 +2355,4 @@ dev = ["headless-kivy", "headless-kivy"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "0f5704475e7a2dc1de0bbccc8b1c4bb02b0e001b628b49cd9b4e804a3cfde9ae" +content-hash = "06a5a73099dca4f3d036fb53a91bfef83d72ed04e0908063cda34b71adc30212" diff --git a/pyproject.toml b/pyproject.toml index e02815c9..68c7b526 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ platformdirs = "^4.2.0" dill = "^0.3.8" simpleaudio = "^1.0.4" -python-redux = "^0.15.10" +python-redux = "^0.16.0" python-debouncer = "^0.1.5" python-strtobool = "^1.0.0" python-fake = "^0.1.0" @@ -61,18 +61,18 @@ optional = true [tool.poetry.group.dev.dependencies] poethepoet = "^0.24.4" -pyright = "^1.1.374" +pyright = "^1.1.376" pytest = "^8.0.0" pytest-asyncio = "^0.23.5.post1" pytest-cov = "^4.1.0" pytest-timeout = "^2.3.1" pytest-xdist = "^3.5.0" -ruff = "^0.5.5" +ruff = "^0.6.0" tenacity = "^8.2.3" toml = "^0.10.2" pytest-mock = "^3.14.0" ipython = "^8.23.0" -pyfakefs = { git = "https://github.com/sassanh/pyfakefs.git", branch = "skip_names" } +pyfakefs = { git = "https://github.com/pytest-dev/pyfakefs.git" } [tool.poetry.extras] default = ["headless-kivy"] @@ -93,7 +93,14 @@ typecheck = "pyright -p pyproject.toml ." test = "pytest --cov=ubo_app" sanity = ["typecheck", "lint", "test"] build-docker-images = "sh -c 'docker buildx build . -f scripts/Dockerfile.dev -t ubo-app-dev && docker buildx build . -f scripts/Dockerfile.test -t ubo-app-test'" -test-on-device = "scripts/test_on_device.sh" + +[tool.poe.tasks.test-on-device] +args = [ + { name = "copy", type = "boolean" }, + { name = "deps", type = "boolean" }, + { name = "run", type = "boolean" }, +] +cmd = "scripts/test_on_device.sh" [tool.poe.tasks.deploy-to-device] args = [ @@ -110,7 +117,7 @@ target-version = 'py311' [tool.ruff.lint] select = ["ALL"] -ignore = ["INP001", "PLR0911", "D203", "D213", "PLC0415"] +ignore = ["INP001", "PLR0911", "D203", "D213", "PLC0415", "TD003"] fixable = ["ALL"] unfixable = [] logger-objects = ['ubo_app.logging.logger'] @@ -207,7 +214,7 @@ filterwarnings = [ "ignore: setDaemon\\(\\) is deprecated, set the daemon attribute instead:DeprecationWarning", "ignore:'imghdr' is deprecated:DeprecationWarning", ] -timeout = 120 +timeout = 40 [tool.coverage.report] exclude_also = ["if TYPE_CHECKING:"] diff --git a/scripts/deploy.sh b/scripts/deploy.sh index b4f0a598..190cae61 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -4,9 +4,16 @@ set -o errexit set -o pipefail set -o nounset +# Signal handler +function cleanup() { + perl -i -pe 's/^exclude = .*-voice\/models.*\n//' pyproject.toml +} +trap cleanup ERR +trap cleanup EXIT + perl -i -pe 's/^(packages = \[.*)$/\1\nexclude = ["ubo_app\/services\/*-voice\/models\/*"]/' pyproject.toml poetry build -perl -i -pe 's/^exclude = .*-voice\/models.*\n//' pyproject.toml +cleanup LATEST_VERSION=$(basename $(ls -rt dist/*.whl | tail -n 1)) deps=${deps:-"False"} @@ -41,22 +48,21 @@ function run_on_pod_as_root() { scp dist/$LATEST_VERSION ubo-development-pod:/tmp/ -test "$deps" == "True" && run_on_pod "pip install --upgrade /tmp/$LATEST_VERSION[default]" - -run_on_pod "mv /opt/ubo/env/lib/python3.*/site-packages/ubo_app/services/*-voice/models /tmp/ +run_on_pod "$(if [ "$deps" == "True" ]; then echo "pip install --upgrade /tmp/$LATEST_VERSION[default] &&"; fi)\ +mv /opt/ubo/env/lib/python3.*/site-packages/ubo_app/services/*-voice/models /tmp/ pip install --no-index --upgrade --force-reinstal --no-deps /tmp/$LATEST_VERSION[default] mv /tmp/models /opt/ubo/env/lib/python3.*/site-packages/ubo_app/services/*-voice/" -test "$bootstrap" == "True" && - run_on_pod_as_root "/opt/ubo/env/bin/bootstrap; systemctl restart ubo-system.service" - -test "$env" == "True" && - scp ubo_app/.dev.env ubo-development-pod:/tmp/ && - run_on_pod_as_root "chown ubo:ubo /tmp/.dev.env" && - run_on_pod "mv /tmp/.dev.env /opt/ubo/env/lib/python3.11/site-packages/ubo_app/" - -test "$run" == "True" && - run_on_pod "systemctl --user restart ubo-app.service" +if [ "$bootstrap" == "True" ] || [ "$env" == "True" ]; then + run_on_pod_as_root "$(if [ "$bootstrap" == "True" ]; then echo "/opt/ubo/env/bin/bootstrap && systemctl daemon-reload && systemctl restart ubo-system.service &&"; fi)\ +$(if [ "$env" == "True" ]; then echo "cat <<'EOF' > /tmp/.dev.env +$(cat ubo_app/.dev.env) +EOF && +chown ubo:ubo /tmp/.dev.env && +mv /tmp/.dev.env /opt/ubo/env/lib/python3.*/site-packages/ubo_app/"; fi)" +fi -test "$restart" == "True" && - run_on_pod "killall -9 ubo" +if [ "$run" == "True" ] || [ "$restart" == "True" ]; then + run_on_pod "$(if [ "$run" == "True" ]; then echo "systemctl --user restart ubo-app.service &&"; fi)\ +$(if [ "$restart" == "True" ]; then echo "killall -9 ubo"; fi)" +fi diff --git a/scripts/test_on_device.sh b/scripts/test_on_device.sh index 5673dd12..5e8a5143 100755 --- a/scripts/test_on_device.sh +++ b/scripts/test_on_device.sh @@ -4,7 +4,12 @@ set -o errexit set -o pipefail set -o nounset +copy=${copy:-"False"} +deps=${deps:-"False"} +run=${run:-"False"} + function run_on_pod() { + echo $1 if [ $# -lt 1 ]; then echo "Usage: run_on_pod_out_of_env " return 1 @@ -28,21 +33,23 @@ function run_on_pod_as_root() { return 1 } -run_on_pod_as_root "rm -rf /tmp/test-runner" -git ls-files --others --exclude-standard --cached | rsync --info=progress2 -are ssh --files-from=- --ignore-missing-args ./ ubo-development-pod:/tmp/test-runner/ +if [ "$copy" == "True" ]; then + # Since rsync is not called with -r, it treats ./scripts as an empty directory and its content are ignored, it could be any other random directory inside "./". It is needed solely to create the root directory with ubo:ubo ownership. + (echo './scripts'; git ls-files --others --exclude-standard --cached) | rsync --rsync-path="sudo rsync" --delete --info=progress2 -ae ssh --files-from=- --ignore-missing-args ./ ubo-development-pod:/home/ubo/test-runner/ --chown ubo:ubo +fi -run_on_pod_as_root "chown -R ubo:ubo /tmp/test-runner" -run_on_pod "~/.local/bin/poetry --version || curl -L https://install.python-poetry.org | python3 -" -run_on_pod "killall -9 pytest || true" -run_on_pod "rm -rf ~/test-runner &&\ - mv /tmp/test-runner ~/ &&\ +run_on_pod "$(if [ "$copy" == "True" ]; then echo "(~/.local/bin/poetry --version || \ + curl -L https://install.python-poetry.org | python3 -) &&"; fi) + $(if [ "$run" == "True" ]; then echo "(killall -9 pytest || true) &&"; fi)\ cd ~/test-runner &&\ - ~/.local/bin/poetry config virtualenvs.options.system-site-packages true --local &&\ - ~/.local/bin/poetry env use python3.11 &&\ - ~/.local/bin/poetry install --no-interaction --extras=dev --with=dev &&\ - ~/.local/bin/poetry run poe test --make-screenshots -n1 $*; \ - mv ~/test-runner /tmp/" - -rm -rf tests/**/results/ -run_on_pod "find /tmp/test-runner -printf %P\\\\n | grep '^tests/.*/results$'" | rsync --info=progress2 --delete -are ssh --files-from=- --ignore-missing-args ubo-development-pod:/tmp/test-runner ./ + $(if [ "$run" == "True" ] || [ "$deps" == "True" ]; then echo "~/.local/bin/poetry config virtualenvs.options.system-site-packages true --local &&\ + ~/.local/bin/poetry env use python3.11 &&"; fi)\ + $(if [ "$deps" == "True" ]; then echo "~/.local/bin/poetry install --no-interaction --extras=dev --with=dev &&"; fi)\ + $(if [ "$run" == "True" ]; then echo "~/.local/bin/poetry run poe test --verbosity=2 --capture=no --make-screenshots -n1 $* || true &&"; fi)\ + true" + +if [ "$run" == "True" ]; then + rm -rf tests/**/results/ + run_on_pod "find ~/test-runner -printf %P\\\\n | grep '^tests/.*/results$'" | rsync --rsync-path="sudo rsync" --info=progress2 --delete -are ssh --files-from=- --ignore-missing-args ubo-development-pod:/home/ubo/test-runner ./ +fi diff --git a/tests/conftest.py b/tests/conftest.py index 08a41da4..35f9cde3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,10 +69,10 @@ def pytest_addoption(parser: pytest.Parser) -> None: """Add options to the pytest command line.""" parser.addoption('--override-window-snapshots', action='store_true') parser.addoption('--make-screenshots', action='store_true') - parser.addoption('--use-fake-filesystem', action='store_true') + parser.addoption('--use-fakefs', action='store_true') -@pytest.fixture() +@pytest.fixture def snapshot_prefix() -> str: """Return the prefix for the snapshots.""" from ubo_app.utils import IS_RPI diff --git a/tests/fixtures/app.py b/tests/fixtures/app.py index c2910aac..52885f5e 100644 --- a/tests/fixtures/app.py +++ b/tests/fixtures/app.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import contextlib import gc import json import logging @@ -18,9 +17,15 @@ from pyfakefs.fake_filesystem_unittest import Patcher from str_to_bool import str_to_bool -# It needs to be included in `modules_snapshot` -__import__('numpy') -modules_snapshot = set(sys.modules) +modules_snapshot = set(sys.modules).union( + { + 'kivy.cache', + 'numpy', + 'ubo_app.display', + 'ubo_app.utils.monitor_unit', + }, +) + if TYPE_CHECKING: from collections.abc import AsyncGenerator @@ -58,28 +63,33 @@ def set_app(self: AppContext, app: MenuApp) -> None: PERSISTENT_STORE_PATH.parent.mkdir(parents=True, exist_ok=True) PERSISTENT_STORE_PATH.write_text(json.dumps(self.persistent_store_data)) + self.app = app + self.loop = asyncio.get_event_loop() + self.task = self.loop.create_task(self.app.async_run(async_lib='asyncio')) - from ubo_app.service import start_event_loop + from ubo_app.service import start_event_loop_thread - start_event_loop() - self.app = app - loop = asyncio.get_event_loop() - self.task = loop.create_task(self.app.async_run(async_lib='asyncio')) + start_event_loop_thread(asyncio.new_event_loop()) async def clean_up(self: AppContext) -> None: """Clean up the application.""" - from ubo_app.constants import MAIN_LOOP_GRACE_PERIOD + from redux import FinishAction + + import ubo_app.service + from ubo_app.store.main import store + + store.dispatch(FinishAction()) assert hasattr(self, 'task'), 'App not set for test' self.app.stop() + await self.task app_ref = weakref.ref(self.app) self.app.root.clear_widgets() - with contextlib.suppress(asyncio.CancelledError): - await asyncio.sleep(MAIN_LOOP_GRACE_PERIOD + 0.2) + ubo_app.service.worker_thread.is_finished.wait() del self.app del self.task @@ -126,42 +136,43 @@ class ConditionalFSWrapper: def __init__( self: ConditionalFSWrapper, *, - condition: bool, + use_fake_fs: bool, ) -> None: """Initialize the wrapper.""" - self.condition = condition - - # These needs to be imported before setting up fake fs - import coverage - - from ubo_app.utils import IS_RPI - - if IS_RPI: - picamera_skip_modules = [ - 'picamera2', - 'picamera2.allocators.dmaallocator', - 'picamera2.dma_heap', - ] + if use_fake_fs: + # These needs to be imported before setting up fake fs + import coverage + + from ubo_app.utils import IS_RPI + + if IS_RPI: + picamera_skip_modules = [ + 'picamera2', + 'picamera2.allocators.dmaallocator', + 'picamera2.dma_heap', + ] + else: + picamera_skip_modules = [] + import headless_kivy_pytest.fixtures.snapshot + import pyzbar.pyzbar + import redux_pytest.fixtures.snapshot + + self.patcher = Patcher( + additional_skip_names=[ + coverage, + pytest, + pyzbar.pyzbar, + headless_kivy_pytest.fixtures.snapshot, + redux_pytest.fixtures.snapshot, + *picamera_skip_modules, + ], + ) else: - picamera_skip_modules = [] - import headless_kivy_pytest.fixtures.snapshot - import pyzbar.pyzbar - import redux_pytest.fixtures.snapshot - - self.patcher = Patcher( - additional_skip_names=[ - coverage, - pytest, - pyzbar.pyzbar, - headless_kivy_pytest.fixtures.snapshot, - redux_pytest.fixtures.snapshot, - *picamera_skip_modules, - ], - ) + self.patcher = None def __enter__(self: ConditionalFSWrapper) -> Patcher | None: """Enter the context.""" - if self.condition: + if self.patcher: import os real_paths = [ @@ -195,7 +206,7 @@ def __exit__( traceback: TracebackType | None, ) -> None: """Exit the context.""" - if self.condition: + if self.patcher: return self.patcher.__exit__(exc_type, exc_value, traceback) return None @@ -247,7 +258,7 @@ def patched_config_set(category: str, key: str, value: str) -> None: ) -@pytest.fixture() +@pytest.fixture async def app_context( request: SubRequest, _monkeypatch: pytest.MonkeyPatch, @@ -270,35 +281,29 @@ async def app_context( os.environ['TEST_ROOT_PATH'] = Path().absolute().as_posix() should_use_fake_fs = ( request.config.getoption( - '--use-fake-filesystem', + '--use-fakefs', default=cast( Any, - str_to_bool(os.environ.get('UBO_TEST_USE_FAKE_FILESYSTEM', 'false')) - == 1, + str_to_bool(os.environ.get('UBO_TEST_USE_FAKEFS', 'false')) == 1, ), ) is True ) - with ConditionalFSWrapper(condition=should_use_fake_fs) as patcher: + with ConditionalFSWrapper(use_fake_fs=should_use_fake_fs) as patcher: context = AppContext(request) yield context await context.clean_up() - del patcher + del patcher assert not hasattr(context, 'app'), 'App not cleaned up' del context for module_name in set(sys.modules) - modules_snapshot: - if ( - module_name != 'objc' - and 'kivy.cache' not in module_name - and 'sdbus' not in module_name - and 'ubo_app.display' not in module_name - ): + if not module_name.startswith('sdbus'): del sys.modules[module_name] gc.collect() diff --git a/tests/fixtures/load_services.py b/tests/fixtures/load_services.py index 8bc99136..1e45b12b 100644 --- a/tests/fixtures/load_services.py +++ b/tests/fixtures/load_services.py @@ -39,7 +39,7 @@ def __call__( ) -> Coroutine[None, None, None]: ... -@pytest.fixture() +@pytest.fixture def load_services(wait_for: WaitFor) -> Generator[LoadServices, None, None]: """Load services and wait for them to be ready.""" from ubo_app.load_services import REGISTERED_PATHS diff --git a/tests/fixtures/menu.py b/tests/fixtures/menu.py index e5c67bce..1da8db87 100644 --- a/tests/fixtures/menu.py +++ b/tests/fixtures/menu.py @@ -25,18 +25,16 @@ async def __call__( *, label: str, icon: str | None = None, - timeout: int = 5, ) -> None: ... @overload async def __call__( self: WaitForMenuItem, *, icon: str, - timeout: int = 5, ) -> None: ... -@pytest.fixture() +@pytest.fixture def wait_for_menu_item( app_context: AppContext, wait_for: WaitFor, @@ -47,9 +45,8 @@ async def wait_for_menu_item( *, label: str | None = None, icon: str | None = None, - timeout: int = 5, ) -> None: - @wait_for(timeout=timeout, wait=wait_fixed(0.5), run_async=True) + @wait_for(wait=wait_fixed(0.5), run_async=True) def check() -> None: assert not app_context.app.menu_widget._is_transition_in_progress # noqa: SLF001 current_page = app_context.app.menu_widget.current_screen @@ -73,21 +70,19 @@ async def __call__( self: WaitForEmptyMenu, *, placeholder: str | None = None, - timeout: int = 5, ) -> None: """Wait for the placeholder to show up.""" -@pytest.fixture() +@pytest.fixture def wait_for_empty_menu(app_context: AppContext, wait_for: WaitFor) -> WaitForEmptyMenu: """Wait for the placeholder to show up.""" async def wait_for_empty_menu( *, placeholder: str | None = None, - timeout: int = 5, ) -> None: - @wait_for(timeout=timeout, wait=wait_fixed(0.5), run_async=True) + @wait_for(wait=wait_fixed(0.5), run_async=True) def check() -> None: assert not app_context.app.menu_widget._is_transition_in_progress # noqa: SLF001 current_page = app_context.app.menu_widget.current_screen diff --git a/tests/fixtures/mock_camera.py b/tests/fixtures/mock_camera.py index 02bccce8..37a0a85b 100644 --- a/tests/fixtures/mock_camera.py +++ b/tests/fixtures/mock_camera.py @@ -19,7 +19,7 @@ def set_image(self: MockCamera, image_name: str) -> None: ) -@pytest.fixture() +@pytest.fixture def camera() -> MockCamera: """Camera mocking tools.""" return MockCamera() diff --git a/tests/fixtures/stability.py b/tests/fixtures/stability.py index 380b7c94..f1617c26 100644 --- a/tests/fixtures/stability.py +++ b/tests/fixtures/stability.py @@ -7,7 +7,7 @@ import pytest from headless_kivy_pytest.fixtures.snapshot import write_image -from tenacity import RetryError, stop_after_delay, wait_fixed +from tenacity import RetryError, wait_fixed if TYPE_CHECKING: from collections.abc import Callable, Coroutine @@ -25,7 +25,6 @@ class Stability(Protocol): async def __call__( self: Stability, - timeout: float = 5, initial_wait: float = 0.2, attempts: int = 1, wait: float = 0.5, @@ -74,7 +73,7 @@ async def _run( raise -@pytest.fixture() +@pytest.fixture async def stability( store_snapshot: StoreSnapshot, window_snapshot: WindowSnapshot, @@ -84,7 +83,6 @@ async def stability( """Wait for the screen and store to stabilize.""" async def wrapper( - timeout: float = 5, initial_wait: float = 0.2, attempts: int = 1, wait: float = 0.5, @@ -98,7 +96,6 @@ async def wrapper( @wait_for( run_async=True, wait=wait_fixed(wait), - stop=stop_after_delay(timeout), ) def check() -> None: nonlocal latest_window_hash, latest_store_snapshot diff --git a/tests/fixtures/store.py b/tests/fixtures/store.py index 41640a5c..98441d02 100644 --- a/tests/fixtures/store.py +++ b/tests/fixtures/store.py @@ -10,7 +10,7 @@ from redux import Store -@pytest.fixture() +@pytest.fixture def store() -> Store: """Take a snapshot of the store.""" from ubo_app.store.main import store diff --git a/tests/flows/conftest.py b/tests/flows/conftest.py new file mode 100644 index 00000000..6cc0a7b3 --- /dev/null +++ b/tests/flows/conftest.py @@ -0,0 +1,21 @@ +"""Pytest configuration file for flow tests.""" + +from pathlib import Path + +import pytest + +FLOW_TIMEOUT = 150 + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems( + config: pytest.Config, + items: list[pytest.Item], +) -> None: + """Modify the collection of items and set their timeout.""" + _ = config + current_dir = Path(__file__).parent.absolute().as_posix() + for item in items: + test_module = Path(item.fspath).absolute().as_posix() + if test_module.startswith(current_dir): + item.add_marker(pytest.mark.timeout(FLOW_TIMEOUT)) diff --git a/tests/flows/test_wireless.py b/tests/flows/test_wireless.py index 7cb91617..ed08579c 100644 --- a/tests/flows/test_wireless.py +++ b/tests/flows/test_wireless.py @@ -36,13 +36,11 @@ async def test_wireless_flow( load_services: LoadServices, stability: Stability, wait_for: WaitFor, - needs_finish: None, camera: MockCamera, wait_for_menu_item: WaitForMenuItem, wait_for_empty_menu: WaitForEmptyMenu, ) -> None: """Test the wireless flow.""" - _ = needs_finish from ubo_app.menu_app.menu import MenuApp from ubo_app.store.core import ChooseMenuItemByIconEvent, ChooseMenuItemByLabelEvent from ubo_app.store.main import dispatch, store @@ -82,7 +80,7 @@ def store_snapshot_selector(state: RootState) -> WiFiState | None: app_context.set_app(app) load_services(['camera', 'wifi', 'notifications']) - @wait_for(timeout=20.0, wait=wait_fixed(1), run_async=True) + @wait_for(wait=wait_fixed(1), run_async=True) def check_icon(expected_icon: str) -> None: state = store._state # noqa: SLF001 @@ -149,7 +147,7 @@ def check_icon(expected_icon: str) -> None: # Select "Select" to open the wireless connection list and see the new connection dispatch(ChooseMenuItemByLabelEvent(label='Select')) - @wait_for(timeout=20.0, wait=wait_fixed(1), run_async=True) + @wait_for(wait=wait_fixed(1), run_async=True) def check_connections() -> None: state = store._state # noqa: SLF001 @@ -157,7 +155,7 @@ def check_connections() -> None: assert state.wifi.connections is not None await check_connections() - await wait_for_menu_item(label='ubo-test-ssid', icon='󱚽', timeout=20) + await wait_for_menu_item(label='ubo-test-ssid', icon='󱚽') store_snapshot.take(selector=store_snapshot_selector) window_snapshot.take() @@ -165,27 +163,27 @@ def check_connections() -> None: dispatch(ChooseMenuItemByLabelEvent(label='ubo-test-ssid')) # Wait for the "Disconnect" item to show up - await wait_for_menu_item(label='Disconnect', timeout=10) + await wait_for_menu_item(label='Disconnect') await stability() window_snapshot.take() dispatch(ChooseMenuItemByLabelEvent(label='Disconnect')) # Wait for the "Connect" item to show up - await wait_for_menu_item(label='Connect', timeout=10) + await wait_for_menu_item(label='Connect') await check_icon('󰖪') await stability() store_snapshot.take(selector=store_snapshot_selector) window_snapshot.take() dispatch(ChooseMenuItemByLabelEvent(label='Connect')) - await wait_for_menu_item(label='Disconnect', timeout=10) + await wait_for_menu_item(label='Disconnect') await check_icon('󰤨') await stability() store_snapshot.take(selector=store_snapshot_selector) window_snapshot.take() dispatch(ChooseMenuItemByLabelEvent(label='Delete')) - @wait_for(timeout=20.0, wait=wait_fixed(1), run_async=True) + @wait_for(wait=wait_fixed(1), run_async=True) def check_no_connections() -> None: state = store._state # noqa: SLF001 assert state diff --git a/tests/integration/results/test_services/all_services_register/store-rpi-000.jsonc b/tests/integration/results/test_services/all_services_register/store-rpi-000.jsonc index c608b7db..f3fa03a5 100644 --- a/tests/integration/results/test_services/all_services_register/store-rpi-000.jsonc +++ b/tests/integration/results/test_services/all_services_register/store-rpi-000.jsonc @@ -80,7 +80,7 @@ "is_connected": true }, "lightdm": { - "is_active": true, + "is_active": false, "is_enabled": true }, "main": { @@ -467,7 +467,7 @@ 1, 1 ], - "icon": "[color=#008000]󰪥[/color]", + "icon": "[color=#ffff00]󰝦[/color]", "is_short": false, "key": "lightdm", "label": "LightDM", diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 39daad5c..47e3fde0 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -43,6 +43,6 @@ async def test_all_services_register( app = MenuApp() app_context.set_app(app) load_services(ALL_SERVICES_IDS, timeout=10) - await stability(initial_wait=6, attempts=2, timeout=5, wait=2) + await stability(initial_wait=6, attempts=2, wait=2) store_snapshot.take() window_snapshot.take() diff --git a/tests/monkeypatch.py b/tests/monkeypatch.py index 9e3e2540..4372e22a 100644 --- a/tests/monkeypatch.py +++ b/tests/monkeypatch.py @@ -247,7 +247,7 @@ def _monkeypatch_asyncio_socket(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(asyncio, 'open_connection', Fake()) -@pytest.fixture() +@pytest.fixture def _monkeypatch(monkeypatch: pytest.MonkeyPatch) -> None: """Mock external resources.""" random.seed(0) diff --git a/tests/setup.sh b/tests/setup.sh index fefbae19..dc3a4c11 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -9,12 +9,12 @@ fi if [ "$IS_RPI" = true ]; then # Disconnect all active connections - nmcli connection show --active | grep wifi | awk '{print $1}' | while read conn; do + nmcli connection show --active | grep wifi | awk -F " " '{print $1}' | while read conn; do nmcli connection down "$conn" done # Delete all Wi-Fi connections - nmcli connection show | grep wifi | awk '{print $1}' | while read conn; do + nmcli connection show | grep wifi | awk -F " " '{print $1}' | while read conn; do nmcli connection delete "$conn" done diff --git a/ubo_app/load_services.py b/ubo_app/load_services.py index 7cac3e83..9d5a6b32 100644 --- a/ubo_app/load_services.py +++ b/ubo_app/load_services.py @@ -236,6 +236,16 @@ def register( }, ) + from redux import FinishEvent + + from ubo_app.store.main import store + + def stop() -> None: + unsubscribe() + self.stop() + + unsubscribe = store.subscribe_event(FinishEvent, stop) + self.start() def initiate(self: UboServiceThread) -> None: @@ -293,11 +303,6 @@ def run(self: UboServiceThread) -> None: if asyncio.iscoroutine(result): self.loop.create_task(result, name=f'Setup task for {self.label}') - from redux import FinishEvent - - from ubo_app.store.main import store - - store.subscribe_event(FinishEvent, self.stop) self.loop.run_forever() def __repr__(self: UboServiceThread) -> str: @@ -335,6 +340,8 @@ async def shutdown(self: UboServiceThread) -> None: task for task in asyncio.all_tasks(self.loop) if task is not asyncio.current_task(self.loop) + and task.cancelling() == 0 + and not task.done() ] logger.debug( 'Waiting for tasks to finish', @@ -345,9 +352,15 @@ async def shutdown(self: UboServiceThread) -> None: ) if not tasks: break - for task in tasks: - with contextlib.suppress(BaseException): - await asyncio.wait_for(task, timeout=SERVICES_LOOP_GRACE_PERIOD) + with contextlib.suppress(BaseException): + await asyncio.wait_for( + asyncio.gather( + *tasks, + return_exceptions=True, + ), + timeout=SERVICES_LOOP_GRACE_PERIOD, + ) + await asyncio.sleep(0.1) logger.debug('Stopping event loop', extra={'thread_': self}) self.loop.stop() diff --git a/ubo_app/logging.py b/ubo_app/logging.py index 12291f73..6e901e71 100644 --- a/ubo_app/logging.py +++ b/ubo_app/logging.py @@ -14,6 +14,27 @@ VERBOSE = 5 +def handle_circular_references(obj: object, seen: set[int] | None = None) -> object: + if seen is None: + seen = set() + + obj_id = id(obj) + if obj_id in seen: + return None + + seen.add(obj_id) + + if isinstance(obj, dict): + return { + key: handle_circular_references(value, seen) for key, value in obj.items() + } + if isinstance(obj, list): + return [handle_circular_references(item, seen) for item in obj] + if isinstance(obj, tuple): + return tuple(handle_circular_references(item, seen) for item in obj) + return str(obj) + + class UboLogger(logging.getLoggerClass()): def __init__(self: UboLogger, name: str, level: int = logging.NOTSET) -> None: super().__init__(name, level) @@ -82,7 +103,7 @@ def format(self: ExtraFormatter, record: logging.LogRecord) -> str: extra = {k: v for k, v in record.__dict__.items() if k not in self.def_keys} if len(extra) > 0: string += ' - extra: ' + json.dumps( - extra, + handle_circular_references(extra), sort_keys=True, indent=2, default=str, diff --git a/ubo_app/main.py b/ubo_app/main.py index fa4ed9ba..1db6fe90 100644 --- a/ubo_app/main.py +++ b/ubo_app/main.py @@ -1,6 +1,7 @@ # ruff: noqa: D100, D101, D102, D103, D104, D107 from __future__ import annotations +import asyncio import os from pathlib import Path @@ -36,9 +37,9 @@ def main() -> None: setup() - from ubo_app.service import start_event_loop + from ubo_app.service import start_event_loop_thread - start_event_loop() + start_event_loop_thread(asyncio.get_event_loop()) import headless_kivy.config diff --git a/ubo_app/menu_app/menu_central.py b/ubo_app/menu_app/menu_central.py index b2822eaf..7cc97681 100644 --- a/ubo_app/menu_app/menu_central.py +++ b/ubo_app/menu_app/menu_central.py @@ -38,7 +38,6 @@ UpdateManagerSetUpdateServiceStatusAction, ) from ubo_app.utils.async_ import create_task -from ubo_app.utils.monitor_unit import monitor_unit from .home_page import HomePage @@ -126,19 +125,21 @@ def _(is_active: bool) -> None: # noqa: FBT001 ), ) - def check_update(self: MenuWidgetWithHomePage, status: str) -> None: - dispatch( - UpdateManagerSetUpdateServiceStatusAction( - is_active=status in ('active', 'activating', 'reloading'), - ), - ) - def build(self: UboApp) -> Widget | None: root = super().build() self.menu_widget.padding_top = root.ids.header_layout.height self.menu_widget.padding_bottom = root.ids.footer_layout.height - create_task(monitor_unit('ubo-update.service', self.check_update)) + def check_update(status: str) -> None: + dispatch( + UpdateManagerSetUpdateServiceStatusAction( + is_active=status in ('active', 'activating', 'reloading'), + ), + ) + + from ubo_app.utils.monitor_unit import monitor_unit + + create_task(monitor_unit('ubo-update.service', check_update)) return root diff --git a/ubo_app/service.py b/ubo_app/service.py index f9a55ef4..f7b02791 100644 --- a/ubo_app/service.py +++ b/ubo_app/service.py @@ -27,23 +27,12 @@ class WorkerThread(threading.Thread): + loop: asyncio.AbstractEventLoop + def __init__(self: WorkerThread) -> None: super().__init__() - from ubo_app.error_handlers import loop_exception_handler - - try: - self.loop = asyncio.get_event_loop() - except RuntimeError: - self.loop = asyncio.new_event_loop() - - self.loop.set_exception_handler(loop_exception_handler) - - asyncio.set_event_loop(self.loop) - from ubo_app.constants import DEBUG_MODE - - if DEBUG_MODE: - self.loop.set_debug(enabled=True) + self.is_finished = threading.Event() def run(self: WorkerThread) -> None: if not self.loop.is_running(): @@ -72,6 +61,8 @@ async def shutdown(self: WorkerThread) -> None: task for task in asyncio.all_tasks(self.loop) if task is not asyncio.current_task(self.loop) + and task.cancelling() == 0 + and not task.done() ] logger.debug( 'Waiting for tasks to finish', @@ -82,22 +73,37 @@ async def shutdown(self: WorkerThread) -> None: ) if not tasks: break - for task in tasks: - with contextlib.suppress(BaseException): - await asyncio.wait_for(task, timeout=MAIN_LOOP_GRACE_PERIOD) + with contextlib.suppress(BaseException): + await asyncio.wait_for( + asyncio.gather( + *tasks, + return_exceptions=True, + ), + timeout=MAIN_LOOP_GRACE_PERIOD, + ) + await asyncio.sleep(0.1) logger.debug('Stopping event loop', extra={'thread_': self}) self.loop.stop() + self.is_finished.set() def stop(self: WorkerThread) -> None: self.loop.call_soon_threadsafe(self.loop.create_task, self.shutdown()) -thread = WorkerThread() +worker_thread = WorkerThread() + + +def start_event_loop_thread(loop: asyncio.AbstractEventLoop) -> None: + from ubo_app.constants import DEBUG_MODE + from ubo_app.error_handlers import loop_exception_handler + loop.set_exception_handler(loop_exception_handler) + if DEBUG_MODE: + loop.set_debug(enabled=True) -def start_event_loop() -> None: - thread.start() + worker_thread.loop = loop + worker_thread.start() from redux.basic_types import FinishEvent @@ -105,7 +111,7 @@ def start_event_loop() -> None: def stop() -> None: unsubscribe() - thread.loop.call_soon_threadsafe(thread.loop.create_task, thread.shutdown()) + worker_thread.stop() unsubscribe = subscribe_event(FinishEvent, stop) @@ -114,7 +120,7 @@ def _create_task( task: Coroutine, callback: TaskCreatorCallback | None = None, ) -> Handle: - return thread.run_task(task, callback) + return worker_thread.run_task(task, callback) _ = _create_task diff --git a/ubo_app/services/000-audio/setup.py b/ubo_app/services/000-audio/setup.py index 3f0e1e9f..5fa6e4a7 100644 --- a/ubo_app/services/000-audio/setup.py +++ b/ubo_app/services/000-audio/setup.py @@ -26,7 +26,6 @@ def _run_async_in_thread( **kwargs: Args.kwargs, ) -> None: loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) result = loop.run_until_complete(async_func(*args, **kwargs)) loop.close() return result diff --git a/ubo_app/services/030-wifi/setup.py b/ubo_app/services/030-wifi/setup.py index 9d986db8..17c4e9da 100644 --- a/ubo_app/services/030-wifi/setup.py +++ b/ubo_app/services/030-wifi/setup.py @@ -124,7 +124,7 @@ async def init_service() -> None: ), ) - subscribe_event(WiFiUpdateRequestEvent, lambda: create_task(request_scan())) + subscribe_event(WiFiUpdateRequestEvent, request_scan) if not read_from_persistent_store( key='wifi_has_visited_onboarding', diff --git a/ubo_app/setup.py b/ubo_app/setup.py index 1210af8c..87f37cd5 100644 --- a/ubo_app/setup.py +++ b/ubo_app/setup.py @@ -8,7 +8,7 @@ import numpy as np from fake import Fake -from redux import FinishAction +from redux import FinishAction, FinishEvent if TYPE_CHECKING: from ubo_gui.menu.types import Callable @@ -102,19 +102,30 @@ async def fake_create_subprocess_exec( Fake(_Fake__return_value=Fake(_Fake__await_value=(Fake(), Fake()))), ) - import ubo_app.utils.monitor_unit + from ubo_app.utils import monitor_unit async def fake_monitor_unit(unit: str, callback: Callable[[str], None]) -> None: callback('inactive' if unit == 'ubo-update.service' else 'active') - ubo_app.utils.monitor_unit.monitor_unit = fake_monitor_unit + monitor_unit.monitor_unit = fake_monitor_unit + + from kivy.clock import mainthread import ubo_app.display as _ # noqa: F401 + from ubo_app.store.main import subscribe_event + + subscribe_event(FinishEvent, mainthread(clear_signal_handlers)) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) +def clear_signal_handlers() -> None: + """Clear the signal handlers.""" + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + + def signal_handler(signum: int, _: object) -> None: """Handle the signal.""" from ubo_app import display @@ -122,8 +133,7 @@ def signal_handler(signum: int, _: object) -> None: logger.info('Received signal %s, turning off the display...', signum) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGINT, signal.SIG_DFL) + clear_signal_handlers() display.state.turn_off() display.state.pause() diff --git a/ubo_app/system/bootstrap.py b/ubo_app/system/bootstrap.py index 2b02f01f..37087d05 100644 --- a/ubo_app/system/bootstrap.py +++ b/ubo_app/system/bootstrap.py @@ -255,6 +255,16 @@ def bootstrap(*, with_docker: bool = False, in_packer: bool = False) -> None: reload_daemon() enable_services() + # TODO(sassanh): Disable lightdm to disable piwiz to avoid its visual # noqa: FIX002 + # instructions as ubo by nature doesn't need mouse/keyboard, this is a temporary + # solution until we have a better way to handle this. + # Also `check` is `False` because this service is not available in the light image + # and this same code runs for all images. + subprocess.run( # noqa: S603 + ['/usr/bin/env', 'systemctl', 'disable', 'lightdm'], + check=False, + ) + setup_polkit() if with_docker: diff --git a/ubo_app/utils/async_.py b/ubo_app/utils/async_.py index 72063dba..34f9367e 100644 --- a/ubo_app/utils/async_.py +++ b/ubo_app/utils/async_.py @@ -2,74 +2,28 @@ from __future__ import annotations import asyncio -from threading import current_thread from typing import TYPE_CHECKING, ParamSpec from typing_extensions import TypeVar if TYPE_CHECKING: from asyncio import Handle - from collections.abc import Awaitable, Callable + from collections.abc import Callable, Coroutine from redux.basic_types import TaskCreatorCallback -background_tasks: set[Handle] = set() - - def create_task( - awaitable: Awaitable, + task: Coroutine, callback: TaskCreatorCallback | None = None, ) -> Handle: - async def wrapper() -> None: - from ubo_app.load_services import UboServiceThread - from ubo_app.logging import get_logger - - logger = get_logger('ubo-app') - - try: - thread = current_thread() - logger.verbose( - 'Starting task', - extra={ - 'awaitable': awaitable, - 'thread_': thread, - **( - { - 'ubo_service_path': thread.path.as_posix(), - 'ubo_service_label': thread.label, - } - if isinstance(thread, UboServiceThread) - else {} - ), - }, - ) - await awaitable - except Exception: - task = asyncio.current_task() - thread = current_thread() - logger.exception( - 'Task failed', - extra={ - 'awaitable': awaitable, - 'task': task, - 'thread_': thread, - **( - { - 'ubo_service_path': thread.path.as_posix(), - 'ubo_service_label': thread.label, - } - if isinstance(thread, UboServiceThread) - else {} - ), - }, - ) - import ubo_app.service - handle = ubo_app.service._create_task(wrapper(), callback) # noqa: SLF001 - background_tasks.add(handle) - return handle + def callback_(task: asyncio.Task) -> None: + if callback: + callback(task) + + return ubo_app.service._create_task(task, callback_) # noqa: SLF001 T = TypeVar('T', infer_variance=True) diff --git a/ubo_app/utils/bus_provider.py b/ubo_app/utils/bus_provider.py index e5a27fdf..bc573f92 100644 --- a/ubo_app/utils/bus_provider.py +++ b/ubo_app/utils/bus_provider.py @@ -4,11 +4,28 @@ from __future__ import annotations from threading import current_thread +from typing import TYPE_CHECKING +from redux import FinishEvent from sdbus import SdBus, sd_bus_open_system, sd_bus_open_user, set_default_bus -system_buses = {} -user_buses = {} +from ubo_app.store.main import subscribe_event + +if TYPE_CHECKING: + from headless_kivy.config import Thread + +system_buses: dict[Thread, SdBus] = {} +user_buses: dict[Thread, SdBus] = {} + + +def clean_up() -> None: + """Cleanup the buses.""" + for bus in system_buses.values(): + bus.close() + system_buses.clear() + for bus in user_buses.values(): + bus.close() + user_buses.clear() def get_system_bus() -> SdBus: @@ -26,3 +43,6 @@ def get_user_bus() -> SdBus: if thread not in user_buses: user_buses[thread] = sd_bus_open_user() return user_buses[thread] + + +subscribe_event(FinishEvent, clean_up, keep_ref=False) diff --git a/ubo_app/utils/persistent_store.py b/ubo_app/utils/persistent_store.py index ab94b930..a6d3b13b 100644 --- a/ubo_app/utils/persistent_store.py +++ b/ubo_app/utils/persistent_store.py @@ -41,7 +41,11 @@ async def write(value: T) -> None: current_state[key] = serialized_value Path(PERSISTENT_STORE_PATH).write_text(json.dumps(current_state, indent=2)) - subscribe_event(FinishEvent, write.unsubscribe) + def unsubscribe() -> None: + unsubscribe_event() + write.unsubscribe() + + unsubscribe_event = subscribe_event(FinishEvent, unsubscribe) @overload