From 898814ed1036895bdfabbeaea2a5d9d584738014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Kucmus?= Date: Fri, 11 Oct 2024 23:02:58 +0200 Subject: [PATCH 1/5] Real Cold Start and Logging improvements - Introduce new CI/CD - Cleanup and apply new tool configs in pyproject.toml - Improve the CLI invocation, add log silencing - Stop loading lambda handlers on Smyth's start - this was introduced in 0.5 and is now reverted as it caused problems with multi threading libraries running in the global context of the actual lambda - Remove fake_coldstart since now the handler is actually loading on a cold start - nothing to fake anymore - Fix internal and confusing variable naming - Add stricter typing with `check_untyped_defs` - Add setproctitle for better visibility in tools like top, htop, btop - Merge the runner.py module into the RunnerProcess class - Log application startup errors (catch lifespan startup erros) - Add a log column indicating the issuer of a log (Uvicorn, Smyth or Smyth's suprocesses) --- .github/workflows/build.yml | 70 +++++++++ .github/workflows/code_quality.yaml | 73 --------- .github/workflows/deploy_docs.yaml | 3 + .github/workflows/prepare_release.yml | 42 +++++ .github/workflows/publish.yml | 43 +++--- .github/workflows/test.yml | 56 +++++++ LICENCE => LICENSE.txt | 0 README.md | 6 + docs/user_guide/all_settings.md | 4 - docs/user_guide/concurrency.md | 1 - pyproject.toml | 82 +++++----- src/smyth/__about__.py | 1 + src/smyth/__main__.py | 104 ++++++------- src/smyth/config.py | 25 ++- src/smyth/context.py | 11 +- src/smyth/exceptions.py | 22 ++- src/smyth/runner/fake_context.py | 2 +- src/smyth/runner/process.py | 164 +++++++++++++++++--- src/smyth/runner/runner.py | 116 -------------- src/smyth/server/app.py | 10 +- src/smyth/server/endpoints.py | 22 ++- src/smyth/smyth.py | 45 +++--- src/smyth/types.py | 13 +- src/smyth/utils.py | 211 ++++++++++++++++++++++++++ tests/conftest.py | 8 +- tests/test_context.py | 5 +- 26 files changed, 760 insertions(+), 379 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/code_quality.yaml create mode 100644 .github/workflows/prepare_release.yml create mode 100644 .github/workflows/test.yml rename LICENCE => LICENSE.txt (100%) create mode 100644 src/smyth/__about__.py delete mode 100644 src/smyth/runner/runner.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..a4937aa --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,70 @@ +name: Build + +on: + workflow_call: + workflow_dispatch: + +env: + PYTHONUNBUFFERED: "1" + FORCE_COLOR: "1" + +jobs: + test: + uses: ./.github/workflows/test.yml + permissions: + contents: read + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + needs: + - test + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + + - name: Install Hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + + - name: Set package version from tag + run: hatch version $(git describe --tags --always) + + - name: Build package + run: hatch build + + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + sign-release: + name: >- + Sign the Python 🐍 distribution 📦 with Sigstore + needs: + - build + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: mandatory for sigstore + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v2.1.1 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Store the signature files + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + overwrite: true diff --git a/.github/workflows/code_quality.yaml b/.github/workflows/code_quality.yaml deleted file mode 100644 index 7ecf82f..0000000 --- a/.github/workflows/code_quality.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: Code Quality - -on: - push: - -jobs: - format-check: - name: Format Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version-file: pyproject.toml - - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc - - - name: Run checks - run: | - hatch fmt --check - type-check: - name: Type Check - needs: [format-check] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version-file: pyproject.toml - - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc - - - name: Run type checks - run: | - hatch run types:check - unit-test: - name: Unit Test - needs: [format-check, type-check] - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12"] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc - - - name: Run tests - if: ${{ matrix.python-version != '3.12' }} - run: | - hatch test -i python=${{ matrix.python-version }} - - - name: Run tests with coverage - if: ${{ matrix.python-version == '3.12' }} - run: | - hatch test --cover -i python=${{ matrix.python-version }} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ !contains(github.ref, 'release/')}} diff --git a/.github/workflows/deploy_docs.yaml b/.github/workflows/deploy_docs.yaml index 14dd6da..565c5fd 100644 --- a/.github/workflows/deploy_docs.yaml +++ b/.github/workflows/deploy_docs.yaml @@ -4,6 +4,9 @@ on: push: branches: - main + - master + workflow_call: + workflow_dispatch: jobs: docs-publish: diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml new file mode 100644 index 0000000..7ab3f28 --- /dev/null +++ b/.github/workflows/prepare_release.yml @@ -0,0 +1,42 @@ +name: Prepare release + +on: + push: + tags: + - '*' + workflow_call: + workflow_dispatch: + +env: + PYTHONUNBUFFERED: "1" + FORCE_COLOR: "1" + +jobs: + build: + uses: ./.github/workflows/build.yml + permissions: + contents: read + id-token: write # IMPORTANT: mandatory for sigstore in build.yml + + create-release: + needs: + - build + runs-on: ubuntu-latest + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Release + id: create-draft-release + uses: softprops/action-gh-release@v2 + with: + files: | + ./dist/* + draft: true + - name: Summary + run: | + echo "# Release summary" >> $GITHUB_STEP_SUMMARY + echo "Url: ${{ steps.create-draft-release.outputs.url }}" >> $GITHUB_STEP_SUMMARY + echo "You can now publish the release on GitHub" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 403862b..42109ab 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,29 +4,38 @@ on: release: types: - published + workflow_call: + workflow_dispatch: + +env: + PYTHONUNBUFFERED: "1" + FORCE_COLOR: "1" jobs: - pypi-publish: - name: upload release to PyPI + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes runs-on: ubuntu-latest - environment: release + environment: + name: pypi + url: https://pypi.org/p/smyth permissions: # IMPORTANT: this permission is mandatory for trusted publishing id-token: write - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version-file: pyproject.toml - - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc - - - name: Build package - run: hatch build + contents: read + steps: + - name: Download all the dists from the release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release download + '${{ github.ref_name }}' + --dir dist/ + --pattern * + --repo '${{ github.repository }}' - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..0ffdedf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,56 @@ +name: Test + +on: + push: + branches: + - main + - master + pull_request: + workflow_call: + workflow_dispatch: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PYTHONUNBUFFERED: "1" + FORCE_COLOR: "1" + +jobs: + run: + name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + # - windows-latest + # - macos-latest + + python-version: + - "3.10" + - "3.11" + - "3.12" + + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + + - name: Run static analysis + run: hatch fmt --check + + - name: Run type checking + run: hatch run types:check + + - name: Run tests + run: hatch test -c -py ${{ matrix.python-version }} diff --git a/LICENCE b/LICENSE.txt similarity index 100% rename from LICENCE rename to LICENSE.txt diff --git a/README.md b/README.md index 78cd555..dc3ef03 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # Smyth +[![docs](https://img.shields.io/badge/Docs-Smyth-f5c03b.svg?style=flat&logo=materialformkdocs)](https://mirumee.github.io/smyth/) +![pypi](https://img.shields.io/pypi/v/smyth?style=flat) +![licence](https://img.shields.io/pypi/l/smyth?style=flat) +![pypi downloads](https://img.shields.io/pypi/dm/smyth?style=flat) +![pyversion](https://img.shields.io/pypi/pyversions/smyth?style=flat) + Smyth is a versatile tool designed to enhance your AWS Lambda development experience. It is a pure Python tool that allows for easy customization and state persistence, making your Lambda development more efficient and developer-friendly. ## Features diff --git a/docs/user_guide/all_settings.md b/docs/user_guide/all_settings.md index cf64861..9ac29e6 100644 --- a/docs/user_guide/all_settings.md +++ b/docs/user_guide/all_settings.md @@ -42,10 +42,6 @@ Here's a list of all the settings, including those that are simpler but equally `context_data_function_path` - `str` (default: `"smyth.context.generate_context_data"`) A function similar to the [event generator](event_functions.md), but it constructs the `context`, adding some metadata from Smyth's runtime. You can create and use your own. -### Fake Coldstart - -`fake_coldstart` - `bool` (default: `False`) Makes the subprocess `time.sleep` for a random time between 0.5 and 1 second when a subprocess is cold, imitating the longer first response time of real Lambdas. - ### Log Level `log_level` - `str` (default: `"INFO"`) Log level for Smyth's runner function, which is still part of Smyth but already running in the subprocess. Note that the logging of your Lambda handler code should be set separately. diff --git a/docs/user_guide/concurrency.md b/docs/user_guide/concurrency.md index 00039b2..7e5715f 100644 --- a/docs/user_guide/concurrency.md +++ b/docs/user_guide/concurrency.md @@ -30,7 +30,6 @@ concurrency = 2 handler_path = "smyth_test_app.handlers.product_handler" url_path = "/products/{path:path}" concurrency = 2 -fake_coldstart = true strategy_function_path = "smyth.runner.strategy.round_robin" ``` diff --git a/pyproject.toml b/pyproject.toml index 70a7ddf..8663390 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,11 +4,11 @@ build-backend = "hatchling.build" [project] name = "smyth" -version = "0.5.0" +dynamic = ["version"] description = '' readme = "README.md" requires-python = ">=3.10" -license = "MIT" +license = { file = "LICENSE.txt" } keywords = [] authors = [{ name = "Mirumee", email = "it@mirumee.com" }] classifiers = [ @@ -32,37 +32,52 @@ dependencies = [ "toml", "pydantic", "rich", + "click", "asgiref", "typer", + "setproctitle", ] +[project.urls] +Documentation = "https://mirumee.github.io/smyth/" +Issues = "https://github.com/mirumee/smyth/issues" +Source = "https://github.com/mirumee/smyth" + [project.optional-dependencies] dev = ["ipdb"] types = ["mypy>=1.0.0", "pytest", "types-toml", "pytest-asyncio"] -docs = ["mkdocs-material", "termynal"] +docs = ["mkdocs-material"] + +[tool.hatch.version] +path = "src/smyth/__about__.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/smyth"] + +# Environment configuration + +## Default environment [tool.hatch.envs.default] features = ["dev", "types", "docs"] -[project.urls] -Documentation = "https://mirumee.github.io/smyth/" -Issues = "https://github.com/mirumee/smyth/issues" -Source = "https://github.com/mirumee/smyth" +[tool.hatch.envs.default.scripts] +check = ["hatch fmt", "hatch test -a", "hatch test --cover", "hatch run types:check"] -[project.scripts] -smyth = "smyth.__main__:app" +## Types environment + +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/smyth tests}" + +[tool.mypy] +check_untyped_defs = true + +[[tool.mypy.overrides]] +module = "setproctitle.*" +ignore_missing_imports = true -[tool.hatch.envs.default.scripts] -check = [ - "hatch fmt", - "hatch test -a", - "hatch test --cover", - "hatch run types:check", -] -cov-html = ["hatch test --cover -- --cov-report=html"] -[tool.hatch.envs.hatch-static-analysis] -config-path = "ruff.toml" +## Test environment [tool.hatch.envs.hatch-test] dependencies = [ @@ -72,28 +87,28 @@ dependencies = [ "pytest-memray", "pytest-print", "pytest-cov", + "coverage[toml]", ] [[tool.hatch.envs.hatch-test.matrix]] python = ["3.10", "3.11", "3.12"] -[tool.hatch.envs.types.scripts] -check = "mypy --install-types --non-interactive {args:src/smyth tests}" +## Docs environment [tool.hatch.envs.docs.scripts] build = "mkdocs build --clean --strict" serve = "mkdocs serve --dev-addr localhost:8000" deploy = "mkdocs gh-deploy --force" -[tool.hatch.envs.coverage] -detached = true -dependencies = ["coverage[toml]>=6.2"] -[tool.hatch.envs.coverage.scripts] -combine = "coverage combine {args}" -html = "coverage html --skip-covered --skip-empty" +# Tool configuration -[tool.hatch.build.targets.wheel] -packages = ["src/smyth"] +## Pytest configuration + +[tool.pytest.ini_options] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" + +## Coverage configuration [tool.coverage.run] source_pkgs = ["smyth"] @@ -102,7 +117,6 @@ parallel = true [tool.coverage.paths] smyth = ["src/smyth"] -tests = ["tests"] [tool.coverage.report] exclude_lines = [ @@ -111,7 +125,10 @@ exclude_lines = [ "if TYPE_CHECKING:", "@abstract", ] -# fail_under = 90 # TODO: Uncomment when coverage is good enough +omit = ["*/__about__.py", "*/__main__.py", "*/cli/__init__.py"] +# fail_under = 90 # TODO: Uncomment when coverage is good enough (0.9) + +## Ruff configuration [tool.ruff] line-length = 88 @@ -137,6 +154,3 @@ known-first-party = ["smyth"] [tool.ruff.lint.flake8-pytest-style] fixture-parentheses = false mark-parentheses = false - -[tool.pytest.ini_options] -asyncio_mode = "auto" diff --git a/src/smyth/__about__.py b/src/smyth/__about__.py new file mode 100644 index 0000000..c618245 --- /dev/null +++ b/src/smyth/__about__.py @@ -0,0 +1 @@ +__version__ = "0.0.1.dev0" # This is overwritten by Hatch in CI/CD, don't change it. diff --git a/src/smyth/__main__.py b/src/smyth/__main__.py index b8e10c5..e5e4f91 100644 --- a/src/smyth/__main__.py +++ b/src/smyth/__main__.py @@ -1,89 +1,71 @@ import logging +import logging.config +import os +from enum import Enum from typing import Annotated, Optional import typer import uvicorn +from setproctitle import setproctitle -from smyth.config import get_config, get_config_dict +from smyth.config import get_config, get_config_dict, serialize_config +from smyth.utils import get_logging_config app = typer.Typer() config = get_config(get_config_dict()) -logging_config = { - "version": 1, - "disable_existing_loggers": False, - "filters": { - "smyth_api_filter": { - "()": "smyth.utils.SmythStatusRouteFilter", - "smyth_path_prefix": config.smyth_path_prefix, - }, - }, - "handlers": { - "console": { - "class": "rich.logging.RichHandler", - "formatter": "default", - "markup": True, - "rich_tracebacks": True, - "filters": ["smyth_api_filter"], - }, - }, - "formatters": { - "default": { - "format": "[%(processName)s] %(message)s", - "datefmt": "[%X]", - }, - }, - "loggers": { - "smyth": { - "handlers": ["console"], - "level": config.log_level, - "propagate": False, - }, - "uvicorn": { - "handlers": ["console"], - "level": config.log_level, - "propagate": False, - }, - }, -} -logging.config.dictConfig(logging_config) + LOGGER = logging.getLogger(__name__) +class LogLevel(str, Enum): + DEBUG = "DEBUG" + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + CRITICAL = "CRITICAL" + + @app.command() def run( + # typer does not handle union types smyth_starlette_app: Annotated[ - Optional[str], typer.Argument() # noqa: UP007 - ] = None, # typer does not handle union types - smyth_starlette_app_factory: Annotated[ - str, typer.Argument() + str, + typer.Argument( + help="Path to your SmythStarlette app (or factory with --factory)" + ), # noqa: UP007 ] = "smyth.server.app:create_app", + factory: Annotated[bool, typer.Option(help="Use factory for app creation")] = True, host: Annotated[Optional[str], typer.Option()] = config.host, # noqa: UP007 port: Annotated[Optional[int], typer.Option()] = config.port, # noqa: UP007 - log_level: Annotated[Optional[str], typer.Option()] = config.log_level, # noqa: UP007 + log_level: Annotated[ + Optional[LogLevel], # noqa: UP007 + typer.Option( + help=( + "Override the log level specified in the configuration, " + "only for the main process" + ) + ), + ] = LogLevel(config.log_level), + quiet: Annotated[ + bool, typer.Option(help="Effectively the same as --log-level=ERROR") + ] = False, ): - if smyth_starlette_app and smyth_starlette_app_factory: - raise typer.BadParameter( - "Only one of smyth_starlette_app or smyth_starlette_app_factory " - "should be provided." - ) - - factory = False - if smyth_starlette_app_factory: - smyth_starlette_app = smyth_starlette_app_factory - factory = True - if not smyth_starlette_app: - raise typer.BadParameter( - "One of smyth_starlette_app or smyth_starlette_app_factory " - "should be provided." - ) - if host: config.host = host if port: config.port = port if log_level: - config.log_level = log_level + config.log_level = log_level.value + if quiet: + config.log_level = "ERROR" + + logging_config = get_logging_config( + log_level=config.log_level, filter_path_prefix=config.smyth_path_prefix + ) + + setproctitle("smyth") + os.environ["__SMYTH_CONFIG"] = serialize_config(config) uvicorn.run( smyth_starlette_app, diff --git a/src/smyth/config.py b/src/smyth/config.py index 9206a22..3bfbf2b 100644 --- a/src/smyth/config.py +++ b/src/smyth/config.py @@ -1,4 +1,6 @@ -from dataclasses import dataclass, field +import json +import os +from dataclasses import asdict, dataclass, field from pathlib import Path import toml @@ -13,8 +15,7 @@ class HandlerConfig: timeout: float | None = None event_data_function_path: str = "smyth.event.generate_api_gw_v2_event_data" context_data_function_path: str = "smyth.context.generate_context_data" - fake_coldstart: bool = False - log_level: str = "INFO" + log_level: str = "DEBUG" concurrency: int = 1 strategy_generator_path: str = "smyth.runner.strategy.first_warm" @@ -27,11 +28,14 @@ class Config: log_level: str = "INFO" smyth_path_prefix: str = "/smyth" - def __post_init__(self): - self.handlers = { + @classmethod + def from_dict(cls, config_dict: dict): + handler_data = config_dict.pop("handlers") + handlers = { handler_name: HandlerConfig(**handler_config) - for handler_name, handler_config in self.handlers.items() + for handler_name, handler_config in handler_data.items() } + return cls(**config_dict, handlers=handlers) def get_config_file_path(file_name: str = "pyproject.toml") -> Path: @@ -56,4 +60,11 @@ def get_config_dict(config_file_name: str | None = None) -> dict: def get_config(config_dict: dict) -> Config: """Get config.""" - return Config(**config_dict["tool"]["smyth"]) + if environ_config := os.environ.get("__SMYTH_CONFIG"): + config_data = json.loads(environ_config) + return Config.from_dict(config_data) + return Config.from_dict(config_dict["tool"]["smyth"]) + + +def serialize_config(config: Config) -> str: + return json.dumps(asdict(config)) diff --git a/src/smyth/context.py b/src/smyth/context.py index 571bf8c..2432814 100644 --- a/src/smyth/context.py +++ b/src/smyth/context.py @@ -7,13 +7,12 @@ async def generate_context_data( - request: Request | None, handler: SmythHandler, process: RunnerProcessProtocol + request: Request | None, smyth_handler: SmythHandler, process: RunnerProcessProtocol ): """ The data returned by this function is passed to the `smyth.runner.FaneContext` as kwargs. """ - asdict(handler) context: dict[str, Any] = { "smyth": { "process": { @@ -23,11 +22,11 @@ async def generate_context_data( "last_used_timestamp": process.last_used_timestamp, }, "handler": { - "name": handler.name, - "handler_config": asdict(handler), + "name": smyth_handler.name, + "smyth_handler_config": asdict(smyth_handler), }, } } - if handler.timeout is not None: - context["timeout"] = handler.timeout + if smyth_handler.timeout is not None: + context["timeout"] = smyth_handler.timeout return context diff --git a/src/smyth/exceptions.py b/src/smyth/exceptions.py index c879e11..7a018b7 100644 --- a/src/smyth/exceptions.py +++ b/src/smyth/exceptions.py @@ -1,12 +1,12 @@ -class LambdaRuntimeError(Exception): +class SmythRuntimeError(Exception): """Generic Lambda Runtime exception.""" -class ConfigFileNotFoundError(LambdaRuntimeError): +class ConfigFileNotFoundError(SmythRuntimeError): """Config file not found.""" -class DispatcherError(Exception): +class DispatcherError(SmythRuntimeError): pass @@ -22,9 +22,17 @@ class DestroyedOnLoadError(DispatcherError): pass -class LambdaTimeoutError(DispatcherError): - pass +class SubprocessError(SmythRuntimeError): + """Generic subprocess exception.""" -class LambdaInvocationError(DispatcherError): - pass +class LambdaHandlerLoadError(SubprocessError): + """Error loading a Lambda handler.""" + + +class LambdaTimeoutError(SubprocessError): + """Lambda timeout.""" + + +class LambdaInvocationError(SubprocessError): + """Error invoking a Lambda.""" diff --git a/src/smyth/runner/fake_context.py b/src/smyth/runner/fake_context.py index 90a26a4..2126903 100644 --- a/src/smyth/runner/fake_context.py +++ b/src/smyth/runner/fake_context.py @@ -13,7 +13,7 @@ def __init__(self, name="Fake", version="LATEST", timeout=6, **kwargs): for key, value in kwargs.items(): setattr(self, key, value) - def get_remaining_time_in_millis(self): + def get_remaining_time_in_millis(self) -> int: # type: ignore[override] return int( max( (self.timeout * 1000) diff --git a/src/smyth/runner/process.py b/src/smyth/runner/process.py index f30ae27..b4df41a 100644 --- a/src/smyth/runner/process.py +++ b/src/smyth/runner/process.py @@ -1,14 +1,27 @@ +import inspect import logging -from multiprocessing import Process, Queue +import logging.config +import signal +import sys +import traceback +from multiprocessing import Process, Queue, set_start_method from queue import Empty from time import time from asgiref.sync import sync_to_async +from setproctitle import setproctitle -from smyth.exceptions import LambdaInvocationError -from smyth.runner.runner import lambda_invoker +from smyth.exceptions import ( + LambdaHandlerLoadError, + LambdaInvocationError, + LambdaTimeoutError, + SubprocessError, +) +from smyth.runner.fake_context import FakeLambdaContext from smyth.types import LambdaHandler, RunnerMessage, SmythHandlerState +from smyth.utils import get_logging_config, import_attribute +set_start_method("spawn", force=True) LOGGER = logging.getLogger(__name__) @@ -18,9 +31,7 @@ class RunnerProcess(Process): last_used_timestamp: float state: SmythHandlerState - def __init__( - self, name: str, lambda_handler: LambdaHandler, log_level: str = "INFO" - ): + def __init__(self, name: str, lambda_handler_path: str, log_level: str = "INFO"): self.name = name self.task_counter = 0 self.last_used_timestamp = 0 @@ -28,24 +39,20 @@ def __init__( self.input_queue: Queue[RunnerMessage] = Queue(maxsize=1) self.output_queue: Queue[RunnerMessage] = Queue(maxsize=1) + + self.lambda_handler_path = lambda_handler_path + self.log_level = log_level super().__init__( - target=lambda_invoker, name=name, - kwargs={ - "lambda_handler": lambda_handler, - "input_queue": self.input_queue, - "output_queue": self.output_queue, - "log_level": log_level, - }, ) def stop(self): + self.input_queue.put({"type": "smyth.stop"}) + self.join() self.input_queue.close() self.output_queue.close() self.input_queue.join_thread() self.output_queue.join_thread() - self.terminate() - self.join() def send(self, data) -> RunnerMessage | None: LOGGER.debug("Sending data to process %s: %s", self.name, data) @@ -54,6 +61,14 @@ def send(self, data) -> RunnerMessage | None: self.input_queue.put(data) while True: + if not self.is_alive(): + LOGGER.error( + "Process is not alive, this should generally not happen. " + "Restart Smyth and check your configuration. " + "This often happens when the handler can't be loaded " + "(i.e. an exception is raised when importing the handler)." + ) + raise SubprocessError("Process is not alive") try: message = self.output_queue.get(block=True, timeout=1) except Empty: @@ -66,11 +81,126 @@ def send(self, data) -> RunnerMessage | None: if message["type"] == "smyth.lambda.status": self.state = SmythHandlerState(message["status"]) elif message["type"] == "smyth.lambda.response": + self.state = SmythHandlerState.WARM return message["response"] elif message["type"] == "smyth.lambda.error": - LOGGER.error("Error invoking lambda: %s", message) - raise LambdaInvocationError(message["response"]["message"]) + self.state = SmythHandlerState.WARM + if message["response"]["type"] == "LambdaTimeoutError": + raise LambdaTimeoutError(message["response"]["message"]) + else: + raise LambdaInvocationError(message["response"]["message"]) @sync_to_async(thread_sensitive=False) def asend(self, data) -> RunnerMessage | None: return self.send(data) + + # Backend + + def run(self): + setproctitle(f"smyth:{self.name}") + logging.config.dictConfig(get_logging_config(self.log_level)) + self.lambda_invoker__() + + def get_message__(self): + while True: + try: + message = self.input_queue.get(block=True, timeout=1) + except KeyboardInterrupt: + LOGGER.debug("Stopping process") + return + except Empty: + continue + else: + LOGGER.debug("Received message: %s", message) + if message["type"] == "smyth.stop": + LOGGER.debug("Stopping process") + return + yield message + + def get_event__(self, message): + return message["event"] + + def get_context__(self, message): + return FakeLambdaContext(**message["context"]) + + def import_handler__(self, lambda_handler_path, event, context): + LOGGER.info("Starting cold, importing '%s'", lambda_handler_path) + try: + handler = import_attribute(lambda_handler_path) + except ImportError as error: + raise LambdaHandlerLoadError( + f"Error importing handler: {error}, module not found" + ) from error + except AttributeError as error: + raise LambdaHandlerLoadError( + f"Error importing handler: {error}, attribute in module not found" + ) from error + + sig = inspect.signature(handler) + try: + sig.bind(event, context) + except TypeError: + LOGGER.warning( + "Handler signature does not match event and context, " + "using `event` and `context` as parameters." + ) + return handler + + def set_status__(self, status: SmythHandlerState): + self.output_queue.put({"type": "smyth.lambda.status", "status": status}) + + @staticmethod + def timeout_handler__(signum, frame): + raise LambdaTimeoutError("Lambda timeout") + + def lambda_invoker__(self): + sys.stdin = open("/dev/stdin") + lambda_handler: LambdaHandler | None = None + self.set_status__(SmythHandlerState.COLD) + + for message in self.get_message__(): + if message.get("type") != "smyth.lambda.invoke": + LOGGER.error("Invalid message type: %s", message.get("type")) + continue + + event = self.get_event__(message) + context = self.get_context__(message) + + if not lambda_handler: + lambda_handler = self.import_handler__( + self.lambda_handler_path, + event, + context, + ) + self.set_status__(SmythHandlerState.WARM) + + signal.signal(signal.SIGALRM, self.timeout_handler__) + signal.alarm(int(context.timeout)) + self.set_status__(SmythHandlerState.WORKING) + try: + response = lambda_handler(event, context) + except Exception as error: + LOGGER.exception( + "Error invoking lambda: %s", + error, + extra={"log_setting": "console_full_width"}, + ) + self.output_queue.put( + { + "type": "smyth.lambda.error", + "response": { + "type": type(error).__name__, + "message": str(error), + "stacktrace": traceback.format_exc(), + }, + } + ) + else: + self.output_queue.put( + { + "type": "smyth.lambda.response", + "response": response, + } + ) + finally: + signal.alarm(0) diff --git a/src/smyth/runner/runner.py b/src/smyth/runner/runner.py deleted file mode 100644 index 826dd03..0000000 --- a/src/smyth/runner/runner.py +++ /dev/null @@ -1,116 +0,0 @@ -import logging -import signal -import sys -import traceback -from logging.config import dictConfig -from multiprocessing import Queue -from queue import Empty -from random import randint -from time import sleep - -from smyth.runner.fake_context import FakeLambdaContext -from smyth.types import LambdaHandler - - -def configure_logging(log_level: str): - dictConfig( - { - "version": 1, - "disable_existing_loggers": False, - "handlers": { - "console": { - "class": "rich.logging.RichHandler", - "formatter": "default", - "markup": True, - "rich_tracebacks": True, - }, - }, - "formatters": { - "default": { - "format": "[[bold red]%(processName)s[/]] %(message)s", - "datefmt": "[%X]", - }, - }, - "loggers": { - "smyth": { - "level": log_level, - "propagate": False, - "handlers": ["console"], - }, - }, - } - ) - - -LOGGER = logging.getLogger(__name__) - - -def timeout_handler(signum, frame): - raise Exception("Lambda timeout") - - -def lambda_invoker( - lambda_handler: LambdaHandler, - input_queue: Queue, - output_queue: Queue, - log_level: str, -): - configure_logging(log_level=log_level) - sys.stdin = open("/dev/stdin") - - already_faked_coldstart = False - - while True: - try: - message = input_queue.get(block=True, timeout=1) - except KeyboardInterrupt: - LOGGER.debug("Stopping process") - sys.stdin.close() - break - except Empty: - continue - - LOGGER.debug("Received message: %s", message) - - if message["type"] == "smyth.stop": - LOGGER.debug("Stopping process") - break - - if message.get("type") != "smyth.lambda.invoke": - LOGGER.error("Invalid message type: %s", message.get("type")) - continue - - event, context = message["event"], FakeLambdaContext(**message["context"]) - - if ( - context.smyth["handler"]["handler_config"]["fake_coldstart"] # type: ignore[attr-defined] - and not already_faked_coldstart - ): - LOGGER.info("Faking cold start time") - sleep(randint(500, 1000) / 1000) - already_faked_coldstart = True - - signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(int(context.timeout)) - output_queue.put({"type": "smyth.lambda.status", "status": "working"}) - try: - response = lambda_handler(event, context) - except Exception as error: - LOGGER.error("Error invoking lambda: %s", error) - result = { - "type": "smyth.lambda.error", - "response": { - "type": type(error).__name__, - "message": str(error), - "stacktrace": traceback.format_exc(), - }, - } - else: - result = { - "type": "smyth.lambda.response", - "response": response, - } - finally: - signal.alarm(0) - output_queue.put({"type": "smyth.lambda.status", "status": "warm"}) - output_queue.put(result) diff --git a/src/smyth/server/app.py b/src/smyth/server/app.py index 4912950..d0de465 100644 --- a/src/smyth/server/app.py +++ b/src/smyth/server/app.py @@ -19,7 +19,11 @@ @asynccontextmanager async def lifespan(app: "SmythStarlette"): - app.smyth.start_runners() + try: + app.smyth.start_runners() + except Exception as error: + LOGGER.error("Error starting runners: %s", error) + raise yield app.smyth.stop_runners() @@ -47,6 +51,7 @@ def __init__(self, smyth: Smyth, smyth_path_prefix: str, *args, **kwargs): def create_app(): + LOGGER.debug("Creating app") config = get_config(get_config_dict()) smyth = Smyth() @@ -55,7 +60,7 @@ def create_app(): smyth.add_handler( name=handler_name, path=handler_config.url_path, - lambda_handler=import_attribute(handler_config.handler_path), + lambda_handler_path=handler_config.handler_path, timeout=handler_config.timeout, event_data_function=import_attribute( handler_config.event_data_function_path @@ -63,7 +68,6 @@ def create_app(): context_data_function=import_attribute( handler_config.context_data_function_path ), - fake_coldstart=handler_config.fake_coldstart, log_level=handler_config.log_level, concurrency=handler_config.concurrency, strategy_generator=import_attribute(handler_config.strategy_generator_path), diff --git a/src/smyth/server/endpoints.py b/src/smyth/server/endpoints.py index f30b013..1cc01c5 100644 --- a/src/smyth/server/endpoints.py +++ b/src/smyth/server/endpoints.py @@ -6,7 +6,7 @@ from starlette.responses import JSONResponse, Response from smyth.event import generate_lambda_invokation_event_data -from smyth.exceptions import LambdaInvocationError, LambdaTimeoutError +from smyth.exceptions import LambdaInvocationError, LambdaTimeoutError, SubprocessError from smyth.smyth import Smyth from smyth.types import EventDataCallable, SmythHandler @@ -15,18 +15,24 @@ async def dispatch( smyth: Smyth, - handler: SmythHandler, + smyth_handler: SmythHandler, request: Request, event_data_generator: EventDataCallable | None = None, ): + """ + Dispatches a request to Smyth and translates a Smyth + response to a Starlette response. + """ try: result = await smyth.dispatch( - handler, request, event_data_function=event_data_generator + smyth_handler, request, event_data_function=event_data_generator ) except LambdaInvocationError as error: return Response(str(error), status_code=status.HTTP_502_BAD_GATEWAY) except LambdaTimeoutError: return Response("Lambda timeout", status_code=status.HTTP_408_REQUEST_TIMEOUT) + except SubprocessError as error: + return Response(str(error), status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) if not result: return Response( @@ -42,23 +48,23 @@ async def dispatch( async def lambda_invoker_endpoint(request: Request): smyth: Smyth = request.app.smyth - handler = smyth.get_handler_for_request(request.url.path) - return await dispatch(smyth, handler, request) + smyth_handler = smyth.get_handler_for_request(request.url.path) + return await dispatch(smyth, smyth_handler, request) async def invocation_endpoint(request: Request): smyth: Smyth = request.app.smyth function = request.path_params["function"] try: - handler = smyth.get_handler_for_name(function) + smyth_handler = smyth.get_handler_for_name(function) except KeyError: return Response( f"Function {function} not found", status_code=status.HTTP_404_NOT_FOUND ) - handler.event_data_function = generate_lambda_invokation_event_data + smyth_handler.event_data_function = generate_lambda_invokation_event_data return await dispatch( smyth, - handler, + smyth_handler, request, event_data_generator=generate_lambda_invokation_event_data, ) diff --git a/src/smyth/smyth.py b/src/smyth/smyth.py index f78459b..f3a6c7d 100644 --- a/src/smyth/smyth.py +++ b/src/smyth/smyth.py @@ -1,4 +1,5 @@ import logging +import logging.config from collections.abc import Iterator from starlette.requests import Request @@ -12,7 +13,6 @@ from smyth.types import ( ContextDataCallable, EventDataCallable, - LambdaHandler, RunnerProcessProtocol, SmythHandler, StrategyGenerator, @@ -22,12 +22,12 @@ class Smyth: - handlers: dict[str, SmythHandler] + smyth_handlers: dict[str, SmythHandler] processes: dict[str, list[RunnerProcessProtocol]] strategy_generators: dict[str, Iterator["RunnerProcessProtocol"]] def __init__(self) -> None: - self.handlers = {} + self.smyth_handlers = {} self.processes = {} self.strategy_generators = {} @@ -35,23 +35,21 @@ def add_handler( self, name: str, path: str, - lambda_handler: LambdaHandler, + lambda_handler_path: str, timeout: float | None = None, event_data_function: EventDataCallable = generate_api_gw_v2_event_data, context_data_function: ContextDataCallable = generate_context_data, - fake_coldstart: bool = False, log_level: str = "INFO", concurrency: int = 1, strategy_generator: StrategyGenerator = first_warm, ): - self.handlers[name] = SmythHandler( + self.smyth_handlers[name] = SmythHandler( name=name, url_path=compile_path(path)[0], - lambda_handler=lambda_handler, + lambda_handler_path=lambda_handler_path, event_data_function=event_data_function, context_data_function=context_data_function, timeout=timeout, - fake_coldstart=fake_coldstart, log_level=log_level, concurrency=concurrency, strategy_generator=strategy_generator, @@ -65,12 +63,12 @@ def __exit__(self, exc_type, exc_value, traceback): self.stop_runners() def start_runners(self): - for handler_name, handler_config in self.handlers.items(): + for handler_name, handler_config in self.smyth_handlers.items(): self.processes[handler_name] = [] for index in range(handler_config.concurrency): process = RunnerProcess( name=f"{handler_name}:{index}", - lambda_handler=handler_config.lambda_handler, + lambda_handler_path=handler_config.lambda_handler_path, log_level=handler_config.log_level, ) process.start() @@ -93,7 +91,7 @@ def stop_runners(self): process.join() def get_handler_for_request(self, path: str) -> SmythHandler: - for handler_def in self.handlers.values(): + for handler_def in self.smyth_handlers.values(): if handler_def.url_path.match(path): return handler_def raise ProcessDefinitionNotFoundError( @@ -101,25 +99,32 @@ def get_handler_for_request(self, path: str) -> SmythHandler: ) def get_handler_for_name(self, name: str) -> SmythHandler: - return self.handlers[name] + return self.smyth_handlers[name] async def dispatch( self, - handler: SmythHandler, + smyth_handler: SmythHandler, request: Request, event_data_function: EventDataCallable | None = None, ): - process = next(self.strategy_generators[handler.name]) + """ + Smyth.dispatch is used upon a request that would normally be formed by an + AWS trigger. It is responsible for finding the appropriate process + for the request, invoking the process, and translating the response + """ + process = next(self.strategy_generators[smyth_handler.name]) if process is None: raise ProcessDefinitionNotFoundError( - f"No process definition found for handler {handler.name}" + f"No process definition found for handler {smyth_handler.name}" ) if event_data_function is None: - event_data_function = handler.event_data_function + event_data_function = smyth_handler.event_data_function event_data = await event_data_function(request) - context_data = await handler.context_data_function(request, handler, process) + context_data = await smyth_handler.context_data_function( + request, smyth_handler, process + ) return await process.asend( { @@ -130,6 +135,12 @@ async def dispatch( ) async def invoke(self, handler: SmythHandler, event_data: dict): + """ + Smyth.invoke is used to invoke a handler directly, without going through + Starlette or when a direct invocation is needed (e.g., when invoking + a lambda with boto3) - on direct invocation the event holds only the data + passed in the invokation. There's no Starlette request involved. + """ process = next(self.strategy_generators[handler.name]) if process is None: raise ProcessDefinitionNotFoundError( diff --git a/src/smyth/types.py b/src/smyth/types.py index 02cdd14..1b8f896 100644 --- a/src/smyth/types.py +++ b/src/smyth/types.py @@ -34,16 +34,25 @@ class RunnerProcessProtocol(Protocol): async def asend(self, data) -> RunnerMessage | None: ... + def stop(self): ... + + def send(self, data) -> RunnerMessage | None: ... + + def is_alive(self) -> bool: ... + + def terminate(self): ... + + def join(self): ... + @dataclass class SmythHandler: name: str url_path: Pattern[str] - lambda_handler: LambdaHandler + lambda_handler_path: str event_data_function: EventDataCallable context_data_function: ContextDataCallable strategy_generator: StrategyGenerator timeout: float | None = None - fake_coldstart: bool = False log_level: str = "INFO" concurrency: int = 1 diff --git a/src/smyth/utils.py b/src/smyth/utils.py index 51af62c..31b0aa9 100644 --- a/src/smyth/utils.py +++ b/src/smyth/utils.py @@ -1,5 +1,53 @@ import logging +from collections.abc import Callable, Iterable +from datetime import datetime from importlib import import_module +from logging import LogRecord +from pathlib import Path + +from rich.console import Console, ConsoleRenderable, RenderableType +from rich.logging import RichHandler +from rich.table import Table +from rich.text import Text, TextType +from rich.traceback import Traceback + +FormatTimeCallable = Callable[[datetime], Text] + + +def get_logging_config(log_level: str, filter_path_prefix: str | None = None) -> dict: + logging_config = { + "version": 1, + "disable_existing_loggers": False, + "filters": {}, + "handlers": { + "console": { + "class": "smyth.utils.SmythRichHandler", + "markup": True, + "rich_tracebacks": True, + "filters": [], + "show_path": False, + }, + }, + "loggers": { + "smyth": { + "handlers": ["console"], + "level": log_level, + "propagate": False, + }, + "uvicorn": { + "handlers": ["console"], + "level": log_level, + "propagate": False, + }, + }, + } + if filter_path_prefix: + logging_config["filters"]["smyth_api_filter"] = { # type: ignore[index] + "()": "smyth.utils.SmythStatusRouteFilter", + "smyth_path_prefix": filter_path_prefix, + } + logging_config["handlers"]["console"]["filters"].append("smyth_api_filter") # type: ignore[index] + return logging_config class SmythStatusRouteFilter(logging.Filter): @@ -11,6 +59,169 @@ def filter(self, record): return record.getMessage().find(self.smyth_path_prefix) == -1 +class LogRender: + """ + Derived from `rich._log_render.LogRender`. + """ + + def __init__( + self, + show_time: bool = True, + show_level: bool = False, + show_path: bool = True, + time_format: str | FormatTimeCallable = "[%X]", + omit_repeated_times: bool = True, + level_width: int | None = 8, + ) -> None: + self.show_time = show_time + self.show_level = show_level + self.show_path = show_path + self.time_format = time_format + self.omit_repeated_times = omit_repeated_times + self.level_width = level_width + self._last_time: Text | None = None + + def create_header_row(self, record: LogRecord) -> RenderableType: + issuer = record.name.split(".")[0] + + if issuer == "smyth": + issuer = "[bold yellow]Smyth[/]" + process_name = record.processName + elif issuer == "uvicorn": + issuer = "[bold blue]Uvicorn[/]" + process_name = f"Worker[{record.process}]" + else: + issuer = issuer.capitalize() + process_name = record.processName + + return Text.from_markup( + f"{issuer}:" f"[bold]{process_name}[/]", + style="log.process", + ) + + def create_time_row(self, log_time, console, time_format) -> RenderableType | None: + log_time = log_time or console.get_datetime() + time_format = time_format or self.time_format + if callable(time_format): + log_time_display = time_format(log_time) + else: + log_time_display = Text(log_time.strftime(time_format)) + if log_time_display == self._last_time and self.omit_repeated_times: + return Text(" " * len(log_time_display)) + else: + self._last_time = log_time_display + return log_time_display + + def create_path_row( + self, path: str, line_no: int | None, link_path: str | None + ) -> RenderableType: + path_text = Text() + path_text.append(path, style=f"link file://{link_path}" if link_path else "") + if line_no: + path_text.append(":") + path_text.append( + f"{line_no}", + style=f"link file://{link_path}#{line_no}" if link_path else "", + ) + return path_text + + def configure_columns(self, record: LogRecord) -> tuple[bool, bool, bool]: + full_width = getattr(record, "log_setting", None) == "console_full_width" + + if full_width: + show_time = False + show_level = False + show_path = False + else: + show_time = self.show_time + show_level = self.show_level + show_path = self.show_path + + return show_time, show_level, show_path + + def __call__( + self, + record: LogRecord, + console: "Console", + renderables: Iterable["ConsoleRenderable"], + log_time: datetime | None = None, + time_format: str | FormatTimeCallable | None = None, + level: TextType = "", + path: str | None = None, + line_no: int | None = None, + link_path: str | None = None, + ) -> "Table": + from rich.containers import Renderables + from rich.table import Table + + show_time, show_level, show_path = self.configure_columns(record) + output = Table.grid(padding=(0, 1)) + output.expand = True + output.add_column(justify="left", min_width=22) + row: list[RenderableType] = [] + + row.append(self.create_header_row(record)) + + if show_time: + output.add_column(style="log.time") + create_time_row = self.create_time_row(log_time, console, time_format) + if create_time_row: + row.append(create_time_row) + if show_level: + output.add_column(style="log.level", width=self.level_width) + row.append(level) + + output.add_column(ratio=1, style="log.message", overflow="fold") + row.append(Renderables(renderables)) + + if show_path and path: + output.add_column(style="log.path") + row.append(self.create_path_row(path, line_no, link_path)) + + output.add_row(*row) + return output + + +class SmythRichHandler(RichHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.rich_render = LogRender( + show_time=True, + show_level=True, + show_path=False, + time_format="[%X]", + omit_repeated_times=True, + level_width=8, + ) + + def render( + self, + *, + record: LogRecord, + traceback: Traceback | None, + message_renderable: "ConsoleRenderable", + ) -> "ConsoleRenderable": + path = Path(record.pathname).name + level = self.get_level_text(record) + time_format = None if self.formatter is None else self.formatter.datefmt + log_time = datetime.fromtimestamp(record.created) + + log_renderable = self.rich_render( + record=record, + console=self.console, + renderables=[message_renderable] + if not traceback + else [message_renderable, traceback], + log_time=log_time, + time_format=time_format, + level=level, + path=path, + line_no=record.lineno, + link_path=record.pathname if self.enable_link_path else None, + ) + return log_renderable + + def import_attribute(python_path: str): module_name, handler_name = python_path.rsplit(".", 1) module = import_module(module_name) diff --git a/tests/conftest.py b/tests/conftest.py index 4cc7fb6..2ed18f8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,16 +16,20 @@ def smyth_handler( return SmythHandler( name="test_handler", url_path=re.compile(r"/test_handler"), - lambda_handler=mock_lambda_handler, + lambda_handler_path="tests.conftest.example_handler", event_data_function=mock_event_data_function, context_data_function=mock_context_data_function, strategy_generator=mock_strategy_generator, ) +def example_handler(event, context): + return {"statusCode": 200, "body": "Hello, World!"} + + @pytest.fixture def mock_lambda_handler(): - return Mock() + return example_handler @pytest.fixture diff --git a/tests/test_context.py b/tests/test_context.py index cf8404b..81b8e77 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -12,12 +12,11 @@ async def test_generate_context_data( assert await generate_context_data(None, smyth_handler, mock_runner_process) == { "smyth": { "handler": { - "handler_config": { + "smyth_handler_config": { "concurrency": 1, "context_data_function": ANY, "event_data_function": ANY, - "fake_coldstart": False, - "lambda_handler": ANY, + "lambda_handler_path": "tests.conftest.example_handler", "log_level": "INFO", "name": "test_handler", "strategy_generator": ANY, From 97a2999388ace6ead5884b3925d49f4e9ac4501b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Kucmus?= Date: Sun, 13 Oct 2024 15:44:33 +0200 Subject: [PATCH 2/5] add tests, coverage is 87% --- README.md | 1 - pyproject.toml | 36 ++++-- src/smyth/event.py | 2 +- src/smyth/runner/fake_context.py | 18 ++- src/smyth/runner/strategy.py | 26 +++-- src/smyth/server/endpoints.py | 10 +- src/smyth/utils.py | 2 +- tests/conftest.py | 31 ++++- tests/runner/__init__.py | 0 tests/runner/test_fake_context.py | 51 ++++++++ tests/runner/test_process.py | 186 ++++++++++++++++++++++++++++++ tests/runner/test_strategy.py | 66 +++++++++++ tests/server/__init__.py | 0 tests/server/test_app.py | 91 +++++++++++++++ tests/server/test_endpoints.py | 121 +++++++++++++++++++ tests/test_config.py | 104 +++++++++++++++++ tests/test_event.py | 49 ++++++++ tests/test_smyth.py | 101 ++++++++++++++++ tests/test_utils.py | 31 +++++ 19 files changed, 891 insertions(+), 35 deletions(-) create mode 100644 tests/runner/__init__.py create mode 100644 tests/runner/test_fake_context.py create mode 100644 tests/runner/test_process.py create mode 100644 tests/runner/test_strategy.py create mode 100644 tests/server/__init__.py create mode 100644 tests/server/test_app.py create mode 100644 tests/server/test_endpoints.py create mode 100644 tests/test_config.py create mode 100644 tests/test_event.py create mode 100644 tests/test_smyth.py create mode 100644 tests/test_utils.py diff --git a/README.md b/README.md index dc3ef03..81a08ee 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,6 @@ The combination of Uvicorn reload process and HTTP server process with what is b ## TODO - [ ] Write tests -- [ ] Properly handle Uvicorn exit, kill the LambdaProcesses gracefully - [x] Publish on PyPi ## Name diff --git a/pyproject.toml b/pyproject.toml index 8663390..d632d9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,12 +5,12 @@ build-backend = "hatchling.build" [project] name = "smyth" dynamic = ["version"] -description = '' +description = "Smyth is a versatile tool designed to enhance your AWS Lambda development experience. It is a pure Python tool that allows for easy customization and state persistence, making your Lambda development more efficient and developer-friendly." readme = "README.md" requires-python = ">=3.10" license = { file = "LICENSE.txt" } keywords = [] -authors = [{ name = "Mirumee", email = "it@mirumee.com" }] +authors = [{ name = "Mirumee", email = "hello@mirumee.com" }] classifiers = [ "Environment :: Console", "Intended Audience :: Developers", @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Build Tools", "Topic :: Software Development :: Libraries :: Python Modules", @@ -62,12 +63,17 @@ packages = ["src/smyth"] features = ["dev", "types", "docs"] [tool.hatch.envs.default.scripts] -check = ["hatch fmt", "hatch test -a", "hatch test --cover", "hatch run types:check"] +check = [ + "hatch fmt", + "hatch test -a", + "hatch test --cover", + "hatch run types:check", +] ## Types environment [tool.hatch.envs.types.scripts] -check = "mypy --install-types --non-interactive {args:src/smyth tests}" +check = "mypy --install-types --non-interactive {args:src/smyth}" [tool.mypy] check_untyped_defs = true @@ -83,15 +89,17 @@ ignore_missing_imports = true dependencies = [ "asynctest", "ipdb", - "pytest-asyncio", + "anyio", + "pytest-mock", "pytest-memray", "pytest-print", "pytest-cov", "coverage[toml]", + "httpx", ] [[tool.hatch.envs.hatch-test.matrix]] -python = ["3.10", "3.11", "3.12"] +python = ["3.10", "3.11", "3.12", "3.13"] ## Docs environment @@ -119,14 +127,20 @@ parallel = true smyth = ["src/smyth"] [tool.coverage.report] -exclude_lines = [ - "no cov", +exclude_also = [ + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", "if __name__ == .__main__.:", "if TYPE_CHECKING:", - "@abstract", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", ] omit = ["*/__about__.py", "*/__main__.py", "*/cli/__init__.py"] -# fail_under = 90 # TODO: Uncomment when coverage is good enough (0.9) +fail_under = 80 ## Ruff configuration @@ -140,7 +154,7 @@ docstring-code-line-length = 80 [tool.ruff.lint] select = ["E", "F", "G", "I", "N", "Q", "UP", "C90", "T20", "TID"] -unfixable = ["UP007"] # typer does not handle PEP604 annotations +unfixable = ["UP007"] # typer does not handle PEP604 annotations [tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" diff --git a/src/smyth/event.py b/src/smyth/event.py index 8c75980..7ae24bc 100644 --- a/src/smyth/event.py +++ b/src/smyth/event.py @@ -28,5 +28,5 @@ async def generate_api_gw_v2_event_data(request: Request): } -async def generate_lambda_invokation_event_data(request: Request): +async def generate_lambda_invocation_event_data(request: Request): return await request.json() diff --git a/src/smyth/runner/fake_context.py b/src/smyth/runner/fake_context.py index 2126903..276f64c 100644 --- a/src/smyth/runner/fake_context.py +++ b/src/smyth/runner/fake_context.py @@ -5,11 +5,27 @@ class FakeLambdaContext(LambdaContext): - def __init__(self, name="Fake", version="LATEST", timeout=6, **kwargs): + def __init__( + self, + name: str | None = None, + version: str | None = "LATEST", + timeout: int | None = None, + **kwargs, + ): + if name is None: + name = "Fake" self.name = name + + if version is None: + version = "LATEST" self.version = version + self.created = time() + + if timeout is None: + timeout = 6 self.timeout = timeout + for key, value in kwargs.items(): setattr(self, key, value) diff --git a/src/smyth/runner/strategy.py b/src/smyth/runner/strategy.py index a58ce7a..0d83c3f 100644 --- a/src/smyth/runner/strategy.py +++ b/src/smyth/runner/strategy.py @@ -1,5 +1,6 @@ from collections.abc import Iterator +from smyth.exceptions import NoAvailableProcessError from smyth.types import RunnerProcessProtocol, SmythHandlerState @@ -26,16 +27,17 @@ def first_warm( lead to faster response times.""" while True: - best_candidate = None + warm = None + cold = None for process in processes[handler_name]: - if ( - process.state == SmythHandlerState.WARM - or process.state == SmythHandlerState.COLD - and not best_candidate - ): - best_candidate = process - - if best_candidate is None: - raise Exception("No process available") - - yield best_candidate + if process.state == SmythHandlerState.WARM: + warm = process + elif process.state == SmythHandlerState.COLD: + cold = process + + if warm is not None: + yield warm + elif cold is not None: + yield cold + else: + raise NoAvailableProcessError("No process available") diff --git a/src/smyth/server/endpoints.py b/src/smyth/server/endpoints.py index 1cc01c5..680fed8 100644 --- a/src/smyth/server/endpoints.py +++ b/src/smyth/server/endpoints.py @@ -5,7 +5,7 @@ from starlette.requests import Request from starlette.responses import JSONResponse, Response -from smyth.event import generate_lambda_invokation_event_data +from smyth.event import generate_lambda_invocation_event_data from smyth.exceptions import LambdaInvocationError, LambdaTimeoutError, SubprocessError from smyth.smyth import Smyth from smyth.types import EventDataCallable, SmythHandler @@ -17,7 +17,7 @@ async def dispatch( smyth: Smyth, smyth_handler: SmythHandler, request: Request, - event_data_generator: EventDataCallable | None = None, + event_data_function: EventDataCallable | None = None, ): """ Dispatches a request to Smyth and translates a Smyth @@ -25,7 +25,7 @@ async def dispatch( """ try: result = await smyth.dispatch( - smyth_handler, request, event_data_function=event_data_generator + smyth_handler, request, event_data_function=event_data_function ) except LambdaInvocationError as error: return Response(str(error), status_code=status.HTTP_502_BAD_GATEWAY) @@ -61,12 +61,12 @@ async def invocation_endpoint(request: Request): return Response( f"Function {function} not found", status_code=status.HTTP_404_NOT_FOUND ) - smyth_handler.event_data_function = generate_lambda_invokation_event_data + smyth_handler.event_data_function = generate_lambda_invocation_event_data return await dispatch( smyth, smyth_handler, request, - event_data_generator=generate_lambda_invokation_event_data, + event_data_function=generate_lambda_invocation_event_data, ) diff --git a/src/smyth/utils.py b/src/smyth/utils.py index 31b0aa9..e10f257 100644 --- a/src/smyth/utils.py +++ b/src/smyth/utils.py @@ -59,7 +59,7 @@ def filter(self, record): return record.getMessage().find(self.smyth_path_prefix) == -1 -class LogRender: +class LogRender: # pragma: no cover """ Derived from `rich._log_render.LogRender`. """ diff --git a/tests/conftest.py b/tests/conftest.py index 2ed18f8..18264d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,17 @@ import re -from unittest.mock import Mock +from unittest.mock import AsyncMock, Mock import pytest +from smyth.config import Config, HandlerConfig from smyth.types import RunnerProcessProtocol, SmythHandler, SmythHandlerState +@pytest.fixture(autouse=True) +def anyio_backend(): + return "asyncio", {"use_uvloop": True} + + @pytest.fixture def smyth_handler( mock_lambda_handler, @@ -23,6 +29,25 @@ def smyth_handler( ) +@pytest.fixture +def config(): + return Config( + host="0.0.0.0", + port=8080, + handlers={ + "order_handler": HandlerConfig( + handler_path="tests.conftest.example_handler", + url_path=r"/test_handler", + ), + "product_handler": HandlerConfig( + handler_path="tests.conftest.example_handler", + url_path=r"/products/{path:path}", + ), + }, + log_level="INFO", + ) + + def example_handler(event, context): return {"statusCode": 200, "body": "Hello, World!"} @@ -34,12 +59,12 @@ def mock_lambda_handler(): @pytest.fixture def mock_event_data_function(): - return Mock() + return AsyncMock() @pytest.fixture def mock_context_data_function(): - return Mock() + return AsyncMock() @pytest.fixture diff --git a/tests/runner/__init__.py b/tests/runner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/runner/test_fake_context.py b/tests/runner/test_fake_context.py new file mode 100644 index 0000000..849c4d1 --- /dev/null +++ b/tests/runner/test_fake_context.py @@ -0,0 +1,51 @@ +from time import strftime + +import pytest + +from smyth.runner.fake_context import FakeLambdaContext + + +def test_fake_lambda_context(): + context = FakeLambdaContext() + assert context.function_name == "Fake" + assert context.function_version == "LATEST" + assert context.invoked_function_arn == "arn:aws:lambda:serverless:Fake" + assert context.memory_limit_in_mb == "1024" + assert context.aws_request_id == "1234567890" + assert context.log_group_name == "/aws/lambda/Fake" + assert context.log_stream_name == ( + f"{strftime('%Y/%m/%d')}/[$LATEST]58419525dade4d17a495dceeeed44708" + ) + + +@pytest.mark.parametrize( + ( + "name", + "version", + "timeout", + "expected_name", + "expected_version", + "expected_timeout", + ), + [ + ("test", "test", 60, "test", "test", 60), + ("test", "test", None, "test", "test", 6), + ("test", None, 120, "test", "LATEST", 120), + (None, "test", 6, "Fake", "test", 6), + (None, None, 6, "Fake", "LATEST", 6), + ], +) +def test_fake_lambda_context_with_params( + name, version, timeout, expected_name, expected_version, expected_timeout +): + context = FakeLambdaContext(name=name, version=version, timeout=timeout) + assert context.function_name == expected_name + assert context.function_version == expected_version + assert context.timeout == expected_timeout + assert context.invoked_function_arn == f"arn:aws:lambda:serverless:{expected_name}" + assert context.memory_limit_in_mb == "1024" + assert context.aws_request_id == "1234567890" + assert context.log_group_name == f"/aws/lambda/{expected_name}" + assert context.log_stream_name == ( + f"{strftime('%Y/%m/%d')}/[${expected_version}]58419525dade4d17a495dceeeed44708" + ) diff --git a/tests/runner/test_process.py b/tests/runner/test_process.py new file mode 100644 index 0000000..5cdec56 --- /dev/null +++ b/tests/runner/test_process.py @@ -0,0 +1,186 @@ +from queue import Empty + +import pytest + +from smyth.exceptions import LambdaHandlerLoadError, LambdaTimeoutError +from smyth.runner.fake_context import FakeLambdaContext +from smyth.runner.process import RunnerProcess +from smyth.types import SmythHandlerState + +pytestmark = pytest.mark.anyio + + +@pytest.fixture +def mock_setproctitle(mocker): + return mocker.patch("smyth.runner.process.setproctitle") + + +@pytest.fixture +def mock_logging_dictconfig(mocker): + return mocker.patch("logging.config.dictConfig") + + +@pytest.fixture +def runner_process(): + return RunnerProcess("test_process", "tests.conftest.example_handler") + + +def test_init_process(runner_process): + assert runner_process.name == "test_process" + assert runner_process.task_counter == 0 + assert runner_process.last_used_timestamp == 0 + assert runner_process.state == SmythHandlerState.COLD + assert runner_process.lambda_handler_path == "tests.conftest.example_handler" + assert runner_process.log_level == "INFO" + + +def test_stop_process(runner_process): + runner_process.start() + assert runner_process.is_alive() is True + runner_process.stop() + assert runner_process.is_alive() is False + + +@pytest.mark.skip(reason="This needs more thought") +def test_send_process(runner_process): + pass + + +def test_run(mocker, mock_setproctitle, mock_logging_dictconfig, runner_process): + mock_lambda_invoker__ = mocker.patch.object( + runner_process, "lambda_invoker__", autospec=True + ) + mock_get_logging_config = mocker.patch( + "smyth.runner.process.get_logging_config", autospec=True + ) + runner_process.run() + mock_setproctitle.assert_called_once_with(f"smyth:{runner_process.name}") + mock_logging_dictconfig.assert_called_once_with( + mock_get_logging_config.return_value + ) + mock_lambda_invoker__.assert_called_once() + + +def test_get_message(mocker, runner_process): + mock_input_queue = mocker.patch.object(runner_process, "input_queue", autospec=True) + mock_input_queue.get.side_effect = [ + {"type": "smyth.lambda.invoke", "event": {}, "context": {}}, + Empty, + {"type": "smyth.lambda.invoke", "event": {}, "context": {}}, + {"type": "smyth.stop"}, + ] + + messages = list(runner_process.get_message__()) + + assert len(messages) == 2 + assert messages[0]["type"] == "smyth.lambda.invoke" + assert messages[1]["type"] == "smyth.lambda.invoke" + + +def test_get_event(mocker, runner_process): + assert ( + runner_process.get_event__({"type": "smyth.lambda.invoke", "event": {}}) == {} + ) + + +def test_get_context(mocker, runner_process): + assert isinstance( + runner_process.get_context__({"type": "smyth.lambda.invoke", "context": {}}), + FakeLambdaContext, + ) + + assert ( + runner_process.get_context__( + {"type": "smyth.lambda.invoke", "context": {}} + ).timeout + == 6 + ) + + +def test_import_handler(mocker, runner_process): + mock_import_attribute = mocker.patch( + "smyth.runner.process.import_attribute", autospec=True + ) + + runner_process.import_handler__("tests.conftest.example_handler", {}, {}) + mock_import_attribute.assert_called_once_with("tests.conftest.example_handler") + + mock_import_attribute.side_effect = AttributeError + with pytest.raises(LambdaHandlerLoadError): + runner_process.import_handler__("tests.conftest.example_handler", {}, {}) + + mock_import_attribute.side_effect = ImportError + with pytest.raises(LambdaHandlerLoadError): + runner_process.import_handler__("tests.conftest.example_handler", {}, {}) + + mock_import_attribute.side_effect = Exception + with pytest.raises(Exception): + runner_process.import_handler__("tests.conftest.example_handler", {}, {}) + + +def test_set_status(runner_process): + runner_process.set_status__(SmythHandlerState.COLD) + assert runner_process.state == SmythHandlerState.COLD + + +def test_timeout_handler(runner_process): + with pytest.raises(LambdaTimeoutError): + runner_process.timeout_handler__(None, None) + + +def test_lambda_invoker(mocker, runner_process): + mock_handler = mocker.Mock() + mock_import_attribute = mocker.patch( + "smyth.runner.process.import_attribute", + autospec=True, + return_value=mock_handler, + ) + mock_signal = mocker.patch("signal.signal", autospec=True) + mock_signal.side_effect = lambda signum, frame: None + + mocker.patch.object(runner_process, "output_queue", autospec=True) + + mocker.patch.object( + runner_process, + "get_message__", + autospec=True, + return_value=[ + {"type": "smyth.lambda.invoke", "event": {"test": "1"}, "context": {}}, + {"type": "smyth.lambda.invoke", "event": {"test": "2"}, "context": {}}, + ], + ) + mocker.patch.object( + runner_process, + "get_event__", + autospec=True, + side_effect=[ + {"test": "1"}, + {"test": "2"}, + ], + ) + mock_get_context__ = mocker.patch.object( + runner_process, "get_context__", autospec=True + ) + mock_set_status__ = mocker.patch.object( + runner_process, "set_status__", autospec=True + ) + + runner_process.lambda_invoker__() + assert mock_import_attribute.call_count == 1 + + mock_set_status__.assert_has_calls( + [ + mocker.call(SmythHandlerState.COLD), + mocker.call(SmythHandlerState.WARM), + mocker.call(SmythHandlerState.WORKING), + mocker.call(SmythHandlerState.WORKING), + ] + ) + + assert mock_handler.call_count == 2 + mock_handler.assert_has_calls( + [ + mocker.call({"test": "1"}, mock_get_context__.return_value), + mocker.call({"test": "2"}, mock_get_context__.return_value), + ] + ) diff --git a/tests/runner/test_strategy.py b/tests/runner/test_strategy.py new file mode 100644 index 0000000..6f141bf --- /dev/null +++ b/tests/runner/test_strategy.py @@ -0,0 +1,66 @@ +import pytest + +from smyth.exceptions import NoAvailableProcessError +from smyth.runner.strategy import first_warm, round_robin +from smyth.types import SmythHandlerState + + +def test_round_robin(): + strat = round_robin("test_handler", {"test_handler": [1, 2, 3]}) + assert next(strat) == 1 + assert next(strat) == 2 + assert next(strat) == 3 + assert next(strat) == 1 + assert next(strat) == 2 + + +def test_first_warm(mocker): + mock_cold_process = mocker.Mock() + mock_cold_process.state = SmythHandlerState.COLD + mock_working_process = mocker.Mock() + mock_working_process.state = SmythHandlerState.WORKING + mock_warm_process = mocker.Mock() + mock_warm_process.state = SmythHandlerState.WARM + + strat = first_warm( + "test_handler", + {"test_handler": [mock_working_process, mock_cold_process, mock_warm_process]}, + ) + + assert next(strat) == mock_warm_process + assert next(strat) == mock_warm_process + assert next(strat) == mock_warm_process + + +def test_first_warm_no_warm(mocker): + mock_cold_process = mocker.Mock() + mock_cold_process.state = SmythHandlerState.COLD + mock_working_process = mocker.Mock() + mock_working_process.state = SmythHandlerState.WORKING + mock_warm_process = mocker.Mock() + mock_warm_process.state = SmythHandlerState.WARM + + strat = first_warm( + "test_handler", + {"test_handler": [mock_working_process, mock_cold_process, mock_cold_process]}, + ) + + assert next(strat) == mock_cold_process + assert next(strat) == mock_cold_process + mock_working_process.state = SmythHandlerState.WARM + assert next(strat) == mock_working_process + + +def test_first_warm_no_available(mocker): + mock_working_process = mocker.Mock() + mock_working_process.state = SmythHandlerState.WORKING + + strat = first_warm( + "test_handler", + {"test_handler": [mock_working_process]}, + ) + + with pytest.raises(NoAvailableProcessError) as excinfo: + next(strat) + + assert excinfo.errisinstance(NoAvailableProcessError) diff --git a/tests/server/__init__.py b/tests/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/server/test_app.py b/tests/server/test_app.py new file mode 100644 index 0000000..8e084e4 --- /dev/null +++ b/tests/server/test_app.py @@ -0,0 +1,91 @@ +import pytest +from starlette.routing import Route + +from smyth.context import generate_context_data +from smyth.event import generate_api_gw_v2_event_data +from smyth.runner.strategy import first_warm +from smyth.server.app import SmythStarlette, create_app, lifespan +from smyth.server.endpoints import ( + invocation_endpoint, + lambda_invoker_endpoint, + status_endpoint, +) + +pytestmark = pytest.mark.anyio + + +@pytest.fixture +def mock_get_config(mocker, config): + return mocker.patch("smyth.server.app.get_config", return_value=config) + + +def test_create_app(mocker, mock_get_config): + mock_smyth = mocker.Mock() + mocker.patch("smyth.server.app.Smyth", autospec=True, return_value=mock_smyth) + + app = create_app() + + assert app.smyth == mock_smyth + mock_smyth.add_handler.assert_has_calls( + [ + mocker.call( + name="order_handler", + path=r"/test_handler", + lambda_handler_path="tests.conftest.example_handler", + timeout=None, + event_data_function=generate_api_gw_v2_event_data, + context_data_function=generate_context_data, + log_level="DEBUG", + concurrency=1, + strategy_generator=first_warm, + ), + mocker.call( + name="product_handler", + path=r"/products/{path:path}", + lambda_handler_path="tests.conftest.example_handler", + timeout=None, + event_data_function=generate_api_gw_v2_event_data, + context_data_function=generate_context_data, + log_level="DEBUG", + concurrency=1, + strategy_generator=first_warm, + ), + ] + ) + + +def test_smyth_starlette(mocker): + mock_smyth = mocker.Mock() + + app = SmythStarlette(smyth=mock_smyth, smyth_path_prefix="/smyth") + + assert app.smyth == mock_smyth + assert app.routes == [ + Route("/smyth/api/status", status_endpoint, methods=["GET", "HEAD"]), + Route( + "/2015-03-31/functions/{function:str}/invocations", + invocation_endpoint, + methods=["POST"], + ), + Route( + "/{path:path}", + lambda_invoker_endpoint, + methods=["DELETE", "GET", "HEAD", "POST", "PUT"], + ), + ] + + +async def test_lifespan(mocker): + mock_app = mocker.Mock() + + async with lifespan(mock_app): + mock_app.smyth.start_runners.assert_called_once_with() + + mock_app.smyth.stop_runners.assert_called_once_with() + + mock_app = mocker.Mock() + mock_app.smyth.start_runners.side_effect = Exception("Test") + + with pytest.raises(Exception): + async with lifespan(mock_app): + pass diff --git a/tests/server/test_endpoints.py b/tests/server/test_endpoints.py new file mode 100644 index 0000000..c4cb2fd --- /dev/null +++ b/tests/server/test_endpoints.py @@ -0,0 +1,121 @@ +import pytest +from starlette.testclient import TestClient + +from smyth.exceptions import LambdaInvocationError, LambdaTimeoutError, SubprocessError +from smyth.server.app import SmythStarlette +from smyth.server.endpoints import dispatch + +pytestmark = pytest.mark.anyio + + +@pytest.fixture +def mock_smyth_dispatch(mocker): + return mocker.AsyncMock() + + +@pytest.fixture +def mock_smyth(mocker, mock_smyth_dispatch): + smyth = mocker.Mock() + smyth.handlers = { + "order_handler": mocker.Mock(name="order_handler"), + "product_handler": mocker.Mock(name="product_handler"), + } + smyth.processes = { + "order_handler": [ + mocker.Mock(name="process1", task_counter=0, state="cold"), + ], + "product_handler": [ + mocker.Mock(name="process2", task_counter=0, state="cold"), + mocker.Mock(name="process3", task_counter=0, state="cold"), + ], + } + smyth.dispatch = mock_smyth_dispatch + return smyth + + +@pytest.fixture +def app(mock_smyth): + return SmythStarlette(smyth=mock_smyth, smyth_path_prefix="/smyth") + + +@pytest.fixture +def test_client(app): + return TestClient(app) + + +@pytest.mark.parametrize( + ("side_effect", "expected_status_code", "expected_body"), + [ + (None, 200, b"Hello, World!"), + (LambdaInvocationError("Test error"), 502, b"Test error"), + (LambdaTimeoutError("Test error"), 408, b"Lambda timeout"), + (SubprocessError("Test error"), 500, b"Test error"), + ], +) +async def test_dispatch( + mocker, + mock_smyth, + mock_smyth_dispatch, + side_effect, + expected_status_code, + expected_body, +): + mock_request = mocker.Mock() + mock_event_data_function = mocker.Mock() + mock_smyth_dispatch.return_value = { + "body": "Hello, World!", + "statusCode": 200, + "headers": {}, + } + mock_smyth_dispatch.side_effect = side_effect + response = await dispatch( + smyth=mock_smyth, + smyth_handler=mock_smyth.handlers["order_handler"], + request=mock_request, + event_data_function=mock_event_data_function, + ) + assert response.status_code == expected_status_code + assert response.body == expected_body + + +def test_status_endpoint(test_client): + response = test_client.get("/smyth/api/status") + assert response.status_code == 200 + assert response.json() == { + "lambda handlers": { + "order_handler": { + "processes": [ + { + "state": "cold", + "task_counter": 0, + } + ] + }, + "product_handler": { + "processes": [ + { + "state": "cold", + "task_counter": 0, + }, + { + "state": "cold", + "task_counter": 0, + }, + ] + }, + }, + } + + +def test_invocation_endpoint(test_client, mock_smyth, mock_smyth_dispatch): + mock_smyth_dispatch.return_value = { + "body": "Hello, World!", + "statusCode": 200, + "headers": {}, + } + response = test_client.post( + "/2015-03-31/functions/order_handler/invocations", + json={"test": "test"}, + ) + assert response.status_code == 200 + assert response.text == "Hello, World!" diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..4c5e6fb --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,104 @@ +import json +import os +from dataclasses import asdict +from pathlib import Path + +import pytest + +from smyth.config import ( + Config, + HandlerConfig, + get_config, + get_config_dict, + get_config_file_path, + serialize_config, +) +from smyth.exceptions import ConfigFileNotFoundError + + +def test_get_config_file_path(): + assert get_config_file_path() == get_config_file_path("pyproject.toml") + + with pytest.raises(ConfigFileNotFoundError): + get_config_file_path("not_existing.toml") + + +def test_get_config_dict(mocker): + mock_toml = mocker.patch("smyth.config.toml.load") + mock_get_config_file_path = mocker.patch( + "smyth.config.get_config_file_path", return_value=Path("pyproject.toml") + ) + + assert get_config_dict() == mock_toml.return_value + mock_toml.assert_called_once_with(mock_get_config_file_path.return_value) + mock_get_config_file_path.assert_called_once_with() + + mock_get_config_file_path.reset_mock() + mock_toml.reset_mock() + mock_get_config_file_path.return_value = Path("other.toml") + + assert get_config_dict("other.toml") == mock_toml.return_value + mock_toml.assert_called_once_with(mock_get_config_file_path.return_value) + mock_get_config_file_path.assert_called_once_with("other.toml") + + +def test_get_config(): + config_dict = { + "tool": { + "smyth": { + "host": "0.0.0.0", + "port": 8080, + "handlers": { + "order_handler": { + "handler_path": "tests.conftest.example_handler", + "url_path": "/test_handler", + }, + "product_handler": { + "handler_path": "tests.conftest.example_handler", + "url_path": "/products/{path:path}", + }, + }, + "log_level": "INFO", + } + } + } + + config = get_config(config_dict) + + assert config.host == "0.0.0.0" + assert config.port == 8080 + assert config.handlers == { + "order_handler": HandlerConfig( + handler_path="tests.conftest.example_handler", + url_path=r"/test_handler", + ), + "product_handler": HandlerConfig( + handler_path="tests.conftest.example_handler", + url_path=r"/products/{path:path}", + ), + } + assert config.log_level == "INFO" + + os.environ["__SMYTH_CONFIG"] = serialize_config(config) + + assert get_config(None) == config + + +def test_serialize_config(): + config = Config( + host="0.0.0.0", + port=8080, + handlers={ + "order_handler": HandlerConfig( + handler_path="tests.conftest.example_handler", + url_path=r"/test_handler", + ), + "product_handler": HandlerConfig( + handler_path="tests.conftest.example_handler", + url_path=r"/products/{path:path}", + ), + }, + log_level="INFO", + ) + + assert serialize_config(config) == json.dumps(asdict(config)) diff --git a/tests/test_event.py b/tests/test_event.py new file mode 100644 index 0000000..0e2c33a --- /dev/null +++ b/tests/test_event.py @@ -0,0 +1,49 @@ +import pytest + +from smyth.event import ( + generate_api_gw_v2_event_data, + generate_lambda_invocation_event_data, +) + +pytestmark = pytest.mark.anyio + + +async def test_generate_api_gw_v2_event_data(mocker): + mock_request = mocker.Mock() + mock_request.body = mocker.AsyncMock(return_value=b"") + mock_request.headers = {} + mock_request.query_params = {} + mock_request.client.host = "127.0.0.1" + mock_request.method = "GET" + mock_request.url.path = "/test" + mock_request.url.query = "" + mock_request.url.scheme = "http" + + assert await generate_api_gw_v2_event_data(mock_request) == { + "version": "2.0", + "rawPath": "/test", + "body": "", + "isBase64Encoded": False, + "headers": {}, + "queryStringParameters": {}, + "requestContext": { + "http": { + "method": "GET", + "path": "/test", + "protocol": "http", + "sourceIp": "127.0.0.1", + }, + "routeKey": "GET /test", + "accountId": "offlineContext_accountId", + "stage": "$default", + }, + "routeKey": "GET /test", + "rawQueryString": "", + } + + +async def test_generate_lambda_invokation_event_data(mocker): + mock_request = mocker.Mock() + mock_request.json = mocker.AsyncMock(return_value={"test": "test"}) + + assert await generate_lambda_invocation_event_data(mock_request) == {"test": "test"} diff --git a/tests/test_smyth.py b/tests/test_smyth.py new file mode 100644 index 0000000..670e930 --- /dev/null +++ b/tests/test_smyth.py @@ -0,0 +1,101 @@ +import pytest + +from smyth.exceptions import ProcessDefinitionNotFoundError +from smyth.runner.strategy import first_warm +from smyth.smyth import Smyth +from smyth.types import SmythHandlerState + +pytestmark = pytest.mark.anyio + + +@pytest.fixture +def smyth(mock_event_data_function, mock_context_data_function): + smyth = Smyth() + smyth.add_handler( + name="test_handler", + path="/test_handler", + lambda_handler_path="tests.conftest.example_handler", + timeout=1, + event_data_function=mock_event_data_function, + context_data_function=mock_context_data_function, + log_level="DEBUG", + concurrency=1, + strategy_generator=first_warm, + ) + return smyth + + +def test_smyth_add_handler(smyth, mock_event_data_function, mock_context_data_function): + handler = smyth.get_handler_for_name("test_handler") + assert handler.name == "test_handler" + assert handler.url_path.match("/test_handler") + assert handler.lambda_handler_path == "tests.conftest.example_handler" + assert handler.event_data_function == mock_event_data_function + assert handler.context_data_function == mock_context_data_function + assert handler.strategy_generator == first_warm + assert handler.timeout == 1 + assert handler.log_level == "DEBUG" + assert handler.concurrency == 1 + + +def test_context_enter_exit(mocker): + smyth = Smyth() + mocker.patch.object(smyth, "start_runners") + mocker.patch.object(smyth, "stop_runners") + with smyth: + pass + assert smyth.start_runners.called + assert smyth.stop_runners.called + + +def test_start_stop_runners(smyth): + smyth.start_runners() + assert smyth.processes["test_handler"][0].state == SmythHandlerState.COLD + assert smyth.processes["test_handler"][0].task_counter == 0 + smyth.stop_runners() + + +def test_get_handler_for_request(smyth): + handler = smyth.get_handler_for_request("/test_handler") + assert handler.name == "test_handler" + + with pytest.raises(ProcessDefinitionNotFoundError): + smyth.get_handler_for_request("/test_handler_2") + + +async def test_smyth_dispatch( + smyth, mocker, mock_event_data_function, mock_context_data_function +): + mock_asend = mocker.patch("smyth.runner.process.RunnerProcess.asend") + mock_request = mocker.Mock() + mock_request.method = "GET" + mock_request.url.path = "/test_handler" + + with smyth: + response = await smyth.dispatch( + smyth.get_handler_for_name("test_handler"), + mock_request, + ) + + assert mock_asend.await_args[0][0]["type"] == "smyth.lambda.invoke" + assert mock_asend.await_args[0][0]["event"] == await mock_event_data_function() + assert mock_asend.await_args[0][0]["context"] == await mock_context_data_function() + assert response == mock_asend.return_value + + +async def test_invoke( + smyth, mocker, mock_event_data_function, mock_context_data_function +): + mock_asend = mocker.patch("smyth.runner.process.RunnerProcess.asend") + event_data = {"test": "data"} + + with smyth: + response = await smyth.invoke( + smyth.get_handler_for_name("test_handler"), + event_data, + ) + + assert mock_asend.await_args[0][0]["type"] == "smyth.lambda.invoke" + assert mock_asend.await_args[0][0]["event"] == event_data + assert mock_asend.await_args[0][0]["context"] == await mock_context_data_function() + assert response == mock_asend.return_value diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..75c4117 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,31 @@ +from smyth.utils import get_logging_config, import_attribute + + +def test_get_logging_config(): + logging_config = get_logging_config("DEBUG") + assert logging_config["version"] == 1 + assert logging_config["disable_existing_loggers"] is False + assert logging_config["filters"] == {} + assert ( + logging_config["handlers"]["console"]["class"] == "smyth.utils.SmythRichHandler" + ) + assert logging_config["handlers"]["console"]["markup"] is True + assert logging_config["handlers"]["console"]["rich_tracebacks"] is True + assert logging_config["handlers"]["console"]["filters"] == [] + assert logging_config["handlers"]["console"]["show_path"] is False + assert logging_config["loggers"]["smyth"]["handlers"] == ["console"] + assert logging_config["loggers"]["smyth"]["level"] == "DEBUG" + assert logging_config["loggers"]["smyth"]["propagate"] is False + assert logging_config["loggers"]["uvicorn"]["handlers"] == ["console"] + assert logging_config["loggers"]["uvicorn"]["level"] == "DEBUG" + assert logging_config["loggers"]["uvicorn"]["propagate"] is False + + logging_config = get_logging_config("INFO", "/smyth") + assert ( + logging_config["filters"]["smyth_api_filter"]["smyth_path_prefix"] == "/smyth" + ) + assert logging_config["handlers"]["console"]["filters"] == ["smyth_api_filter"] + + +def test_import_attribute(): + assert import_attribute("smyth.utils.get_logging_config") == get_logging_config From 2cd67560f831b432b641281106465de45d430866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Kucmus?= Date: Sun, 13 Oct 2024 19:54:50 +0200 Subject: [PATCH 3/5] add uvloop to test deps, release docs on published release, add 3.13 to test matrix --- .github/workflows/deploy_docs.yaml | 7 +++---- .github/workflows/test.yml | 1 + pyproject.toml | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy_docs.yaml b/.github/workflows/deploy_docs.yaml index 565c5fd..b8726d0 100644 --- a/.github/workflows/deploy_docs.yaml +++ b/.github/workflows/deploy_docs.yaml @@ -1,10 +1,9 @@ name: Deploy documentation on: - push: - branches: - - main - - master + release: + types: + - published workflow_call: workflow_dispatch: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0ffdedf..3224eaa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,6 +33,7 @@ jobs: - "3.10" - "3.11" - "3.12" + - "3.13" steps: - name: Checkout source code diff --git a/pyproject.toml b/pyproject.toml index d632d9a..8b30b37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ dependencies = [ "pytest-cov", "coverage[toml]", "httpx", + "uvloop", ] [[tool.hatch.envs.hatch-test.matrix]] From 4b6394b32914d74fdd9929ba9e25cf64945a7d9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Kucmus?= Date: Sun, 13 Oct 2024 20:18:09 +0200 Subject: [PATCH 4/5] use beta uvloop for 3.13 compat --- pyproject.toml | 7 ++++--- tests/conftest.py | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8b30b37..8d1913a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,10 @@ dependencies = [ "pytest-cov", "coverage[toml]", "httpx", - "uvloop", + # uvloop 0.20.0 is broken on Python 3.13 + # https://github.com/MagicStack/uvloop/issues/622 + # waiting for 0.21.0 release + "uvloop==0.21.0b1", ] [[tool.hatch.envs.hatch-test.matrix]] @@ -114,8 +117,6 @@ deploy = "mkdocs gh-deploy --force" ## Pytest configuration [tool.pytest.ini_options] -asyncio_mode = "auto" -asyncio_default_fixture_loop_scope = "function" ## Coverage configuration diff --git a/tests/conftest.py b/tests/conftest.py index 18264d2..d74b249 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,12 @@ @pytest.fixture(autouse=True) def anyio_backend(): + # uvloop is not available for Python 3.13 + # https://github.com/MagicStack/uvloop/issues/622 + # waiting for 0.21.0 release + # import sys + # if sys.version_info < (3, 13): + # return "asyncio" return "asyncio", {"use_uvloop": True} From 9d455fc7f68b61b10c63478bed7250156a600773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Kucmus?= Date: Sun, 13 Oct 2024 21:20:56 +0200 Subject: [PATCH 5/5] adjust the docs --- docs/index.md | 3 +++ docs/user_guide/custom_entrypoint.md | 12 +++++++----- docs/user_guide/non_http.md | 5 +++-- pyproject.toml | 4 ++-- tests/conftest.py | 6 ------ 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/index.md b/docs/index.md index 71fa99f..b389cd6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -104,6 +104,9 @@ sequenceDiagram STAR->>STAR: Lookup handlers by path STAR->>+PROC: Send event and context + alt "Process is cold" + PROC<<->>HAND: Import handler + end PROC->>+HAND: Invoke handler HAND->>-PROC: Result PROC->>-STAR: Result diff --git a/docs/user_guide/custom_entrypoint.md b/docs/user_guide/custom_entrypoint.md index 60b997b..a7bf3a1 100644 --- a/docs/user_guide/custom_entrypoint.md +++ b/docs/user_guide/custom_entrypoint.md @@ -24,9 +24,10 @@ Here's an example `smyth_conf.py` file: ```python title="my_project/etc/smyth_conf.py" linenums="1" import uvicorn -from starlette.requests import Request -from smyth.smyth import Smyth from smyth.server.app import SmythStarlette +from smyth.smyth import Smyth +from starlette.requests import Request + def my_handler(event, context): return {"statusCode": 200, "body": "Hello, World!"} @@ -46,16 +47,17 @@ smyth = Smyth() smyth.add_handler( name="hello", path="/hello", - lambda_handler=my_handler, + lambda_handler_path="smyth_run.my_handler", timeout=1, concurrency=1, - event_data_generator=my_event_data_generator, + event_data_function=my_event_data_generator, ) app = SmythStarlette(smyth=smyth, smyth_path_prefix="/smyth") if __name__ == "__main__": - uvicorn.run("smyth_conf:app", host="0.0.0.0", port=8080, reload=True) + uvicorn.run("smyth_run:app", host="0.0.0.0", port=8080, reload=True) + ``` Normally, the handler would be imported, but including the custom event generator in this file is a good use case. Use the `SmythStarlette` subclass of `Starlette` - it ensures all subprocesses are run at server start and killed on stop (using ASGI Lifetime). Create a Smyth instance and pass it to your `SmythStarlette` instance. Here, you can fine-tune logging, change Uvicorn settings, etc. diff --git a/docs/user_guide/non_http.md b/docs/user_guide/non_http.md index 683c610..54f0238 100644 --- a/docs/user_guide/non_http.md +++ b/docs/user_guide/non_http.md @@ -43,7 +43,7 @@ In your `etc` directory, create a `smyth_run.py` file. smyth.add_handler( name="hello", path="/hello", - lambda_handler=my_handler, + lambda_handler_path="smyth_run.my_handler", timeout=60, concurrency=10, strategy_generator=round_robin, @@ -65,6 +65,7 @@ In your `etc` directory, create a `smyth_run.py` file. if __name__ == "__main__": with smyth: asyncio.run(main()) + ``` ### Import and Declare the Basics @@ -118,7 +119,7 @@ smyth = Smyth() smyth.add_handler( name="hello", path="/hello", - lambda_handler=my_handler, + lambda_handler_path="smyth_run.my_handler", timeout=60, concurrency=10, strategy_generator=round_robin, diff --git a/pyproject.toml b/pyproject.toml index 8d1913a..45d04cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ Source = "https://github.com/mirumee/smyth" [project.optional-dependencies] dev = ["ipdb"] types = ["mypy>=1.0.0", "pytest", "types-toml", "pytest-asyncio"] -docs = ["mkdocs-material"] +docs = ["mkdocs-material", "termynal"] [tool.hatch.version] path = "src/smyth/__about__.py" @@ -156,7 +156,7 @@ docstring-code-line-length = 80 [tool.ruff.lint] select = ["E", "F", "G", "I", "N", "Q", "UP", "C90", "T20", "TID"] -unfixable = ["UP007"] # typer does not handle PEP604 annotations +unfixable = ["UP007"] # typer does not handle PEP604 annotations [tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" diff --git a/tests/conftest.py b/tests/conftest.py index d74b249..18264d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,12 +9,6 @@ @pytest.fixture(autouse=True) def anyio_backend(): - # uvloop is not available for Python 3.13 - # https://github.com/MagicStack/uvloop/issues/622 - # waiting for 0.21.0 release - # import sys - # if sys.version_info < (3, 13): - # return "asyncio" return "asyncio", {"use_uvloop": True}