From cbd6add902eff6c700142dcdf684f1b107fe71c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Kucmus?= Date: Sun, 22 Dec 2024 22:29:07 +0100 Subject: [PATCH] feat!: add the AWS Lambda environment variables and allow to overload them in config BREAKING CHANGE: to allow access to more context, the event generator functions now take the the Starlette.Request, a SmythHandler instance and a RunnerProcessProtocol compatible type. Previously there was only the request --- .gitignore | 162 +++++++++++++++++++++++++-- docs/user_guide/all_settings.md | 36 +++--- docs/user_guide/custom_entrypoint.md | 3 +- docs/user_guide/environment.md | 61 ++++++++++ docs/user_guide/event_functions.md | 5 +- docs/user_guide/index.md | 2 +- mkdocs.yml | 2 +- pyproject.toml | 16 +-- src/smyth/config.py | 9 ++ src/smyth/context.py | 4 +- src/smyth/event.py | 10 +- src/smyth/runner/fake_context.py | 37 +++--- src/smyth/runner/process.py | 20 +++- src/smyth/server/app.py | 1 + src/smyth/smyth.py | 6 +- src/smyth/types.py | 73 +++++++++++- tests/conftest.py | 48 +++++--- tests/runner/test_fake_context.py | 46 +++++--- tests/runner/test_process.py | 2 +- tests/server/test_app.py | 2 + tests/test_config.py | 55 +++------ tests/test_context.py | 1 + tests/test_event.py | 8 +- 23 files changed, 465 insertions(+), 144 deletions(-) create mode 100644 docs/user_guide/environment.md diff --git a/.gitignore b/.gitignore index d50861f..89711b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + # Distribution / packaging .Python -env/ build/ develop-eggs/ dist/ @@ -12,20 +19,155 @@ lib64/ parts/ sdist/ var/ +wheels/ +share/python-wheels/ *.egg-info/ .installed.cfg *.egg +MANIFEST +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec -.ruff_cache/ -__pycache__ +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock -.DS_Store +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments .env -.pytest_cache -.python-version +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ -package -*.zip -.mypy_cache -.coverage +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# PyPI configuration file +.pypirc + +.ruff_cache/ diff --git a/docs/user_guide/all_settings.md b/docs/user_guide/all_settings.md index 8acd846..b636e37 100644 --- a/docs/user_guide/all_settings.md +++ b/docs/user_guide/all_settings.md @@ -4,22 +4,24 @@ Here's a list of all the settings, including those that are simpler but equally ## Smyth Settings -### Host +### Socket Binding `host` - `str` (default: `"0.0.0.0"`) Used by Uvicorn to bind to an address. -### Port - `port` - `int` (default: `8080`) Used by Uvicorn as the bind port. -### Log Level +### Logging `log_level` - `str` (default: `"INFO"`) Sets the logging level for the `uvicorn` and `smyth` logging handlers. -### Smyth Path Prefix +### Smyth Internals `smyth_path_prefix` - `str` (default: `"/smyth"`) The path prefix used for Smyth's status endpoint. Change this if, for any reason, it collides with your path routing. +### Environment + +`env` - `dict[str, str]` (default: `{}`) Environment variables to apply to every handler. Read more about [environment variables here](environment.md). + ## Handler Settings ### Handler Path @@ -28,32 +30,30 @@ Here's a list of all the settings, including those that are simpler but equally ### URL Path -`url_path` - `str` (required) The Starlette routing path on which your handler will be exposed. +`url_path` - `str` (required) The [Starlette routing](https://www.starlette.io/routing/#http-routing) path on which your handler will be exposed. -### Timeout +### Environment -`timeout` - `float` (default: `None`, which means no timeout) The time in seconds after which the Lambda Handler raises a Timeout Exception, simulating Lambda's real-life timeouts. +`env` - `dict[str, str]` (default: `{}`) Environment variables to apply to this handler - keys defined here take precedence over the ones defined in `tool.smyth.env` and be otherwise merged. Read more about [environment variables here](environment.md). -### Event Data Function +### Customization `event_data_function_path` - `str` (default: `"smyth.event.generate_api_gw_v2_event_data"`) Read more about [event functions here](event_functions.md). -### Context Data Function - `context_data_function_path` - `str` (default: `"smyth.context.generate_context_data"`) A function similar to the [event generator](event_functions.md), but it constructs the `context`, adding some metadata from Smyth's runtime. You can create and use your own. -### Log Level - -`log_level` - `str` (default: `"INFO"`) Log level for Smyth's runner function, which is still part of Smyth but already running in the subprocess. Note that the logging of your Lambda handler code should be set separately. +### Behaviour -### Concurrency +`timeout` - `float` (default: `None`, which means no timeout) The time in seconds after which the Lambda Handler raises a Timeout Exception, simulating Lambda's real-life timeouts. `concurrency` - `int` (default: `1`) Read more about [concurrency here](concurrency.md). -### Strategy Generator - `strategy_generator_path` - `str` (default: `"smyth.runner.strategy.first_warm"`) Read more about [dispatch strategies here](concurrency.md/#dispatch-strategy). +### Logging + +`log_level` - `str` (default: `"INFO"`) Log level for Smyth's runner function, which is still part of Smyth but already running in the subprocess. Note that the logging of your Lambda handler code should be set separately. + ## `pyproject.toml` example @@ -65,7 +65,7 @@ log_level = "INFO" smyth_path_prefix = "/smyth" [tool.smyth.handlers.lambda_handler] -handler_path = "nimara_search_algolia_backend.app.lambda_handler" +handler_path = "myproject.app.lambda_handler" url_path = "{path:path}" timeout = 300 event_data_function_path = "smyth.event.generate_api_gw_v2_event_data" diff --git a/docs/user_guide/custom_entrypoint.md b/docs/user_guide/custom_entrypoint.md index a7bf3a1..bd741ef 100644 --- a/docs/user_guide/custom_entrypoint.md +++ b/docs/user_guide/custom_entrypoint.md @@ -26,13 +26,14 @@ Here's an example `smyth_conf.py` file: import uvicorn from smyth.server.app import SmythStarlette from smyth.smyth import Smyth +from smyth.types import EventData, RunnerProcessProtocol, SmythHandler from starlette.requests import Request def my_handler(event, context): return {"statusCode": 200, "body": "Hello, World!"} -async def my_event_data_generator(request: Request): +async def my_event_data_generator(request: Request, smyth_handler: SmythHandler, process: RunnerProcessProtocol) -> EventData: return { "requestContext": { "http": { diff --git a/docs/user_guide/environment.md b/docs/user_guide/environment.md new file mode 100644 index 0000000..207b4d9 --- /dev/null +++ b/docs/user_guide/environment.md @@ -0,0 +1,61 @@ +# Environment + +Smyth allows you to overwrite the environemnt variables that are passed to the handlers to better reflect the actual AWS Lambda environment while allowing you to change things while developing locally. + + +```toml title='pyproject.toml' linenums="1" +[tool.smyth] +host = "0.0.0.0" +port = 8080 +... + +[tool.smyth.env] +AWS_ENDPOINT = "http://localstack:4566" +AWS_LAMBDA_FUNCTION_VERSION = "$SMYTH" + +[tool.smyth.handlers.my_special_version_handler] +handler_path = "mypyoject.app.my_special_version_handler" +url_path = "{path:path}" +... + +[tool.smyth.handlers.my_special_version_handler.env] +AWS_LAMBDA_FUNCTION_VERSION = "34" +``` + +The config above allows you to set a specific env var for every defined handler and overwrite or set +specific values for individual handlers. In the example every handler would receive the +`AWS_ENDPOINT = "http://localstack:4566"` and `AWS_LAMBDA_FUNCTION_VERSION = "$SMYTH"` env vars with +the exception of `my_special_version_handler` which will have a different version. + +## Fake context + +The `smyth.runner.fake_context.FakeLambdaContext` class used by Smyth will also consume of of the environment variables. + +## Default variables + +In the table bellow you will find which keys are set by Smyth when a handler is being invoked. +Smyth will look for the key in the following order: + +1. the handler configuration `env` key +2. the smyth global configuration `env` key +3. `os.environ` +4. when none of the above contain the key the default value is assigned + +| Key | Default Value | +| ----------------------------------- | ---------------------------------------------------------------------- | +| `"_HANDLER"` | `self.lambda_handler_path` | +| `"AWS_ACCESS_KEY_ID"` | `"000000000000"` | +| `"AWS_SECRET_ACCESS_KEY"` | `"test"` | +| `"AWS_SESSION_TOKEN"` | `"test"` | +| `"AWS_DEFAULT_REGION"` | `"eu-central-1"` | +| `"AWS_REGION"` | `"eu-central-1"` | +| `"AWS_EXECUTION_ENV"` | `"AWS_Lambda_python{sys.version_info.major}.{sys.version_info.minor}"` | +| `"AWS_LAMBDA_FUNCTION_MEMORY_SIZE"` | `"128"` | +| `"AWS_LAMBDA_FUNCTION_NAME"` | `self.name` | +| `"AWS_LAMBDA_FUNCTION_VERSION"` | `"$LATEST"` | +| `"AWS_LAMBDA_INITIALIZATION_TYPE"` | `"on-demand"` | +| `"AWS_LAMBDA_LOG_GROUP_NAME"` | `"/aws/lambda/{self.name}"` | +| `"AWS_LAMBDA_LOG_STREAM_NAME"` | `"{strftime('%Y/%m/%d')}/[$LATEST]smyth_aws_lambda_log_stream_name"` | +| `"AWS_LAMBDA_RUNTIME_API"` | `"127.0.0.1:9001"` | +| `"AWS_XRAY_CONTEXT_MISSING"` | `"LOG_ERROR"` | +| `"AWS_XRAY_DAEMON_ADDRESS"` | `"127.0.0.1:2000"` | diff --git a/docs/user_guide/event_functions.md b/docs/user_guide/event_functions.md index 0330749..96dfa56 100644 --- a/docs/user_guide/event_functions.md +++ b/docs/user_guide/event_functions.md @@ -13,7 +13,10 @@ The first one builds a minimal API Gateway Proxy V2 event to simulate a Lambda b If you need to work with events not covered by Smyth, you can create and provide your own. Assuming a simplified API Gateway V1 event, you can create a generator like this: ```python title="my_project/src/smyth_utils/event.py" linenums="1" -async def generate_api_gw_v1_event_data(request: Request): +from smyth.types import EventData, RunnerProcessProtocol, SmythHandler + + +async def generate_api_gw_v1_event_data(request: Request, smyth_handler: SmythHandler, process: RunnerProcessProtocol) -> EventData: source_ip = None if request.client: source_ip = request.client.host diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index 2f3e648..b51a0ac 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -1,4 +1,4 @@ -# User Guide +# First Steps Smyth is built to have minimal or no impact on the project you are working on. That said, it comes with features that allow you to customize Smyth to the needs of your Lambda project. diff --git a/mkdocs.yml b/mkdocs.yml index 678ba43..dfc115d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,7 +18,6 @@ theme: - content.code.copy - content.code.annotate - content.tabs.link - - navigation.indexes - navigation.footer - navigation.tracking - navigation.expand @@ -31,6 +30,7 @@ nav: - user_guide/event_functions.md - user_guide/invoke.md - user_guide/concurrency.md + - user_guide/environment.md - user_guide/all_settings.md - user_guide/custom_entrypoint.md - user_guide/non_http.md diff --git a/pyproject.toml b/pyproject.toml index be67de0..d30de4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,9 +66,10 @@ features = ["dev", "types", "docs"] [tool.hatch.envs.default.scripts] check = [ "hatch fmt", - "hatch test -a", + "hatch test -a -p", "hatch test --cover", "hatch run types:check", + "hatch run docs:build", ] ## Types environment @@ -79,21 +80,16 @@ check = "mypy --install-types --non-interactive {args:src/smyth}" ## Test environment [tool.hatch.envs.hatch-test] -dependencies = [ - "asynctest", +extra-dependencies = [ "ipdb", "anyio", - "pytest-mock", - "pytest-memray", + "pytest-freezer", "pytest-print", "pytest-cov", - "coverage[toml]", "httpx", - # uvloop 0.20.0 is broken on Python 3.13 - # https://github.com/MagicStack/uvloop/issues/622 - # waiting for 0.21.0 release - "uvloop==0.21.0b1", + "uvloop>=0.21", ] +extra-args = [] [[tool.hatch.envs.hatch-test.matrix]] python = ["3.10", "3.11", "3.12", "3.13"] diff --git a/src/smyth/config.py b/src/smyth/config.py index 5a1b480..2fccba3 100644 --- a/src/smyth/config.py +++ b/src/smyth/config.py @@ -1,5 +1,6 @@ import json import os +from copy import deepcopy from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any @@ -7,6 +8,7 @@ import toml from smyth.exceptions import ConfigFileNotFoundError +from smyth.types import Environ @dataclass @@ -19,6 +21,12 @@ class HandlerConfig: log_level: str = "DEBUG" concurrency: int = 1 strategy_generator_path: str = "smyth.runner.strategy.first_warm" + env: Environ = field(default_factory=dict) + + def get_env_overrides(self, config: "Config") -> Environ: + env = deepcopy(config.env) + env.update(self.env) + return env @dataclass @@ -28,6 +36,7 @@ class Config: handlers: dict[str, HandlerConfig] = field(default_factory=dict) log_level: str = "INFO" smyth_path_prefix: str = "/smyth" + env: Environ = field(default_factory=dict) @classmethod def from_dict(cls, config_dict: dict[str, Any]) -> "Config": diff --git a/src/smyth/context.py b/src/smyth/context.py index 461b30e..f5ef6df 100644 --- a/src/smyth/context.py +++ b/src/smyth/context.py @@ -3,12 +3,12 @@ from starlette.requests import Request -from smyth.types import RunnerProcessProtocol, SmythHandler +from smyth.types import ContextData, RunnerProcessProtocol, SmythHandler async def generate_context_data( request: Request | None, smyth_handler: SmythHandler, process: RunnerProcessProtocol -) -> dict[str, Any]: +) -> ContextData: """ The data returned by this function is passed to the `smyth.runner.FaneContext` as kwargs. diff --git a/src/smyth/event.py b/src/smyth/event.py index b4e19cf..c7da74d 100644 --- a/src/smyth/event.py +++ b/src/smyth/event.py @@ -2,10 +2,12 @@ from starlette.requests import Request -from smyth.types import EventData +from smyth.types import EventData, RunnerProcessProtocol, SmythHandler -async def generate_api_gw_v2_event_data(request: Request) -> EventData: +async def generate_api_gw_v2_event_data( + request: Request, smyth_handler: SmythHandler, process: RunnerProcessProtocol +) -> EventData: source_ip = None if request.client: source_ip = request.client.host @@ -32,5 +34,7 @@ async def generate_api_gw_v2_event_data(request: Request) -> EventData: } -async def generate_lambda_invocation_event_data(request: Request) -> Any: +async def generate_lambda_invocation_event_data( + request: Request, smyth_handler: SmythHandler, process: RunnerProcessProtocol +) -> Any: return await request.json() diff --git a/src/smyth/runner/fake_context.py b/src/smyth/runner/fake_context.py index 48821f3..031d17c 100644 --- a/src/smyth/runner/fake_context.py +++ b/src/smyth/runner/fake_context.py @@ -1,3 +1,4 @@ +import os import sys from collections.abc import Callable from time import strftime, time @@ -10,23 +11,23 @@ class FakeLambdaContext(LambdaContext): def __init__( self, name: str | None = None, - version: str | None = "LATEST", + version: str | None = None, timeout: int | None = None, **kwargs: Any, ): if name is None: - name = "Fake" - self.name = name + name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME", "Fake") + self._name = name if version is None: - version = "LATEST" - self.version = version + version = os.environ.get("AWS_LAMBDA_FUNCTION_VERSION", "$LATEST") + self._version = version - self.created = time() + self._created = time() if timeout is None: timeout = 6 - self.timeout = timeout + self._timeout = timeout for key, value in kwargs.items(): setattr(self, key, value) @@ -34,28 +35,28 @@ def __init__( def get_remaining_time_in_millis(self) -> int: # type: ignore[override] return int( max( - (self.timeout * 1000) - - (int(round(time() * 1000)) - int(round(self.created * 1000))), + (self._timeout * 1000) + - (int(round(time() * 1000)) - int(round(self._created * 1000))), 0, ) ) @property def function_name(self) -> str: - return self.name + return self._name @property def function_version(self) -> str: - return self.version + return self._version @property def invoked_function_arn(self) -> str: - return "arn:aws:lambda:serverless:" + self.name + return "arn:aws:lambda:serverless:" + self._name @property # This indeed is a string in the real context hence the ignore[override] def memory_limit_in_mb(self) -> str: # type: ignore[override] - return "1024" + return os.environ.get("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "128") @property def aws_request_id(self) -> str: @@ -63,15 +64,13 @@ def aws_request_id(self) -> str: @property def log_group_name(self) -> str: - return "/aws/lambda/" + self.name + return os.environ.get("AWS_LAMBDA_LOG_GROUP_NAME", f"/aws/lambda/{self._name}") @property def log_stream_name(self) -> str: - return ( - strftime("%Y/%m/%d") - + "/[$" - + self.version - + "]58419525dade4d17a495dceeeed44708" + return os.environ.get( + "AWS_LAMBDA_LOG_STREAM_NAME", + f"{strftime('%Y/%m/%d')}/[{self._version}]smyth_aws_lambda_log_stream_name", ) @property diff --git a/src/smyth/runner/process.py b/src/smyth/runner/process.py index 2146e01..8b2e9c3 100644 --- a/src/smyth/runner/process.py +++ b/src/smyth/runner/process.py @@ -1,6 +1,7 @@ import inspect import logging import logging.config +import os import signal import sys import traceback @@ -44,11 +45,25 @@ class RunnerProcess(Process): last_used_timestamp: float state: SmythHandlerState - def __init__(self, name: str, lambda_handler_path: str, log_level: str = "INFO"): + def __init__( + self, + name: str, + lambda_handler_path: str, + log_level: str = "INFO", + environ_override: dict[str, str] | None = None, + ): self.name = name self.task_counter = 0 self.last_used_timestamp = 0 self.state = SmythHandlerState.COLD + self.environ_override = environ_override + + self.environ: dict[str, str] = { + "_HANDLER": lambda_handler_path, + } + + if environ_override: + self.environ.update(environ_override) self.input_queue: Queue[RunnerInputMessage] = Queue(maxsize=1) self.output_queue: Queue[RunnerOutputMessage] = Queue(maxsize=1) @@ -113,6 +128,7 @@ def asend(self, data: RunnerInputMessage) -> LambdaResponse | None: def run(self) -> None: setproctitle(f"smyth:{self.name}") logging.config.dictConfig(get_logging_config(self.log_level)) + os.environ.update(self.environ) self.lambda_invoker__() def get_message__(self) -> Generator[RunnerInputMessage, None, None]: @@ -197,7 +213,7 @@ def lambda_invoker__(self) -> None: self.set_status__(SmythHandlerState.WARM) signal.signal(signal.SIGALRM, self.timeout_handler__) - signal.alarm(int(context.timeout)) + signal.alarm(int(context._timeout)) self.set_status__(SmythHandlerState.WORKING) try: response = lambda_handler(event, context) diff --git a/src/smyth/server/app.py b/src/smyth/server/app.py index 9ed5bab..253bcee 100644 --- a/src/smyth/server/app.py +++ b/src/smyth/server/app.py @@ -73,6 +73,7 @@ def create_app() -> SmythStarlette: log_level=handler_config.log_level, concurrency=handler_config.concurrency, strategy_generator=import_attribute(handler_config.strategy_generator_path), + env_overrides=handler_config.get_env_overrides(config), ) app = SmythStarlette(smyth=smyth, smyth_path_prefix=config.smyth_path_prefix) diff --git a/src/smyth/smyth.py b/src/smyth/smyth.py index 23eff92..a0c3b73 100644 --- a/src/smyth/smyth.py +++ b/src/smyth/smyth.py @@ -14,6 +14,7 @@ from smyth.runner.strategy import first_warm from smyth.types import ( ContextDataCallable, + Environ, EventData, EventDataCallable, LambdaResponse, @@ -49,6 +50,7 @@ def add_handler( log_level: str = "INFO", concurrency: int = 1, strategy_generator: StrategyGenerator = first_warm, + env_overrides: Environ | None = None, ) -> None: self.smyth_handlers[name] = SmythHandler( name=name, @@ -60,6 +62,7 @@ def add_handler( log_level=log_level, concurrency=concurrency, strategy_generator=strategy_generator, + env_overrides=env_overrides, ) def __enter__(self: Self) -> Self: @@ -82,6 +85,7 @@ def start_runners(self) -> None: name=f"{handler_name}:{index}", lambda_handler_path=handler_config.lambda_handler_path, log_level=handler_config.log_level, + environ_override=handler_config.get_environ(), ) process.start() LOGGER.info("Started process %s", process.name) @@ -134,7 +138,7 @@ async def dispatch( if event_data_function is None: event_data_function = smyth_handler.event_data_function - event_data = await event_data_function(request) + event_data = await event_data_function(request, smyth_handler, process) context_data = await smyth_handler.context_data_function( request, smyth_handler, process ) diff --git a/src/smyth/types.py b/src/smyth/types.py index c4ce881..c21e728 100644 --- a/src/smyth/types.py +++ b/src/smyth/types.py @@ -1,7 +1,10 @@ +import os +import sys from collections.abc import Awaitable, Callable, Iterator, MutableMapping from dataclasses import dataclass from enum import Enum from re import Pattern +from time import strftime from typing import Annotated, Any, Literal, Protocol, TypeAlias from aws_lambda_powertools.utilities.typing import LambdaContext @@ -10,7 +13,9 @@ LambdaEvent: TypeAlias = MutableMapping[str, Any] EventData: TypeAlias = dict[str, Any] -EventDataCallable: TypeAlias = Callable[[Request], Awaitable[EventData]] +EventDataCallable: TypeAlias = Callable[ + [Request, "SmythHandler", "RunnerProcessProtocol"], Awaitable[EventData] +] ContextData: TypeAlias = dict[str, Any] ContextDataCallable: TypeAlias = Callable[ [Request | None, "SmythHandler", "RunnerProcessProtocol"], Awaitable[ContextData] @@ -19,6 +24,7 @@ [str, dict[str, list["RunnerProcessProtocol"]]], Iterator["RunnerProcessProtocol"], ] +Environ: TypeAlias = dict[str, str] class SmythHandlerState(str, Enum): @@ -77,6 +83,8 @@ class RunnerProcessProtocol(Protocol): async def asend(self, data: RunnerInputMessage) -> LambdaResponse | None: ... + def start(self) -> None: ... + def stop(self) -> None: ... def send(self, data: RunnerInputMessage) -> LambdaResponse | None: ... @@ -99,3 +107,66 @@ class SmythHandler: timeout: float | None = None log_level: str = "INFO" concurrency: int = 1 + env_overrides: Environ | None = None + + def _get_env_value(self, key: str, default: str) -> str: + """ + Helper method to retrieve an environment variable value with the following + precedence: + 1. `self.env_overrides` if defined. + 2. `os.environ` if the key exists. + 3. The provided default value. + """ + if self.env_overrides and key in self.env_overrides: + return self.env_overrides[key] + return os.environ.get(key, default) + + def get_environ(self) -> Environ: + envs = { + "_HANDLER": self._get_env_value("_HANDLER", self.lambda_handler_path), + "AWS_ACCESS_KEY_ID": self._get_env_value( + "AWS_ACCESS_KEY_ID", "000000000000" + ), + "AWS_SECRET_ACCESS_KEY": self._get_env_value( + "AWS_SECRET_ACCESS_KEY", "test" + ), + "AWS_SESSION_TOKEN": self._get_env_value("AWS_SESSION_TOKEN", "test"), + "AWS_DEFAULT_REGION": self._get_env_value( + "AWS_DEFAULT_REGION", "eu-central-1" + ), + "AWS_REGION": self._get_env_value("AWS_REGION", "eu-central-1"), + "AWS_EXECUTION_ENV": self._get_env_value( + "AWS_EXECUTION_ENV", + f"AWS_Lambda_python{sys.version_info.major}.{sys.version_info.minor}", + ), + "AWS_LAMBDA_FUNCTION_MEMORY_SIZE": self._get_env_value( + "AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "128" + ), + "AWS_LAMBDA_FUNCTION_NAME": self._get_env_value( + "AWS_LAMBDA_FUNCTION_NAME", self.name + ), + "AWS_LAMBDA_FUNCTION_VERSION": self._get_env_value( + "AWS_LAMBDA_FUNCTION_VERSION", "$LATEST" + ), + "AWS_LAMBDA_INITIALIZATION_TYPE": self._get_env_value( + "AWS_LAMBDA_INITIALIZATION_TYPE", "on-demand" + ), + "AWS_LAMBDA_LOG_GROUP_NAME": self._get_env_value( + "AWS_LAMBDA_LOG_GROUP_NAME", f"/aws/lambda/{self.name}" + ), + "AWS_LAMBDA_LOG_STREAM_NAME": self._get_env_value( + "AWS_LAMBDA_LOG_STREAM_NAME", + f"{strftime('%Y/%m/%d')}/[$LATEST]smyth_aws_lambda_log_stream_name", + ), + "AWS_LAMBDA_RUNTIME_API": self._get_env_value( + "AWS_LAMBDA_RUNTIME_API", "127.0.0.1:9001" + ), + "AWS_XRAY_CONTEXT_MISSING": self._get_env_value( + "AWS_XRAY_CONTEXT_MISSING", "LOG_ERROR" + ), + "AWS_XRAY_DAEMON_ADDRESS": self._get_env_value( + "AWS_XRAY_DAEMON_ADDRESS", "127.0.0.1:2000" + ), + } + + return envs diff --git a/tests/conftest.py b/tests/conftest.py index 1c5b96a..7bea934 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import pytest -from smyth.config import Config, HandlerConfig +from smyth.config import Config from smyth.types import RunnerProcessProtocol, SmythHandler, SmythHandlerState @@ -26,26 +26,40 @@ def smyth_handler( event_data_function=mock_event_data_function, context_data_function=mock_context_data_function, strategy_generator=mock_strategy_generator, + env_overrides={ + "TEST_ENV": "test", + }, ) @pytest.fixture -def config(): - return Config( - host="0.0.0.0", - port=8080, - handlers={ - "order_handler": HandlerConfig( - handler_path="tests.conftest.example_handler", - url_path=r"/test_handler", - ), - "product_handler": HandlerConfig( - handler_path="tests.conftest.example_handler", - url_path=r"/products/{path:path}", - ), - }, - log_level="INFO", - ) +def config_toml_dict(): + return { + "tool": { + "smyth": { + "host": "0.0.0.0", + "port": 8080, + "env": {"ROOT_ENV": "root", "TEST_ENV": "root"}, + "handlers": { + "order_handler": { + "handler_path": "tests.conftest.example_handler", + "url_path": "/test_handler", + "env": {"TEST_ENV": "child"}, + }, + "product_handler": { + "handler_path": "tests.conftest.example_handler", + "url_path": "/products/{path:path}", + }, + }, + "log_level": "INFO", + } + } + } + + +@pytest.fixture +def config(config_toml_dict): + return Config.from_dict(config_toml_dict["tool"]["smyth"]) def example_handler(event, context): diff --git a/tests/runner/test_fake_context.py b/tests/runner/test_fake_context.py index 849c4d1..e83c83b 100644 --- a/tests/runner/test_fake_context.py +++ b/tests/runner/test_fake_context.py @@ -1,20 +1,29 @@ -from time import strftime - import pytest +from freezegun.api import FrozenDateTimeFactory +from pytest_mock import MockerFixture from smyth.runner.fake_context import FakeLambdaContext -def test_fake_lambda_context(): +def test_fake_lambda_context(freezer: FrozenDateTimeFactory, mocker: MockerFixture): + freezer.move_to("2024-12-20 00:00:00") + expected_name = "Test Name Set In Test" + environ = mocker.patch.dict("os.environ", clear=True) + environ.update( + { + "AWS_LAMBDA_FUNCTION_NAME": expected_name, + } + ) context = FakeLambdaContext() - assert context.function_name == "Fake" - assert context.function_version == "LATEST" - assert context.invoked_function_arn == "arn:aws:lambda:serverless:Fake" - assert context.memory_limit_in_mb == "1024" + + assert context.function_name == expected_name + assert context.function_version == "$LATEST" + assert context.invoked_function_arn == f"arn:aws:lambda:serverless:{expected_name}" + assert context.memory_limit_in_mb == "128" assert context.aws_request_id == "1234567890" - assert context.log_group_name == "/aws/lambda/Fake" + assert context.log_group_name == f"/aws/lambda/{expected_name}" assert context.log_stream_name == ( - f"{strftime('%Y/%m/%d')}/[$LATEST]58419525dade4d17a495dceeeed44708" + "2024/12/20/[$LATEST]smyth_aws_lambda_log_stream_name" ) @@ -30,22 +39,29 @@ def test_fake_lambda_context(): [ ("test", "test", 60, "test", "test", 60), ("test", "test", None, "test", "test", 6), - ("test", None, 120, "test", "LATEST", 120), + ("test", None, 120, "test", "$LATEST", 120), (None, "test", 6, "Fake", "test", 6), - (None, None, 6, "Fake", "LATEST", 6), + (None, None, 6, "Fake", "$LATEST", 6), ], ) def test_fake_lambda_context_with_params( - name, version, timeout, expected_name, expected_version, expected_timeout + name, + version, + timeout, + expected_name, + expected_version, + expected_timeout, + freezer: FrozenDateTimeFactory, ): + freezer.move_to("2024-12-20 00:00:00") context = FakeLambdaContext(name=name, version=version, timeout=timeout) assert context.function_name == expected_name assert context.function_version == expected_version - assert context.timeout == expected_timeout + assert context._timeout == expected_timeout assert context.invoked_function_arn == f"arn:aws:lambda:serverless:{expected_name}" - assert context.memory_limit_in_mb == "1024" + assert context.memory_limit_in_mb == "128" assert context.aws_request_id == "1234567890" assert context.log_group_name == f"/aws/lambda/{expected_name}" assert context.log_stream_name == ( - f"{strftime('%Y/%m/%d')}/[${expected_version}]58419525dade4d17a495dceeeed44708" + f"2024/12/20/[{expected_version}]smyth_aws_lambda_log_stream_name" ) diff --git a/tests/runner/test_process.py b/tests/runner/test_process.py index 62e0dad..9d83812 100644 --- a/tests/runner/test_process.py +++ b/tests/runner/test_process.py @@ -100,7 +100,7 @@ def test_get_context(mocker, runner_process): assert ( runner_process.get_context__( RunnerInputMessage(type="smyth.lambda.invoke", event={}, context={}) - ).timeout + )._timeout == 6 ) diff --git a/tests/server/test_app.py b/tests/server/test_app.py index 8e084e4..ae7e757 100644 --- a/tests/server/test_app.py +++ b/tests/server/test_app.py @@ -38,6 +38,7 @@ def test_create_app(mocker, mock_get_config): log_level="DEBUG", concurrency=1, strategy_generator=first_warm, + env_overrides={"TEST_ENV": "child", "ROOT_ENV": "root"}, ), mocker.call( name="product_handler", @@ -49,6 +50,7 @@ def test_create_app(mocker, mock_get_config): log_level="DEBUG", concurrency=1, strategy_generator=first_warm, + env_overrides={"TEST_ENV": "root", "ROOT_ENV": "root"}, ), ] ) diff --git a/tests/test_config.py b/tests/test_config.py index 4c5e6fb..71ae06c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,7 +6,6 @@ import pytest from smyth.config import ( - Config, HandlerConfig, get_config, get_config_dict, @@ -42,28 +41,8 @@ def test_get_config_dict(mocker): mock_get_config_file_path.assert_called_once_with("other.toml") -def test_get_config(): - config_dict = { - "tool": { - "smyth": { - "host": "0.0.0.0", - "port": 8080, - "handlers": { - "order_handler": { - "handler_path": "tests.conftest.example_handler", - "url_path": "/test_handler", - }, - "product_handler": { - "handler_path": "tests.conftest.example_handler", - "url_path": "/products/{path:path}", - }, - }, - "log_level": "INFO", - } - } - } - - config = get_config(config_dict) +def test_get_config(config_toml_dict): + config = get_config(config_toml_dict) assert config.host == "0.0.0.0" assert config.port == 8080 @@ -71,6 +50,7 @@ def test_get_config(): "order_handler": HandlerConfig( handler_path="tests.conftest.example_handler", url_path=r"/test_handler", + env={"TEST_ENV": "child"}, ), "product_handler": HandlerConfig( handler_path="tests.conftest.example_handler", @@ -84,21 +64,18 @@ def test_get_config(): assert get_config(None) == config -def test_serialize_config(): - config = Config( - host="0.0.0.0", - port=8080, - handlers={ - "order_handler": HandlerConfig( - handler_path="tests.conftest.example_handler", - url_path=r"/test_handler", - ), - "product_handler": HandlerConfig( - handler_path="tests.conftest.example_handler", - url_path=r"/products/{path:path}", - ), - }, - log_level="INFO", +def test_serialize_config(config): + assert serialize_config(config) == json.dumps(asdict(config)) + + +def test_get_env_overrides(config): + handler_config = HandlerConfig( + handler_path="tests.conftest.example_handler", + url_path=r"/test_handler", + env={"TEST_ENV": "test"}, ) - assert serialize_config(config) == json.dumps(asdict(config)) + assert handler_config.get_env_overrides(config) == { + "ROOT_ENV": "root", + "TEST_ENV": "test", + } diff --git a/tests/test_context.py b/tests/test_context.py index 81b8e77..ed746d6 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -22,6 +22,7 @@ async def test_generate_context_data( "strategy_generator": ANY, "timeout": None, "url_path": re.compile("/test_handler"), + "env_overrides": {"TEST_ENV": "test"}, }, "name": "test_handler", }, diff --git a/tests/test_event.py b/tests/test_event.py index 0e2c33a..88ae18f 100644 --- a/tests/test_event.py +++ b/tests/test_event.py @@ -19,7 +19,9 @@ async def test_generate_api_gw_v2_event_data(mocker): mock_request.url.query = "" mock_request.url.scheme = "http" - assert await generate_api_gw_v2_event_data(mock_request) == { + assert await generate_api_gw_v2_event_data( + mock_request, mocker.Mock(), mocker.Mock() + ) == { "version": "2.0", "rawPath": "/test", "body": "", @@ -46,4 +48,6 @@ async def test_generate_lambda_invokation_event_data(mocker): mock_request = mocker.Mock() mock_request.json = mocker.AsyncMock(return_value={"test": "test"}) - assert await generate_lambda_invocation_event_data(mock_request) == {"test": "test"} + assert await generate_lambda_invocation_event_data( + mock_request, mocker.Mock(), mocker.Mock() + ) == {"test": "test"}