From 95e54ffbd4f10af12f78316dd5e39584da1081e0 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:13:22 +0200 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8FMypy:=20webserver=20(#6193)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/models_library/basic_types.py | 5 + .../src/models_library/projects_nodes.py | 9 +- .../src/models_library/projects_nodes_io.py | 13 +- .../servicelib/aiohttp/requests_validation.py | 2 +- .../src/servicelib/logging_utils.py | 130 +++++++-------- .../tests/test_logging_utils.py | 154 +++++++++++++++--- .../simcore_service_agent/core/application.py | 4 +- .../core/application.py | 4 +- .../services/solver_job_models_converters.py | 4 +- .../src/simcore_service_autoscaling/main.py | 5 +- .../src/simcore_service_catalog/main.py | 5 +- .../simcore_service_clusters_keeper/main.py | 5 +- .../src/simcore_service_dask_sidecar/tasks.py | 4 +- .../core/application.py | 4 +- .../api/routes/dynamic_services.py | 7 +- .../core/application.py | 4 +- .../simcore_service_dynamic_scheduler/main.py | 5 +- .../core/application.py | 4 +- .../src/simcore_service_efs_guardian/main.py | 5 +- .../src/simcore_service_invitations/main.py | 5 +- .../src/simcore_service_payments/main.py | 5 +- .../web_main.py | 5 +- .../src/simcore_service_storage/cli.py | 4 +- .../catalog/_handlers.py | 13 +- .../catalog/_models.py | 6 +- .../catalog/exceptions.py | 2 +- .../catalog/plugin.py | 3 +- .../clusters/_handlers.py | 2 +- .../diagnostics/_handlers.py | 3 +- .../simcore_service_webserver/email/_core.py | 26 +-- .../folders/_folders_handlers.py | 2 +- .../groups/_handlers.py | 4 +- .../groups/exceptions.py | 2 +- .../groups/models.py | 2 +- .../invitations/settings.py | 7 +- .../src/simcore_service_webserver/log.py | 3 +- .../login/_auth_api.py | 11 +- .../login/_auth_handlers.py | 3 +- .../login/_registration_handlers.py | 4 +- .../simcore_service_webserver/login/_sql.py | 65 +++++--- .../login/handlers_confirmation.py | 6 +- .../simcore_service_webserver/login/plugin.py | 16 +- .../login/storage.py | 8 +- .../login/utils_email.py | 1 + .../long_running_tasks.py | 2 +- .../meta_modeling/_iterations.py | 3 +- .../meta_modeling/_version_control.py | 14 +- .../simcore_service_webserver/products/_db.py | 5 +- .../products/_events.py | 11 +- .../projects/_comments_handlers.py | 2 +- .../projects/_common_models.py | 4 +- .../projects/_crud_handlers.py | 16 +- .../projects/_crud_handlers_models.py | 6 +- .../projects/_db_utils.py | 4 +- .../projects/_metadata_db.py | 8 +- .../projects/_nodes_api.py | 6 +- .../projects/_nodes_handlers.py | 9 +- .../projects/_permalink_api.py | 1 - .../projects/_ports_api.py | 15 +- .../projects/_ports_handlers.py | 5 +- .../projects/_states_handlers.py | 11 +- .../simcore_service_webserver/projects/db.py | 16 +- .../projects/lock.py | 3 +- .../projects/projects_api.py | 5 +- .../resource_manager/registry.py | 2 +- .../_pricing_plans_admin_handlers.py | 4 +- .../resource_usage/_pricing_plans_handlers.py | 4 +- .../resource_usage/_service_runs_handlers.py | 29 ++-- .../socketio/_handlers.py | 4 +- .../statics/_events.py | 5 +- .../simcore_service_webserver/storage/api.py | 4 +- .../studies_dispatcher/_projects.py | 8 +- .../studies_dispatcher/_redirects_handlers.py | 2 +- .../studies_dispatcher/_users.py | 5 +- .../tags/_handlers.py | 2 +- .../users/_handlers.py | 8 +- .../users/_preferences_handlers.py | 4 +- .../simcore_service_webserver/users/api.py | 1 + .../utils_aiohttp.py | 2 +- .../version_control/_handlers.py | 8 +- .../version_control/db.py | 2 +- .../wallets/_groups_handlers.py | 4 +- .../wallets/_payments_handlers.py | 4 +- .../tests/unit/with_dbs/03/test_email.py | 25 ++- 84 files changed, 537 insertions(+), 317 deletions(-) diff --git a/packages/models-library/src/models_library/basic_types.py b/packages/models-library/src/models_library/basic_types.py index 0910d04dc1e..a70709ca802 100644 --- a/packages/models-library/src/models_library/basic_types.py +++ b/packages/models-library/src/models_library/basic_types.py @@ -11,6 +11,7 @@ ) from .basic_regex import ( + PROPERTY_KEY_RE, SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS, SIMPLE_VERSION_RE, UUID_RE, @@ -155,3 +156,7 @@ class BuildTargetEnum(str, Enum): CACHE = "cache" PRODUCTION = "production" DEVELOPMENT = "development" + + +class KeyIDStr(ConstrainedStr): + regex = re.compile(PROPERTY_KEY_RE) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 38994e54b15..318f7149ab4 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -2,7 +2,6 @@ Models Node as a central element in a project's pipeline """ -import re from copy import deepcopy from typing import Any, ClassVar, TypeAlias, Union @@ -18,8 +17,7 @@ validator, ) -from .basic_regex import PROPERTY_KEY_RE -from .basic_types import EnvVarKey, HttpUrlWithCustomMinLength +from .basic_types import EnvVarKey, HttpUrlWithCustomMinLength, KeyIDStr from .projects_access import AccessEnum from .projects_nodes_io import ( DatCoreFileLink, @@ -57,10 +55,6 @@ ] -class KeyIDStr(ConstrainedStr): - regex = re.compile(PROPERTY_KEY_RE) - - InputID: TypeAlias = KeyIDStr OutputID: TypeAlias = KeyIDStr @@ -238,6 +232,7 @@ def convert_from_enum(cls, v): class Config: extra = Extra.forbid + # NOTE: exporting without this trick does not make runHash as nullable. # It is a Pydantic issue see https://github.com/samuelcolvin/pydantic/issues/1270 @staticmethod diff --git a/packages/models-library/src/models_library/projects_nodes_io.py b/packages/models-library/src/models_library/projects_nodes_io.py index 00a91cbca2b..b2d88485489 100644 --- a/packages/models-library/src/models_library/projects_nodes_io.py +++ b/packages/models-library/src/models_library/projects_nodes_io.py @@ -8,9 +8,10 @@ import re from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias +from typing import Any, ClassVar, TypeAlias from uuid import UUID +from models_library.basic_types import KeyIDStr from pydantic import ( AnyUrl, BaseModel, @@ -23,15 +24,11 @@ from .basic_regex import ( DATCORE_FILE_ID_RE, - PROPERTY_KEY_RE, SIMCORE_S3_DIRECTORY_ID_RE, SIMCORE_S3_FILE_ID_RE, UUID_RE, ) -if TYPE_CHECKING: - pass - NodeID = UUID @@ -107,10 +104,9 @@ class PortLink(BaseModel): description="The node to get the port output from", alias="nodeUuid", ) - output: str = Field( + output: KeyIDStr = Field( ..., description="The port key in the node given by nodeUuid", - regex=PROPERTY_KEY_RE, ) class Config: @@ -183,8 +179,7 @@ class SimCoreFileLink(BaseFileLink): dataset: str | None = Field( default=None, - deprecated=True - # TODO: Remove with storage refactoring + deprecated=True, ) @validator("store", always=True) diff --git a/packages/service-library/src/servicelib/aiohttp/requests_validation.py b/packages/service-library/src/servicelib/aiohttp/requests_validation.py index 4353c8fad0d..2fd5d0e41f0 100644 --- a/packages/service-library/src/servicelib/aiohttp/requests_validation.py +++ b/packages/service-library/src/servicelib/aiohttp/requests_validation.py @@ -15,9 +15,9 @@ from aiohttp import web from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, Extra, ValidationError, parse_obj_as -from servicelib.aiohttp import status from ..mimetype_constants import MIMETYPE_APPLICATION_JSON +from . import status ModelClass = TypeVar("ModelClass", bound=BaseModel) ModelOrListOrDictType = TypeVar("ModelOrListOrDictType", bound=BaseModel | list | dict) diff --git a/packages/service-library/src/servicelib/logging_utils.py b/packages/service-library/src/servicelib/logging_utils.py index 3e1476fc59e..a9626f2ba39 100644 --- a/packages/service-library/src/servicelib/logging_utils.py +++ b/packages/service-library/src/servicelib/logging_utils.py @@ -14,7 +14,7 @@ from datetime import datetime from inspect import getframeinfo, stack from pathlib import Path -from typing import Any, TypeAlias, TypedDict +from typing import Any, Iterator, TypeAlias, TypedDict, TypeVar from .utils_secrets import mask_sensitive_data @@ -56,11 +56,11 @@ class CustomFormatter(logging.Formatter): 2. Overrides 'filename' with the value of 'file_name_override', if it exists. """ - def __init__(self, fmt: str, log_format_local_dev_enabled: bool): + def __init__(self, fmt: str, *, log_format_local_dev_enabled: bool) -> None: super().__init__(fmt) self.log_format_local_dev_enabled = log_format_local_dev_enabled - def format(self, record): + def format(self, record) -> str: if hasattr(record, "func_name_override"): record.funcName = record.func_name_override if hasattr(record, "file_name_override"): @@ -86,7 +86,7 @@ def format(self, record): # log_level=%{WORD:log_level} \| log_timestamp=%{TIMESTAMP_ISO8601:log_timestamp} \| log_source=%{DATA:log_source} \| log_msg=%{GREEDYDATA:log_msg} -def config_all_loggers(log_format_local_dev_enabled: bool): +def config_all_loggers(*, log_format_local_dev_enabled: bool) -> None: """ Applies common configuration to ALL registered loggers """ @@ -102,19 +102,26 @@ def config_all_loggers(log_format_local_dev_enabled: bool): fmt = LOCAL_FORMATTING for logger in loggers: - set_logging_handler(logger, fmt, log_format_local_dev_enabled) + set_logging_handler( + logger, fmt=fmt, log_format_local_dev_enabled=log_format_local_dev_enabled + ) def set_logging_handler( logger: logging.Logger, + *, fmt: str, log_format_local_dev_enabled: bool, ) -> None: for handler in logger.handlers: - handler.setFormatter(CustomFormatter(fmt, log_format_local_dev_enabled)) + handler.setFormatter( + CustomFormatter( + fmt, log_format_local_dev_enabled=log_format_local_dev_enabled + ) + ) -def test_logger_propagation(logger: logging.Logger): +def test_logger_propagation(logger: logging.Logger) -> None: """log propagation and levels can sometimes be daunting to get it right. This function uses the `logger`` passed as argument to log the same message at different levels @@ -161,7 +168,9 @@ def _log_arguments( # Before to the function execution, log function details. logger_obj.log( level, - "Arguments: %s - Begin function", + "%s:%s(%s) - Begin function", + func.__module__.split(".")[-1], + func.__name__, formatted_arguments, extra=extra_args, ) @@ -169,67 +178,59 @@ def _log_arguments( return extra_args +def _log_return_value( + logger_obj: logging.Logger, + level: int, + func: Callable, + result: Any, + extra_args: dict[str, str], +) -> None: + logger_obj.log( + level, + "%s:%s returned %r - End function", + func.__module__.split(".")[-1], + func.__name__, + result, + extra=extra_args, + ) + + +F = TypeVar("F", bound=Callable[..., Any]) + + def log_decorator( - logger=None, level: int = logging.DEBUG, *, log_traceback: bool = False -): - # Build logger object - logger_obj = logger or _logger + logger: logging.Logger | None, level: int = logging.DEBUG +) -> Callable[[F], F]: + the_logger = logger or _logger - def log_decorator_info(func): + def decorator(func: F) -> F: if iscoroutinefunction(func): @functools.wraps(func) - async def log_decorator_wrapper(*args, **kwargs): - extra_args = _log_arguments(logger_obj, level, func, *args, **kwargs) - try: - # log return value from the function - value = await func(*args, **kwargs) - logger_obj.log( - level, "Returned: - End function %r", value, extra=extra_args - ) - except: - # log exception if occurs in function - logger_obj.log( - level, - "Exception: %s", - extra=extra_args, - exc_info=log_traceback, - ) - raise - # Return function value - return value - - else: + async def async_wrapper(*args: Any, **kwargs: Any) -> Any: + extra_args = _log_arguments(the_logger, level, func, *args, **kwargs) + with log_catch(the_logger, reraise=True): + result = await func(*args, **kwargs) + _log_return_value(the_logger, level, func, result, extra_args) + return result - @functools.wraps(func) - def log_decorator_wrapper(*args, **kwargs): - extra_args = _log_arguments(logger_obj, level, func, *args, **kwargs) - try: - # log return value from the function - value = func(*args, **kwargs) - logger_obj.log( - level, "Returned: - End function %r", value, extra=extra_args - ) - except: - # log exception if occurs in function - logger_obj.log( - level, - "Exception: %s", - extra=extra_args, - exc_info=log_traceback, - ) - raise - # Return function value - return value - - # Return the pointer to the function - return log_decorator_wrapper - - return log_decorator_info + return async_wrapper # type: ignore[return-value] # decorators typing is hard + + @functools.wraps(func) + def sync_wrapper(*args: Any, **kwargs: Any) -> Any: + extra_args = _log_arguments(the_logger, level, func, *args, **kwargs) + with log_catch(the_logger, reraise=True): + result = func(*args, **kwargs) + _log_return_value(the_logger, level, func, result, extra_args) + return result + + return sync_wrapper # type: ignore[return-value] # decorators typing is hard + + return decorator @contextmanager -def log_catch(logger: logging.Logger, reraise: bool = True): +def log_catch(logger: logging.Logger, *, reraise: bool = True) -> Iterator[None]: try: yield except asyncio.CancelledError: @@ -257,7 +258,7 @@ def get_log_record_extra(*, user_id: int | str | None = None) -> LogExtra | None return extra or None -def _un_capitalize(s): +def _un_capitalize(s: str) -> str: return s[:1].lower() + s[1:] if s else "" @@ -277,15 +278,16 @@ def log_context( kwargs: dict[str, Any] = {} if extra: kwargs["extra"] = extra - - logger.log(level, "Starting " + msg + " ...", *args, **kwargs) + log_msg = f"Starting {msg} ..." + logger.log(level, log_msg, *args, **kwargs) yield duration = ( f" in {(datetime.now() - start ).total_seconds()}s" # noqa: DTZ005 if log_duration else "" ) - logger.log(level, "Finished " + msg + duration, *args, **kwargs) + log_msg = f"Finished {msg}{duration}" + logger.log(level, log_msg, *args, **kwargs) def guess_message_log_level(message: str) -> LogLevelInt: diff --git a/packages/service-library/tests/test_logging_utils.py b/packages/service-library/tests/test_logging_utils.py index c32031dbeb0..a641442a58e 100644 --- a/packages/service-library/tests/test_logging_utils.py +++ b/packages/service-library/tests/test_logging_utils.py @@ -1,62 +1,164 @@ # pylint:disable=redefined-outer-name import logging -from threading import Thread from typing import Any import pytest +from faker import Faker from servicelib.logging_utils import ( + LogExtra, LogLevelInt, LogMessageStr, guess_message_log_level, log_context, log_decorator, ) -from servicelib.utils import logged_gather _logger = logging.getLogger(__name__) @pytest.mark.parametrize("logger", [None, _logger]) -@pytest.mark.parametrize("log_traceback", [True, False]) async def test_error_regression_async_def( - caplog: pytest.LogCaptureFixture, logger: logging.Logger | None, log_traceback: bool + caplog: pytest.LogCaptureFixture, logger: logging.Logger | None, faker: Faker ): - @log_decorator(logger, logging.ERROR, log_traceback=log_traceback) - async def _raising_error() -> None: + # NOTE: change the log level so that the log is visible + caplog.set_level(logging.INFO) + + @log_decorator(logger, logging.INFO) + async def _not_raising_fct( + argument1: int, argument2: str, *, keyword_arg1: bool, keyword_arg2: str + ) -> int: + assert argument1 is not None + assert argument2 is not None + assert keyword_arg1 is not None + assert keyword_arg2 is not None + return 0 + + @log_decorator(logger, logging.INFO) + async def _raising_error( + argument1: int, argument2: str, *, keyword_arg1: bool, keyword_arg2: str + ) -> None: + assert argument1 is not None + assert argument2 is not None + assert keyword_arg1 is not None + assert keyword_arg2 is not None msg = "Raising as expected" raise RuntimeError(msg) caplog.clear() + argument1 = faker.pyint() + argument2 = faker.pystr() + key_argument1 = faker.pybool() + key_argument2 = faker.pystr() + + result = await _not_raising_fct( + argument1, argument2, keyword_arg1=key_argument1, keyword_arg2=key_argument2 + ) + assert result == 0 + assert len(caplog.records) == 2 + info_record = caplog.records[0] + assert info_record.levelno == logging.INFO + assert ( + f"{_not_raising_fct.__module__.split('.')[-1]}:{_not_raising_fct.__name__}({argument1!r}, {argument2!r}, keyword_arg1={key_argument1!r}, keyword_arg2={key_argument2!r})" + in info_record.message + ) + return_record = caplog.records[1] + assert return_record.levelno == logging.INFO + assert not return_record.exc_text + assert ( + f"{_not_raising_fct.__module__.split('.')[-1]}:{_not_raising_fct.__name__} returned {result!r}" + in return_record.message + ) - await logged_gather(_raising_error(), reraise=False) - - if log_traceback: - assert "Traceback" in caplog.text - else: - assert "Traceback" not in caplog.text + caplog.clear() + with pytest.raises(RuntimeError): + await _raising_error( + argument1, argument2, keyword_arg1=key_argument1, keyword_arg2=key_argument2 + ) + + assert len(caplog.records) == 2 + info_record = caplog.records[0] + assert info_record.levelno == logging.INFO + assert ( + f"{_raising_error.__module__.split('.')[-1]}:{_raising_error.__name__}({argument1!r}, {argument2!r}, keyword_arg1={key_argument1!r}, keyword_arg2={key_argument2!r})" + in info_record.message + ) + error_record = caplog.records[1] + assert error_record.levelno == logging.ERROR + assert error_record.exc_text + assert "Traceback" in error_record.exc_text @pytest.mark.parametrize("logger", [None, _logger]) -@pytest.mark.parametrize("log_traceback", [True, False]) -async def test_error_regression_def( - caplog: pytest.LogCaptureFixture, logger: logging.Logger | None, log_traceback: bool +def test_error_regression_sync_def( + caplog: pytest.LogCaptureFixture, logger: logging.Logger | None, faker: Faker ): - @log_decorator(logger, logging.ERROR, log_traceback=log_traceback) - def _raising_error() -> None: + # NOTE: change the log level so that the log is visible + caplog.set_level(logging.INFO) + + @log_decorator(logger, logging.INFO) + def _not_raising_fct( + argument1: int, argument2: str, *, keyword_arg1: bool, keyword_arg2: str + ) -> int: + assert argument1 is not None + assert argument2 is not None + assert keyword_arg1 is not None + assert keyword_arg2 is not None + return 0 + + @log_decorator(logger, logging.INFO) + def _raising_error( + argument1: int, argument2: str, *, keyword_arg1: bool, keyword_arg2: str + ) -> None: + assert argument1 is not None + assert argument2 is not None + assert keyword_arg1 is not None + assert keyword_arg2 is not None msg = "Raising as expected" raise RuntimeError(msg) caplog.clear() + argument1 = faker.pyint() + argument2 = faker.pystr() + key_argument1 = faker.pybool() + key_argument2 = faker.pystr() + + result = _not_raising_fct( + argument1, argument2, keyword_arg1=key_argument1, keyword_arg2=key_argument2 + ) + assert result == 0 + assert len(caplog.records) == 2 + info_record = caplog.records[0] + assert info_record.levelno == logging.INFO + assert ( + f"{_not_raising_fct.__module__.split('.')[-1]}:{_not_raising_fct.__name__}({argument1!r}, {argument2!r}, keyword_arg1={key_argument1!r}, keyword_arg2={key_argument2!r})" + in info_record.message + ) + return_record = caplog.records[1] + assert return_record.levelno == logging.INFO + assert not return_record.exc_text + assert ( + f"{_not_raising_fct.__module__.split('.')[-1]}:{_not_raising_fct.__name__} returned {result!r}" + in return_record.message + ) - thread = Thread(target=_raising_error) - thread.start() - thread.join() - - if log_traceback: - assert "Traceback" in caplog.text - else: - assert "Traceback" not in caplog.text + caplog.clear() + with pytest.raises(RuntimeError): + _raising_error( + argument1, argument2, keyword_arg1=key_argument1, keyword_arg2=key_argument2 + ) + + assert len(caplog.records) == 2 + info_record = caplog.records[0] + assert info_record.levelno == logging.INFO + assert ( + f"{_raising_error.__module__.split('.')[-1]}:{_raising_error.__name__}({argument1!r}, {argument2!r}, keyword_arg1={key_argument1!r}, keyword_arg2={key_argument2!r})" + in info_record.message + ) + error_record = caplog.records[1] + assert error_record.levelno == logging.ERROR + assert error_record.exc_text + assert "Traceback" in error_record.exc_text @pytest.mark.parametrize( @@ -113,7 +215,7 @@ def test_log_context( caplog: pytest.LogCaptureFixture, msg: str, args: tuple[Any, ...], - extra: dict[str, Any] | None, + extra: LogExtra | None, ): caplog.clear() diff --git a/services/agent/src/simcore_service_agent/core/application.py b/services/agent/src/simcore_service_agent/core/application.py index ec2530c5ab2..1c2211b16f3 100644 --- a/services/agent/src/simcore_service_agent/core/application.py +++ b/services/agent/src/simcore_service_agent/core/application.py @@ -29,7 +29,9 @@ def _setup_logger(settings: ApplicationSettings): # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 logging.basicConfig(level=settings.LOGLEVEL.value) # NOSONAR logging.root.setLevel(settings.LOGLEVEL.value) - config_all_loggers(settings.AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED) + config_all_loggers( + log_format_local_dev_enabled=settings.AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED + ) def create_app() -> FastAPI: diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index 1adbb9043cf..eb05ea1d50f 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -42,7 +42,9 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI: logging.basicConfig(level=settings.log_level) logging.root.setLevel(settings.log_level) - config_all_loggers(settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED) + config_all_loggers( + log_format_local_dev_enabled=settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED + ) _logger.debug("App settings:\n%s", settings.json(indent=2)) # Labeling diff --git a/services/api-server/src/simcore_service_api_server/services/solver_job_models_converters.py b/services/api-server/src/simcore_service_api_server/services/solver_job_models_converters.py index 2ca4aab74d6..137463e1263 100644 --- a/services/api-server/src/simcore_service_api_server/services/solver_job_models_converters.py +++ b/services/api-server/src/simcore_service_api_server/services/solver_job_models_converters.py @@ -2,6 +2,7 @@ Helper functions to convert models used in services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py """ + import urllib.parse import uuid from collections.abc import Callable @@ -10,7 +11,8 @@ import arrow from models_library.api_schemas_webserver.projects import ProjectCreateNew, ProjectGet -from models_library.projects_nodes import InputID, KeyIDStr +from models_library.basic_types import KeyIDStr +from models_library.projects_nodes import InputID from pydantic import parse_obj_as from ..models.basic_types import VersionStr diff --git a/services/autoscaling/src/simcore_service_autoscaling/main.py b/services/autoscaling/src/simcore_service_autoscaling/main.py index 3c0bc9b3722..90272bcb12f 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/main.py +++ b/services/autoscaling/src/simcore_service_autoscaling/main.py @@ -1,6 +1,7 @@ """Main application to be deployed by uvicorn (or equivalent) server """ + import logging from fastapi import FastAPI @@ -11,7 +12,9 @@ the_settings = ApplicationSettings.create_from_envs() logging.basicConfig(level=the_settings.log_level) logging.root.setLevel(the_settings.log_level) -config_all_loggers(the_settings.AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED) +config_all_loggers( + log_format_local_dev_enabled=the_settings.AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED +) # SINGLETON FastAPI app the_app: FastAPI = create_app(the_settings) diff --git a/services/catalog/src/simcore_service_catalog/main.py b/services/catalog/src/simcore_service_catalog/main.py index 22f1eefd78f..f5c9f4ee97c 100644 --- a/services/catalog/src/simcore_service_catalog/main.py +++ b/services/catalog/src/simcore_service_catalog/main.py @@ -1,5 +1,6 @@ """Main application to be deployed in for example uvicorn. """ + import logging from fastapi import FastAPI @@ -12,7 +13,9 @@ # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 logging.basicConfig(level=_the_settings.CATALOG_LOG_LEVEL.value) # NOSONAR logging.root.setLevel(_the_settings.CATALOG_LOG_LEVEL.value) -config_all_loggers(_the_settings.CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED) +config_all_loggers( + log_format_local_dev_enabled=_the_settings.CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED +) # SINGLETON FastAPI app diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py index 0264fbbd307..5e55931c36c 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py @@ -1,6 +1,7 @@ """Main application to be deployed by uvicorn (or equivalent) server """ + import logging from fastapi import FastAPI @@ -11,7 +12,9 @@ the_settings = ApplicationSettings.create_from_envs() logging.basicConfig(level=the_settings.log_level) logging.root.setLevel(the_settings.log_level) -config_all_loggers(the_settings.CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED) +config_all_loggers( + log_format_local_dev_enabled=the_settings.CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED +) # SINGLETON FastAPI app the_app: FastAPI = create_app(the_settings) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py index 78a52589bf4..79dfd08cbdb 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py @@ -62,7 +62,9 @@ async def dask_setup(worker: distributed.Worker) -> None: # removing them solves dual propagation of logs for handler in logging.getLogger("distributed").handlers: logging.getLogger("distributed").removeHandler(handler) - config_all_loggers(settings.DASK_LOG_FORMAT_LOCAL_DEV_ENABLED) + config_all_loggers( + log_format_local_dev_enabled=settings.DASK_LOG_FORMAT_LOCAL_DEV_ENABLED + ) logger.info("Setting up worker...") logger.info("Settings: %s", pformat(settings.dict())) diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py index f3b7b82c5c2..2b57a46ed14 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py @@ -38,7 +38,9 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: logging.basicConfig(level=settings.LOG_LEVEL.value) logging.root.setLevel(settings.LOG_LEVEL.value) - config_all_loggers(settings.DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED) + config_all_loggers( + log_format_local_dev_enabled=settings.DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED + ) # keep mostly quiet noisy loggers quiet_level: int = max( diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py index 3afbac29118..a2a99f4bea3 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py @@ -1,6 +1,6 @@ import asyncio import logging -from typing import Annotated, Final, cast +from typing import Annotated, Final import httpx from fastapi import APIRouter, Depends, Header, HTTPException, Request @@ -92,10 +92,7 @@ async def list_tracked_dynamic_services( # NOTE: Review error handling https://github.com/ITISFoundation/osparc-simcore/issues/3194 dynamic_sidecar_running_services = await asyncio.gather(*get_stack_statuse_tasks) - return cast( - list[DynamicServiceGet], - legacy_running_services + dynamic_sidecar_running_services, - ) # mypy + return legacy_running_services + dynamic_sidecar_running_services @router.post( diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 52da0cbb528..a43848298a7 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -113,7 +113,9 @@ def create_base_app(settings: AppSettings | None = None) -> FastAPI: logging.basicConfig(level=settings.LOG_LEVEL.value) logging.root.setLevel(settings.LOG_LEVEL.value) - config_all_loggers(settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED) + config_all_loggers( + log_format_local_dev_enabled=settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED + ) _logger.debug(settings.json(indent=2)) # keep mostly quiet noisy loggers diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py index 3ce21777b79..9a76eff74c4 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py @@ -1,6 +1,7 @@ """Main application to be deployed by uvicorn (or equivalent) server """ + import logging from fastapi import FastAPI @@ -13,7 +14,9 @@ # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 logging.basicConfig(level=_the_settings.log_level) # NOSONAR logging.root.setLevel(_the_settings.log_level) -config_all_loggers(_the_settings.DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED) +config_all_loggers( + log_format_local_dev_enabled=_the_settings.DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED +) # SINGLETON FastAPI app the_app: FastAPI = create_app(_the_settings) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index f7cc2304126..f5910ffbffe 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -113,7 +113,9 @@ def setup_logger(settings: ApplicationSettings): # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 logging.basicConfig(level=settings.log_level) logging.root.setLevel(settings.log_level) - config_all_loggers(settings.DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED) + config_all_loggers( + log_format_local_dev_enabled=settings.DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED + ) def create_base_app() -> FastAPI: diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/main.py b/services/efs-guardian/src/simcore_service_efs_guardian/main.py index 6ab24933b02..dd107472181 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/main.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/main.py @@ -1,6 +1,7 @@ """Main application to be deployed by uvicorn (or equivalent) server """ + import logging from fastapi import FastAPI @@ -11,7 +12,9 @@ the_settings = ApplicationSettings.create_from_envs() logging.basicConfig(level=the_settings.log_level) logging.root.setLevel(the_settings.log_level) -config_all_loggers(the_settings.EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED) +config_all_loggers( + log_format_local_dev_enabled=the_settings.EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED +) # SINGLETON FastAPI app the_app: FastAPI = create_app(the_settings) diff --git a/services/invitations/src/simcore_service_invitations/main.py b/services/invitations/src/simcore_service_invitations/main.py index 5857f366ebd..bddf5d2d3eb 100644 --- a/services/invitations/src/simcore_service_invitations/main.py +++ b/services/invitations/src/simcore_service_invitations/main.py @@ -1,6 +1,7 @@ """Main application to be deployed by uvicorn (or equivalent) server """ + import logging from fastapi import FastAPI @@ -13,7 +14,9 @@ # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 logging.basicConfig(level=the_settings.log_level) # NOSONAR logging.root.setLevel(the_settings.log_level) -config_all_loggers(the_settings.INVITATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED) +config_all_loggers( + log_format_local_dev_enabled=the_settings.INVITATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED +) # SINGLETON FastAPI app the_app: FastAPI = create_app(the_settings) diff --git a/services/payments/src/simcore_service_payments/main.py b/services/payments/src/simcore_service_payments/main.py index eb5596ce454..a39832f283d 100644 --- a/services/payments/src/simcore_service_payments/main.py +++ b/services/payments/src/simcore_service_payments/main.py @@ -1,6 +1,7 @@ """Main application to be deployed by uvicorn (or equivalent) server """ + import logging from fastapi import FastAPI @@ -13,7 +14,9 @@ # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 logging.basicConfig(level=_the_settings.log_level) # NOSONAR logging.root.setLevel(_the_settings.log_level) -config_all_loggers(_the_settings.PAYMENTS_LOG_FORMAT_LOCAL_DEV_ENABLED) +config_all_loggers( + log_format_local_dev_enabled=_the_settings.PAYMENTS_LOG_FORMAT_LOCAL_DEV_ENABLED +) # SINGLETON FastAPI app the_app: FastAPI = create_app(_the_settings) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py index 9b14df05532..b0f4e3a7fe2 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py @@ -1,6 +1,7 @@ """Main application to be deployed by uvicorn (or equivalent) server """ + import logging from fastapi import FastAPI @@ -13,7 +14,9 @@ # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 logging.basicConfig(level=the_settings.log_level) # NOSONAR logging.root.setLevel(the_settings.log_level) -config_all_loggers(the_settings.RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED) +config_all_loggers( + log_format_local_dev_enabled=the_settings.RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED +) # SINGLETON FastAPI app the_app: FastAPI = create_app(the_settings) diff --git a/services/storage/src/simcore_service_storage/cli.py b/services/storage/src/simcore_service_storage/cli.py index a7a004d9225..6a9e209b0da 100644 --- a/services/storage/src/simcore_service_storage/cli.py +++ b/services/storage/src/simcore_service_storage/cli.py @@ -25,7 +25,9 @@ def run(): logging.basicConfig(level=settings_obj.log_level) logging.root.setLevel(settings_obj.log_level) - config_all_loggers(settings_obj.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED) + config_all_loggers( + log_format_local_dev_enabled=settings_obj.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED + ) # keep mostly quiet noisy loggers quiet_level: int = max( diff --git a/services/web/server/src/simcore_service_webserver/catalog/_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/_handlers.py index 619c235be7b..639e2501228 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_handlers.py @@ -4,6 +4,7 @@ should live in the catalog service in his final version """ + import asyncio import logging import urllib.parse @@ -82,7 +83,9 @@ class ListServiceParams(PageQueryParameters): @_handlers_errors.reraise_catalog_exceptions_as_http_errors async def dev_list_services_latest(request: Request): request_ctx = CatalogRequestContext.create(request) - query_params = parse_request_query_parameters_as(ListServiceParams, request) + query_params: ListServiceParams = parse_request_query_parameters_as( + ListServiceParams, request + ) page_items, page_meta = await _api.dev_list_latest_services( request.app, @@ -298,7 +301,9 @@ class _FromServiceOutputParams(BaseModel): async def get_compatible_inputs_given_source_output(request: Request): ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) - query_params = parse_request_query_parameters_as(_FromServiceOutputParams, request) + query_params: _FromServiceOutputParams = parse_request_query_parameters_as( + _FromServiceOutputParams, request + ) # Evaluate and return validated model data = await _api.get_compatible_inputs_given_source_output( @@ -384,7 +389,9 @@ async def get_compatible_outputs_given_target_input(request: Request): """ ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) - query_params = parse_request_query_parameters_as(_ToServiceInputsParams, request) + query_params: _ToServiceInputsParams = parse_request_query_parameters_as( + _ToServiceInputsParams, request + ) data = await _api.get_compatible_outputs_given_target_input( path_params.service_key, diff --git a/services/web/server/src/simcore_service_webserver/catalog/_models.py b/services/web/server/src/simcore_service_webserver/catalog/_models.py index 5c4235350a6..a2872e57f42 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_models.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_models.py @@ -1,6 +1,7 @@ import logging +from collections.abc import Callable from dataclasses import dataclass -from typing import Any, Callable, Final +from typing import Any, Final from aiocache import cached from models_library.api_schemas_webserver.catalog import ( @@ -11,6 +12,7 @@ ) from models_library.services import BaseServiceIOModel from pint import PintError, UnitRegistry +from pint.quantity import Quantity _logger = logging.getLogger(__name__) @@ -44,7 +46,7 @@ def get_html_formatted_unit( if unit_name is None: return None - q = ureg.Quantity(unit_name) + q: Quantity = ureg.Quantity(unit_name) return UnitHtmlFormat(short=f"{q.units:~H}", long=f"{q.units:H}") except (PintError, NotImplementedError): return None diff --git a/services/web/server/src/simcore_service_webserver/catalog/exceptions.py b/services/web/server/src/simcore_service_webserver/catalog/exceptions.py index abe21493033..11f3794661b 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/exceptions.py +++ b/services/web/server/src/simcore_service_webserver/catalog/exceptions.py @@ -1,6 +1,5 @@ """Defines the different exceptions that may arise in the catalog subpackage""" - from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( CatalogForbiddenError, CatalogItemNotFoundError, @@ -31,6 +30,7 @@ def __init__(self, *, service_key: str, service_version: str, **ctxs): self.service_version = service_version +# mypy: disable-error-code=truthy-function assert CatalogForbiddenError # nosec assert CatalogItemNotFoundError # nosec diff --git a/services/web/server/src/simcore_service_webserver/catalog/plugin.py b/services/web/server/src/simcore_service_webserver/catalog/plugin.py index 47700bbe70f..5f4de728fb6 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/plugin.py +++ b/services/web/server/src/simcore_service_webserver/catalog/plugin.py @@ -1,6 +1,7 @@ """ Subsystem to communicate with catalog service """ + import logging from aiohttp import web @@ -22,7 +23,7 @@ def setup_catalog(app: web.Application): # ensures routes are names that corresponds to function names assert all( # nosec - route_def.kwargs["name"] == route_def.handler.__name__ + route_def.kwargs["name"] == route_def.handler.__name__ # type: ignore[attr-defined] # route_def is a RouteDef not an Abstract for route_def in _handlers.routes ) diff --git a/services/web/server/src/simcore_service_webserver/clusters/_handlers.py b/services/web/server/src/simcore_service_webserver/clusters/_handlers.py index 25c84d2ac6c..64993021665 100644 --- a/services/web/server/src/simcore_service_webserver/clusters/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/clusters/_handlers.py @@ -63,7 +63,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _RequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore + user_id: UserID = Field(..., alias=RQT_USERID_KEY) # diff --git a/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py b/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py index af995313cce..13154bf5723 100644 --- a/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py @@ -1,6 +1,7 @@ """ Handler functions and routing for diagnostics """ + import asyncio import logging from contextlib import suppress @@ -53,7 +54,7 @@ async def get_app_diagnostics(request: web.Request): } # allocated memory - query_params = parse_request_query_parameters_as( + query_params: StatusDiagnosticsQueryParam = parse_request_query_parameters_as( StatusDiagnosticsQueryParam, request ) if query_params.top_tracemalloc is not None: diff --git a/services/web/server/src/simcore_service_webserver/email/_core.py b/services/web/server/src/simcore_service_webserver/email/_core.py index e48bd0f8b2a..0c2329cac54 100644 --- a/services/web/server/src/simcore_service_webserver/email/_core.py +++ b/services/web/server/src/simcore_service_webserver/email/_core.py @@ -41,38 +41,39 @@ async def _do_send_mail( if settings.SMTP_PORT == 587: # NOTE: aiosmtplib does not handle port 587 correctly this is a workaround try: - smtp = _create_smtp_client(settings) + smtp_on_587_port = _create_smtp_client(settings) if settings.SMTP_PROTOCOL == EmailProtocol.STARTTLS: _logger.info("Unencrypted connection attempt to mailserver ...") - await smtp.connect(use_tls=False, port=settings.SMTP_PORT) + await smtp_on_587_port.connect(use_tls=False, port=settings.SMTP_PORT) _logger.info("Starting STARTTLS ...") - await smtp.starttls() + await smtp_on_587_port.starttls() elif settings.SMTP_PROTOCOL == EmailProtocol.TLS: - await smtp.connect(use_tls=True, port=settings.SMTP_PORT) + await smtp_on_587_port.connect(use_tls=True, port=settings.SMTP_PORT) elif settings.SMTP_PROTOCOL == EmailProtocol.UNENCRYPTED: _logger.info("Unencrypted connection attempt to mailserver ...") - await smtp.connect(use_tls=False, port=settings.SMTP_PORT) + await smtp_on_587_port.connect(use_tls=False, port=settings.SMTP_PORT) if settings.SMTP_USERNAME and settings.SMTP_PASSWORD: _logger.info("Attempting a login into the email server ...") - await smtp.login( + await smtp_on_587_port.login( settings.SMTP_USERNAME, settings.SMTP_PASSWORD.get_secret_value() ) - await smtp.send_message(message) + await smtp_on_587_port.send_message(message) finally: - await smtp.quit() + await smtp_on_587_port.quit() else: - async with _create_smtp_client(settings) as smtp: + smtp_client = _create_smtp_client(settings) + async with smtp_client: if settings.SMTP_USERNAME and settings.SMTP_PASSWORD: _logger.info("Login email server ...") - await smtp.login( + await smtp_client.login( settings.SMTP_USERNAME, settings.SMTP_PASSWORD.get_secret_value() ) - await smtp.send_message(message) + await smtp_client.send_message(message) MIMEMessage = Union[MIMEText, MIMEMultipart] @@ -107,6 +108,9 @@ class SMTPServerInfo(TypedDict): async def check_email_server_responsiveness(settings: SMTPSettings) -> SMTPServerInfo: """Raises SMTPException if cannot connect otherwise settings""" async with _create_smtp_client(settings) as smtp: + assert smtp.hostname # nosec + assert smtp.port # nosec + assert smtp.timeout # nosec return SMTPServerInfo( hostname=smtp.hostname, port=smtp.port, diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py index 2601c6c65bf..8d42ec6dde3 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py @@ -129,7 +129,7 @@ async def create_folder(request: web.Request): @handle_folders_exceptions async def list_folders(request: web.Request): req_ctx = FoldersRequestContext.parse_obj(request) - query_params = parse_request_query_parameters_as( + query_params: FolderListWithJsonStrQueryParams = parse_request_query_parameters_as( FolderListWithJsonStrQueryParams, request ) diff --git a/services/web/server/src/simcore_service_webserver/groups/_handlers.py b/services/web/server/src/simcore_service_webserver/groups/_handlers.py index b91d4749945..dcf69347056 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/groups/_handlers.py @@ -273,7 +273,9 @@ class _ClassifiersQuery(BaseModel): async def get_group_classifiers(request: web.Request): try: path_params = parse_request_path_parameters_as(_GroupsParams, request) - query_params = parse_request_query_parameters_as(_ClassifiersQuery, request) + query_params: _ClassifiersQuery = parse_request_query_parameters_as( + _ClassifiersQuery, request + ) repo = GroupClassifierRepository(request.app) if not await repo.group_uses_scicrunch(path_params.gid): diff --git a/services/web/server/src/simcore_service_webserver/groups/exceptions.py b/services/web/server/src/simcore_service_webserver/groups/exceptions.py index 549fcf03f16..7556a8cae85 100644 --- a/services/web/server/src/simcore_service_webserver/groups/exceptions.py +++ b/services/web/server/src/simcore_service_webserver/groups/exceptions.py @@ -6,7 +6,7 @@ class GroupsError(WebServerBaseError): msg_template = "{msg}" - def __init__(self, msg: str = None): + def __init__(self, msg: str | None = None): super().__init__(msg=msg or "Unexpected error occured in projects subpackage") diff --git a/services/web/server/src/simcore_service_webserver/groups/models.py b/services/web/server/src/simcore_service_webserver/groups/models.py index 406477cf565..bac7f2987bd 100644 --- a/services/web/server/src/simcore_service_webserver/groups/models.py +++ b/services/web/server/src/simcore_service_webserver/groups/models.py @@ -1,6 +1,6 @@ +# mypy: disable-error-code=truthy-function from ._utils import convert_groups_db_to_schema -# Avoids pycln to remove import assert convert_groups_db_to_schema # nosec __all__: tuple[str, ...] = ("convert_groups_db_to_schema",) diff --git a/services/web/server/src/simcore_service_webserver/invitations/settings.py b/services/web/server/src/simcore_service_webserver/invitations/settings.py index c36cb119a08..025f89955ff 100644 --- a/services/web/server/src/simcore_service_webserver/invitations/settings.py +++ b/services/web/server/src/simcore_service_webserver/invitations/settings.py @@ -5,9 +5,10 @@ """ from functools import cached_property +from typing import Final from aiohttp import web -from pydantic import Field, SecretStr +from pydantic import Field, SecretStr, parse_obj_as from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -18,11 +19,13 @@ from .._constants import APP_SETTINGS_KEY +_INVITATION_VTAG_V1: Final[VersionTag] = parse_obj_as(VersionTag, "v1") + class InvitationsSettings(BaseCustomSettings, MixinServiceSettings): INVITATIONS_HOST: str = "invitations" INVITATIONS_PORT: PortInt = DEFAULT_FASTAPI_PORT - INVITATIONS_VTAG: VersionTag = "v1" + INVITATIONS_VTAG: VersionTag = _INVITATION_VTAG_V1 INVITATIONS_USERNAME: str = Field( ..., diff --git a/services/web/server/src/simcore_service_webserver/log.py b/services/web/server/src/simcore_service_webserver/log.py index 57301f3a6bc..dabc4ead386 100644 --- a/services/web/server/src/simcore_service_webserver/log.py +++ b/services/web/server/src/simcore_service_webserver/log.py @@ -1,6 +1,7 @@ """ Configuration and utilities for service logging """ + import logging from aiodebug import log_slow_callbacks @@ -31,7 +32,7 @@ def setup_logging( # root logging.root.setLevel(level) - config_all_loggers(log_format_local_dev_enabled) + config_all_loggers(log_format_local_dev_enabled=log_format_local_dev_enabled) # Enforces same log-level to aiohttp & gunicorn access loggers # diff --git a/services/web/server/src/simcore_service_webserver/login/_auth_api.py b/services/web/server/src/simcore_service_webserver/login/_auth_api.py index 8d6f5371c90..a5de2c1abc5 100644 --- a/services/web/server/src/simcore_service_webserver/login/_auth_api.py +++ b/services/web/server/src/simcore_service_webserver/login/_auth_api.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Any from aiohttp import web from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON @@ -14,10 +15,10 @@ from .utils import validate_user_status -async def get_user_by_email(app: web.Application, *, email: str) -> dict: +async def get_user_by_email(app: web.Application, *, email: str) -> dict[str, Any]: db: AsyncpgStorage = get_plugin_storage(app) - user: dict = await db.get_user({"email": email}) - return user + user = await db.get_user({"email": email}) + return dict(user) if user else {} async def create_user( @@ -27,7 +28,7 @@ async def create_user( password: str, status_upon_creation: UserStatus, expires_at: datetime | None, -) -> dict: +) -> dict[str, Any]: async with get_database_engine(app).acquire() as conn: user = await UsersRepo.new_user( @@ -44,7 +45,7 @@ async def create_user( async def check_authorized_user_credentials_or_raise( - user: dict, + user: dict[str, Any], password: str, product: Product, ) -> dict: diff --git a/services/web/server/src/simcore_service_webserver/login/_auth_handlers.py b/services/web/server/src/simcore_service_webserver/login/_auth_handlers.py index 0f97e254c9a..a5a69504e99 100644 --- a/services/web/server/src/simcore_service_webserver/login/_auth_handlers.py +++ b/services/web/server/src/simcore_service_webserver/login/_auth_handlers.py @@ -262,6 +262,7 @@ async def login_2fa(request: web.Request): ) user = await db.get_user({"email": login_2fa_.email}) + assert user is not None # nosec # NOTE: a priviledge user should not have called this entrypoint assert UserRole(user["role"]) <= UserRole.USER # nosec @@ -269,7 +270,7 @@ async def login_2fa(request: web.Request): # dispose since code was used await delete_2fa_code(request.app, login_2fa_.email) - return await login_granted_response(request, user=user) + return await login_granted_response(request, user=dict(user)) class LogoutBody(InputSchema): diff --git a/services/web/server/src/simcore_service_webserver/login/_registration_handlers.py b/services/web/server/src/simcore_service_webserver/login/_registration_handlers.py index 6f5a92380e4..318dcfb34f4 100644 --- a/services/web/server/src/simcore_service_webserver/login/_registration_handlers.py +++ b/services/web/server/src/simcore_service_webserver/login/_registration_handlers.py @@ -89,8 +89,8 @@ async def request_product_account(request: web.Request): class _AuthenticatedContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] - product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] + user_id: UserID = Field(..., alias=RQT_USERID_KEY) + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) @routes.post(f"/{API_VTAG}/auth/unregister", name="unregister_account") diff --git a/services/web/server/src/simcore_service_webserver/login/_sql.py b/services/web/server/src/simcore_service_webserver/login/_sql.py index deaeb6d3455..48aa8fc6a90 100644 --- a/services/web/server/src/simcore_service_webserver/login/_sql.py +++ b/services/web/server/src/simcore_service_webserver/login/_sql.py @@ -1,9 +1,22 @@ -def find_one(conn, table, filter_, fields=None): +from collections.abc import Coroutine, Iterable +from typing import Any + +import asyncpg + + +def find_one( + conn: asyncpg.pool.PoolConnectionProxy, + table: str, + filter_: dict[str, Any], + fields=None, +) -> Coroutine[Any, Any, asyncpg.Record | None]: sql, values = find_one_sql(table, filter_, fields) return conn.fetchrow(sql, *values) -def find_one_sql(table, filter_, fields=None): +def find_one_sql( + table: str, filter_: dict[str, Any], fields: Iterable[str] | None = None +) -> tuple[str, list[Any]]: """ >>> find_one_sql('tbl', {'foo': 10, 'bar': 'baz'}) ('SELECT * FROM tbl WHERE bar=$1 AND foo=$2', ['baz', 10]) @@ -13,16 +26,23 @@ def find_one_sql(table, filter_, fields=None): keys, values = _split_dict(filter_) fields = ", ".join(fields) if fields else "*" where = _pairs(keys) - sql = f"SELECT {fields} FROM {table} WHERE {where}" # nosec + sql = f"SELECT {fields} FROM {table} WHERE {where}" # noqa: S608 # @pcrespov SQL injection issue here return sql, values -def insert(conn, table, data, returning="id"): +def insert( + conn: asyncpg.pool.PoolConnectionProxy, + table: str, + data: dict[str, Any], + returning: str = "id", +) -> Coroutine[Any, Any, Any]: sql, values = insert_sql(table, data, returning) return conn.fetchval(sql, *values) -def insert_sql(table, data, returning="id"): +def insert_sql( + table: str, data: dict[str, Any], returning: str = "id" +) -> tuple[str, list[Any]]: """ >>> insert_sql('tbl', {'foo': 'bar', 'id': 1}) ('INSERT INTO tbl (foo, id) VALUES ($1, $2) RETURNING id', ['bar', 1]) @@ -34,7 +54,7 @@ def insert_sql(table, data, returning="id"): ('INSERT INTO tbl (foo, id) VALUES ($1, $2) RETURNING pk', ['bar', 1]) """ keys, values = _split_dict(data) - sql = "INSERT INTO {} ({}) VALUES ({}){}".format( # nosec + sql = "INSERT INTO {} ({}) VALUES ({}){}".format( # noqa: S608 # @pcrespov SQL injection issue here table, ", ".join(keys), ", ".join(_placeholders(data)), @@ -43,12 +63,19 @@ def insert_sql(table, data, returning="id"): return sql, values -def update(conn, table, filter_, updates): +def update( + conn: asyncpg.pool.PoolConnectionProxy, + table: str, + filter_: dict[str, Any], + updates: dict[str, Any], +) -> Coroutine[Any, Any, str]: sql, values = update_sql(table, filter_, updates) return conn.execute(sql, *values) -def update_sql(table, filter_, updates): +def update_sql( + table: str, filter_: dict[str, Any], updates: dict[str, Any] +) -> tuple[str, list[Any]]: """ >>> update_sql('tbl', {'foo': 'a', 'bar': 1}, {'bar': 2, 'baz': 'b'}) ('UPDATE tbl SET bar=$1, baz=$2 WHERE bar=$3 AND foo=$4', [2, 'b', 1, 'a']) @@ -57,27 +84,29 @@ def update_sql(table, filter_, updates): up_keys, up_vals = _split_dict(updates) changes = _pairs(up_keys, sep=", ") where = _pairs(where_keys, start=len(up_keys) + 1) - sql = f"UPDATE {table} SET {changes} WHERE {where}" # nosec + sql = f"UPDATE {table} SET {changes} WHERE {where}" # noqa: S608 # @pcrespov SQL injection issue here return sql, up_vals + where_vals -def delete(conn, table, filter_): +def delete( + conn: asyncpg.pool.PoolConnectionProxy, table: str, filter_: dict[str, Any] +) -> Coroutine[Any, Any, str]: sql, values = delete_sql(table, filter_) return conn.execute(sql, *values) -def delete_sql(table, filter_): +def delete_sql(table: str, filter_: dict[str, Any]) -> tuple[str, list[Any]]: """ >>> delete_sql('tbl', {'foo': 10, 'bar': 'baz'}) ('DELETE FROM tbl WHERE bar=$1 AND foo=$2', ['baz', 10]) """ keys, values = _split_dict(filter_) where = _pairs(keys) - sql = f"DELETE FROM {table} WHERE {where}" # nosec + sql = f"DELETE FROM {table} WHERE {where}" # noqa: S608 # @pcrespov SQL injection issue here return sql, values -def _pairs(keys, *, start=1, sep=" AND "): +def _pairs(keys: Iterable[str], *, start: int = 1, sep: str = " AND ") -> str: """ >>> _pairs(['foo', 'bar', 'baz'], sep=', ') 'foo=$1, bar=$2, baz=$3' @@ -87,7 +116,7 @@ def _pairs(keys, *, start=1, sep=" AND "): return sep.join(f"{k}=${i}" for i, k in enumerate(keys, start)) -def _placeholders(variables): +def _placeholders(variables: Iterable[Any]) -> list[Any]: """Returns placeholders by number of variables >>> _placeholders(['foo', 'bar', 1]) @@ -96,7 +125,7 @@ def _placeholders(variables): return [f"${i}" for i, _ in enumerate(variables, 1)] -def _split_dict(dic): +def _split_dict(dic: dict[str, Any]) -> tuple[list[str], list[Any]]: """Split dict into sorted keys and values >>> _split_dict({'b': 2, 'a': 1}) @@ -104,9 +133,3 @@ def _split_dict(dic): """ keys = sorted(dic.keys()) return keys, [dic[k] for k in keys] - - -if __name__ == "__main__": - import doctest - - print(doctest.testmod(optionflags=doctest.REPORT_ONLY_FIRST_FAILURE)) diff --git a/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py b/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py index ef53b904597..da6c851fa59 100644 --- a/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py +++ b/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py @@ -71,7 +71,8 @@ def _parse_extra_credits_in_usd_or_none( confirmation: ConfirmationTokenDict, ) -> PositiveInt | None: with suppress(ValidationError, JSONDecodeError): - invitation = InvitationData.parse_raw(confirmation.get("data", "EMPTY")) + confirmation_data = confirmation.get("data", "EMPTY") or "EMPTY" + invitation = InvitationData.parse_raw(confirmation_data) return invitation.extra_credits_in_usd return None @@ -234,6 +235,7 @@ async def phone_confirmation(request: web.Request): # updates confirmed phone number try: user = await db.get_user({"email": request_body.email}) + assert user is not None # nosec await db.update_user(user, {"phone": request_body.phone}) except UniqueViolation as err: @@ -242,7 +244,7 @@ async def phone_confirmation(request: web.Request): content_type=MIMETYPE_APPLICATION_JSON, ) from err - return await login_granted_response(request, user=user) + return await login_granted_response(request, user=dict(user)) # fails because of invalid or no code raise web.HTTPUnauthorized( diff --git a/services/web/server/src/simcore_service_webserver/login/plugin.py b/services/web/server/src/simcore_service_webserver/login/plugin.py index 156f25a9892..174bcd55f4a 100644 --- a/services/web/server/src/simcore_service_webserver/login/plugin.py +++ b/services/web/server/src/simcore_service_webserver/login/plugin.py @@ -49,23 +49,15 @@ async def _setup_login_storage_ctx(app: web.Application): assert APP_LOGIN_STORAGE_KEY not in app # nosec settings: PostgresSettings = get_db_plugin_settings(app) - pool: asyncpg.pool.Pool = await asyncpg.create_pool( + async with asyncpg.create_pool( dsn=settings.dsn_with_query, min_size=settings.POSTGRES_MINSIZE, max_size=settings.POSTGRES_MAXSIZE, loop=asyncio.get_event_loop(), - ) - app[APP_LOGIN_STORAGE_KEY] = storage = AsyncpgStorage(pool) + ) as pool: + app[APP_LOGIN_STORAGE_KEY] = AsyncpgStorage(pool) - yield # ---------------- - - if storage.pool is not pool: - log.error("Somebody has changed the db pool") - - try: - await asyncio.wait_for(pool.close(), timeout=MAX_TIME_TO_CLOSE_POOL_SECS) - except asyncio.TimeoutError: - log.exception("Failed to close login storage loop") + yield # ---------------- def setup_login_storage(app: web.Application): diff --git a/services/web/server/src/simcore_service_webserver/login/storage.py b/services/web/server/src/simcore_service_webserver/login/storage.py index ff6ae182dd0..a50b2f283b8 100644 --- a/services/web/server/src/simcore_service_webserver/login/storage.py +++ b/services/web/server/src/simcore_service_webserver/login/storage.py @@ -1,6 +1,6 @@ from datetime import datetime from logging import getLogger -from typing import Literal, TypedDict +from typing import Any, Literal, TypedDict, cast import asyncpg from aiohttp import web @@ -48,11 +48,11 @@ def __init__( # CRUD user # - async def get_user(self, with_data) -> asyncpg.Record: + async def get_user(self, with_data: dict[str, Any]) -> asyncpg.Record | None: async with self.pool.acquire() as conn: return await _sql.find_one(conn, self.user_tbl, with_data) - async def create_user(self, data: dict) -> asyncpg.Record: + async def create_user(self, data: dict[str, Any]) -> asyncpg.Record: async with self.pool.acquire() as conn: user_id = await _sql.insert(conn, self.user_tbl, data) new_user = await _sql.find_one(conn, self.user_tbl, {"id": user_id}) @@ -136,6 +136,6 @@ async def delete_confirmation_and_update_user( def get_plugin_storage(app: web.Application) -> AsyncpgStorage: - storage: AsyncpgStorage = app.get(APP_LOGIN_STORAGE_KEY) + storage = cast(AsyncpgStorage, app.get(APP_LOGIN_STORAGE_KEY)) assert storage, "login plugin was not initialized" # nosec return storage diff --git a/services/web/server/src/simcore_service_webserver/login/utils_email.py b/services/web/server/src/simcore_service_webserver/login/utils_email.py index fb6461c4024..a5746e2fde8 100644 --- a/services/web/server/src/simcore_service_webserver/login/utils_email.py +++ b/services/web/server/src/simcore_service_webserver/login/utils_email.py @@ -20,6 +20,7 @@ async def get_template_path(request: web.Request, filename: str) -> Path: # prevents auto-removal by pycln +# mypy: disable-error-code=truthy-function assert AttachmentTuple # nosec assert send_email_from_template # nosec diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks.py b/services/web/server/src/simcore_service_webserver/long_running_tasks.py index bcea9998144..7f689c28f33 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks.py @@ -4,7 +4,7 @@ from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import Field -from servicelib.aiohttp.long_running_tasks._server import ( +from servicelib.aiohttp.long_running_tasks._constants import ( RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, ) from servicelib.aiohttp.long_running_tasks.server import setup diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py b/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py index 186b2edcd76..2c6473aef36 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py @@ -13,7 +13,8 @@ from models_library.basic_types import MD5Str, SHA1Str from models_library.function_services_catalog import is_iterator_service from models_library.projects import ProjectID -from models_library.projects_nodes import Node, NodeID, OutputID, OutputTypes +from models_library.projects_nodes import Node, OutputID, OutputTypes +from models_library.projects_nodes_io import NodeID from models_library.services import ServiceMetaDataPublished from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, ValidationError diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling/_version_control.py b/services/web/server/src/simcore_service_webserver/meta_modeling/_version_control.py index 28140934cbc..05ddb1ea5e0 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling/_version_control.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling/_version_control.py @@ -100,13 +100,14 @@ async def create_workcopy_and_branch_from_commit( # SEE https://fastapi.tiangolo.com/tutorial/encoder/ project = jsonable_encoder(project, sqlalchemy_safe=True) - commit_id: CommitID - async with self.engine.acquire() as conn: # existance check prevents errors later - if tag := await self.TagsOrm(conn).set_filter(name=tag_name).fetch(): - commit_id = tag.commit_id - return commit_id + if ( + existing_tag := await self.TagsOrm(conn) + .set_filter(name=tag_name) + .fetch() + ): + return existing_tag.commit_id # get workcopy for start_commit_id and update with 'project' repo = ( @@ -162,8 +163,7 @@ async def create_workcopy_and_branch_from_commit( hidden=IS_INTERNAL_OPERATION, ) - commit_id = branch.head_commit_id - return commit_id + return branch.head_commit_id async def get_children_tags( self, repo_id: int, commit_id: int diff --git a/services/web/server/src/simcore_service_webserver/products/_db.py b/services/web/server/src/simcore_service_webserver/products/_db.py index 988fb9f91b1..d0317fdc61b 100644 --- a/services/web/server/src/simcore_service_webserver/products/_db.py +++ b/services/web/server/src/simcore_service_webserver/products/_db.py @@ -1,14 +1,13 @@ import logging -from collections.abc import AsyncIterator from decimal import Decimal -from typing import NamedTuple +from typing import AsyncIterator, NamedTuple import sqlalchemy as sa from aiopg.sa.connection import SAConnection from aiopg.sa.result import ResultProxy, RowProxy from models_library.products import ProductName, ProductStripeInfoGet from simcore_postgres_database.constants import QUANTIZE_EXP_ARG -from simcore_postgres_database.models.products import jinja2_templates +from simcore_postgres_database.models.jinja2_templates import jinja2_templates from simcore_postgres_database.utils_products_prices import ( ProductPriceInfo, get_product_latest_price_info_or_none, diff --git a/services/web/server/src/simcore_service_webserver/products/_events.py b/services/web/server/src/simcore_service_webserver/products/_events.py index 17ebfc8412b..edcbb68abe6 100644 --- a/services/web/server/src/simcore_service_webserver/products/_events.py +++ b/services/web/server/src/simcore_service_webserver/products/_events.py @@ -2,6 +2,7 @@ import tempfile from collections import OrderedDict from pathlib import Path +from typing import cast from aiohttp import web from aiopg.sa.engine import Engine @@ -44,11 +45,11 @@ async def auto_create_products_groups(app: web.Application) -> None: NOTE: could not add this in 'setup_groups' (groups plugin) since it has to be executed BEFORE 'load_products_on_startup' """ - engine: Engine = app[APP_DB_ENGINE_KEY] + engine = cast(Engine, app[APP_DB_ENGINE_KEY]) async with engine.acquire() as connection: async for row in iter_products(connection): - product_name = row.name + product_name = row.name # type: ignore[attr-defined] # sqlalchemy product_group_id = await get_or_create_product_group( connection, product_name ) @@ -78,10 +79,10 @@ async def load_products_on_startup(app: web.Application): async with engine.acquire() as connection: async for row in iter_products(connection): try: - name = row.name + name = row.name # type: ignore[attr-defined] # sqlalchemy payments = await get_product_payment_fields( - connection, product_name=row.name + connection, product_name=name ) app_products[name] = Product( @@ -92,7 +93,7 @@ async def load_products_on_startup(app: web.Application): assert name in FRONTEND_APPS_AVAILABLE # nosec - except ValidationError as err: # noqa: PERF203 + except ValidationError as err: msg = f"Invalid product configuration in db '{row}':\n {err}" raise InvalidConfig(msg) from err diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py index c52246d6dc4..fff41cd016c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py @@ -130,7 +130,7 @@ class Config: async def list_project_comments(request: web.Request): req_ctx = RequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(_ProjectCommentsPathParams, request) - query_params = parse_request_query_parameters_as( + query_params: _ListProjectCommentsQueryParams = parse_request_query_parameters_as( _ListProjectCommentsQueryParams, request ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_common_models.py b/services/web/server/src/simcore_service_webserver/projects/_common_models.py index 3a79cead257..091df0ef49a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_common_models.py +++ b/services/web/server/src/simcore_service_webserver/projects/_common_models.py @@ -13,8 +13,8 @@ class RequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] - product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] + user_id: UserID = Field(..., alias=RQT_USERID_KEY) + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) class ProjectPathParams(BaseModel): diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index a38a384d5fb..dd70fe4a98d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -115,7 +115,9 @@ async def _wrapper(request: web.Request) -> web.StreamResponse: @permission_required("services.pipeline.*") # due to update_pipeline_db async def create_project(request: web.Request): req_ctx = RequestContext.parse_obj(request) - query_params = parse_request_query_parameters_as(ProjectCreateParams, request) + query_params: ProjectCreateParams = parse_request_query_parameters_as( + ProjectCreateParams, request + ) header_params = parse_request_headers_as(ProjectCreateHeaders, request) if query_params.as_template: # create template from await check_user_permission(request, "project.template.create") @@ -130,8 +132,10 @@ async def create_project(request: web.Request): predefined_project = None else: # request w/ body (I found cases in which body = {}) - project_create = await parse_request_body_as( - ProjectCreateNew | ProjectCopyOverride | EmptyModel, request + project_create: ProjectCreateNew | ProjectCopyOverride | EmptyModel = ( + await parse_request_body_as( + ProjectCreateNew | ProjectCopyOverride | EmptyModel, request + ) ) predefined_project = ( project_create.dict( @@ -177,7 +181,7 @@ async def list_projects(request: web.Request): """ req_ctx = RequestContext.parse_obj(request) - query_params = parse_request_query_parameters_as( + query_params: ProjectListWithJsonStrParams = parse_request_query_parameters_as( ProjectListWithJsonStrParams, request ) @@ -226,7 +230,9 @@ async def get_active_project(request: web.Request) -> web.Response: web.HTTPNotFound: If active project is not found """ req_ctx = RequestContext.parse_obj(request) - query_params = parse_request_query_parameters_as(ProjectActiveParams, request) + query_params: ProjectActiveParams = parse_request_query_parameters_as( + ProjectActiveParams, request + ) try: user_active_projects = [] diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py index b3542abdb3f..9d9ec98ed1d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py @@ -25,18 +25,18 @@ class ProjectCreateHeaders(BaseModel): - simcore_user_agent: str = Field( # type: ignore[pydantic-alias] + simcore_user_agent: str = Field( default=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, description="Optional simcore user agent", alias=X_SIMCORE_USER_AGENT, ) - parent_project_uuid: ProjectID | None = Field( # type: ignore[pydantic-alias] + parent_project_uuid: ProjectID | None = Field( default=None, description="Optional parent project UUID", alias=X_SIMCORE_PARENT_PROJECT_UUID, ) - parent_node_id: NodeID | None = Field( # type: ignore[pydantic-alias] + parent_node_id: NodeID | None = Field( default=None, description="Optional parent node ID", alias=X_SIMCORE_PARENT_NODE_ID, diff --git a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py index 2f04c71eb3d..d3576b74b03 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py @@ -195,7 +195,9 @@ async def _get_everyone_group(cls, conn: SAConnection) -> RowProxy: result = await conn.execute( sa.select(groups).where(groups.c.type == GroupType.EVERYONE) ) - return await result.first() + row = await result.first() + assert row is not None # nosec + return row @classmethod async def _list_user_groups( diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index 23bd16cadb6..0bfa7467382 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -40,21 +40,21 @@ async def wrapper(*args, **kwargs) -> Any: return await fct(*args, **kwargs) except DBProjectNotFoundError as err: - raise ProjectNotFoundError(project_uuid=err.project_uuid) from err + raise ProjectNotFoundError(project_uuid=err.project_uuid) from err # type: ignore[attr-defined] # context defined in pydantic error except ProjectNodesNodeNotFoundError as err: raise NodeNotFoundError( - project_uuid=err.project_uuid, node_uuid=err.node_id + project_uuid=err.project_uuid, node_uuid=err.node_id # type: ignore[attr-defined] # context defined in pydantic error ) from err except ProjectNodesNonUniqueNodeFoundError as err: raise ProjectInvalidUsageError from err except DBProjectInvalidParentNodeError as err: raise ParentNodeNotFoundError( - project_uuid=err.project_uuid, node_uuid=err.parent_node_id + project_uuid=err.project_uuid, node_uuid=err.parent_node_id # type: ignore[attr-defined] # context defined in pydantic error ) from err except DBProjectInvalidParentProjectError as err: raise ParentProjectNotFoundError( - project_uuid=err.parent_project_uuid + project_uuid=err.parent_project_uuid # type: ignore[attr-defined] # context defined in pydantic error ) from err except DBProjectInvalidAncestorsError as err: raise ProjectInvalidUsageError from err diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py index 6332f8663d7..67a85d012f4 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py @@ -9,8 +9,8 @@ from aiohttp.client import ClientError from models_library.api_schemas_storage import FileMetaDataGet from models_library.projects import ProjectID -from models_library.projects_nodes import Node, NodeID -from models_library.projects_nodes_io import SimCoreFileLink +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import NodeID, SimCoreFileLink from models_library.users import UserID from pydantic import ( BaseModel, @@ -232,7 +232,7 @@ async def get_node_screenshots( file_url = await get_download_link(app, user_id, filelink) screenshots.append( NodeScreenshot( - thumbnail_url=f"https://placehold.co/170x120?text={text}", # type: ignore[arg-type] + thumbnail_url=f"https://placehold.co/170x120?text={text}", file_url=file_url, ) ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index a92c6e33acf..7c9188bfd64 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -26,8 +26,7 @@ ) from models_library.groups import EVERYONE_GROUP_ID, Group, GroupTypeInModel from models_library.projects import Project, ProjectID -from models_library.projects_nodes import NodeID -from models_library.projects_nodes_io import NodeIDStr +from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.services import ServiceKeyVersion from models_library.services_resources import ServiceResourcesDict from models_library.users import GroupID @@ -521,7 +520,9 @@ async def get_project_services_access_for_gid( ) -> web.Response: req_ctx = RequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) - query_params = parse_request_query_parameters_as(_ServicesAccessQuery, request) + query_params: _ServicesAccessQuery = parse_request_query_parameters_as( + _ServicesAccessQuery, request + ) project = await projects_api.get_project_for_user( request.app, @@ -567,7 +568,7 @@ async def get_project_services_access_for_gid( _, _user_groups, _ = await list_user_groups_with_read_access( app=request.app, user_id=_user_id ) - groups_to_compare.update({int(item.get("gid")) for item in _user_groups}) + groups_to_compare.update({int(item.get("gid", -1)) for item in _user_groups}) groups_to_compare.add(query_params.for_gid) elif _sharing_with_group.group_type == GroupTypeInModel.STANDARD: groups_to_compare = {query_params.for_gid} diff --git a/services/web/server/src/simcore_service_webserver/projects/_permalink_api.py b/services/web/server/src/simcore_service_webserver/projects/_permalink_api.py index 46f007c9445..241376ae76a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_permalink_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_permalink_api.py @@ -40,7 +40,6 @@ async def _create_permalink( request: web.Request, project_id: ProjectID ) -> ProjectPermalink: create = _get_factory(request.app) - assert create # nosec try: permalink: ProjectPermalink = await asyncio.wait_for( diff --git a/services/web/server/src/simcore_service_webserver/projects/_ports_api.py b/services/web/server/src/simcore_service_webserver/projects/_ports_api.py index 689b929b0ef..5704c7cf768 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_ports_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_ports_api.py @@ -8,14 +8,15 @@ TasksOutputs, TasksSelection, ) +from models_library.basic_types import KeyIDStr from models_library.function_services_catalog.api import ( catalog, is_parameter_service, is_probe_service, ) from models_library.projects import ProjectID -from models_library.projects_nodes import Node, NodeID -from models_library.projects_nodes_io import PortLink +from models_library.projects_nodes import Node, OutputsDict +from models_library.projects_nodes_io import NodeID, PortLink from models_library.utils.json_schema import ( JsonSchemaValidationError, jsonschema_validate_data, @@ -124,7 +125,7 @@ def set_inputs_in_project( """ modified = set() for node_id, value in update.items(): - output = {"out_1": value} + output: OutputsDict = {KeyIDStr("out_1"): value} if (node := workbench[node_id]) and node.outputs != output: # validates value against jsonschema @@ -163,7 +164,9 @@ def _get_outputs_in_workbench(workbench: dict[NodeID, Node]) -> dict[NodeID, Any if port.node.inputs: try: # Every port is associated to the output of a task - port_link = _NonStrictPortLink.parse_obj(port.node.inputs["in_1"]) + port_link = _NonStrictPortLink.parse_obj( + port.node.inputs[KeyIDStr("in_1")] + ) # Here we resolve which task and which tasks' output is associated to this port? task_node_id = port_link.node_uuid task_output_name = port_link.output @@ -182,7 +185,7 @@ def _get_outputs_in_workbench(workbench: dict[NodeID, Node]) -> dict[NodeID, Any ) except ValidationError: # not a link - value = port.node.inputs["in_1"] + value = port.node.inputs[KeyIDStr("in_1")] else: value = None @@ -197,7 +200,7 @@ async def _get_computation_tasks_outputs( batch: TasksOutputs = await get_batch_tasks_outputs( app, project_id=project_id, selection=selection ) - return batch.nodes_outputs # type: ignore[no-any-return] + return batch.nodes_outputs async def get_project_outputs( diff --git a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py index e4c914aa2da..2ab24d8c0e5 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py @@ -14,7 +14,8 @@ ProjectOutputGet, ) from models_library.projects import ProjectID -from models_library.projects_nodes import Node, NodeID +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import NodeID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.utils.json_serialization import json_dumps @@ -41,7 +42,7 @@ log = logging.getLogger(__name__) -def _web_json_response_enveloped(data: Any): +def _web_json_response_enveloped(data: Any) -> web.Response: return web.json_response( { "data": jsonable_encoder(data), diff --git a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py index 661559b6351..a687aba4513 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py @@ -1,6 +1,7 @@ """ handlers for project states """ + import contextlib import functools import json @@ -87,7 +88,9 @@ class _OpenProjectQuery(BaseModel): async def open_project(request: web.Request) -> web.Response: req_ctx = RequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) - query_params = parse_request_query_parameters_as(_OpenProjectQuery, request) + query_params: _OpenProjectQuery = parse_request_query_parameters_as( + _OpenProjectQuery, request + ) try: client_session_id = await request.json() @@ -109,9 +112,9 @@ async def open_project(request: web.Request) -> web.Response: project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, include_state=True, - check_permissions="read|write" - if project_type is ProjectType.TEMPLATE - else "read", + check_permissions=( + "read|write" if project_type is ProjectType.TEMPLATE else "read" + ), ) product: Product = get_current_product(request) diff --git a/services/web/server/src/simcore_service_webserver/projects/db.py b/services/web/server/src/simcore_service_webserver/projects/db.py index ee008e97359..059ea7ea404 100644 --- a/services/web/server/src/simcore_service_webserver/projects/db.py +++ b/services/web/server/src/simcore_service_webserver/projects/db.py @@ -7,7 +7,7 @@ import logging from contextlib import AsyncExitStack -from typing import Any +from typing import Any, cast from uuid import uuid1 import sqlalchemy as sa @@ -99,13 +99,13 @@ class ProjectDBAPI(BaseProjectDB): - def __init__(self, app: web.Application): + def __init__(self, app: web.Application) -> None: self._app = app - self._engine = app.get(APP_DB_ENGINE_KEY) + self._engine = cast(Engine, app.get(APP_DB_ENGINE_KEY)) - def _init_engine(self): + def _init_engine(self) -> None: # Delays creation of engine because it setup_db does it on_startup - self._engine = self._app.get(APP_DB_ENGINE_KEY) + self._engine = cast(Engine, self._app.get(APP_DB_ENGINE_KEY)) if self._engine is None: msg = "Database subsystem was not initialized" raise ValueError(msg) @@ -602,9 +602,7 @@ async def replace_project( exclude_foreign=["tags"], for_update=True, ) - user_groups: list[RowProxy] = await self._list_user_groups( - db_connection, user_id - ) + user_groups = await self._list_user_groups(db_connection, user_id) check_project_permissions(current_project, user_id, user_groups, "write") # uuid can ONLY be set upon creation if current_project["uuid"] != new_project_data["uuid"]: @@ -926,7 +924,7 @@ async def update_project_node( async def list_project_nodes(self, project_id: ProjectID) -> list[ProjectNode]: project_nodes_repo = ProjectNodesRepo(project_uuid=project_id) async with self.engine.acquire() as conn: - return await project_nodes_repo.list(conn) # type: ignore[no-any-return] + return await project_nodes_repo.list(conn) async def node_id_exists(self, node_id: NodeID) -> bool: """Returns True if the node id exists in any of the available projects""" diff --git a/services/web/server/src/simcore_service_webserver/projects/lock.py b/services/web/server/src/simcore_service_webserver/projects/lock.py index deeaddf2b78..8a7ff51576d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/lock.py +++ b/services/web/server/src/simcore_service_webserver/projects/lock.py @@ -8,7 +8,8 @@ import redis from aiohttp import web from models_library.projects import ProjectID -from models_library.projects_state import Owner, ProjectLocked, ProjectStatus +from models_library.projects_access import Owner +from models_library.projects_state import ProjectLocked, ProjectStatus from redis.asyncio.lock import Lock from servicelib.background_task import periodic_task from servicelib.logging_utils import log_context diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index d6a53bde808..5eca2ca5ff7 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -32,13 +32,14 @@ ) from models_library.api_schemas_webserver.projects import ProjectPatch from models_library.api_schemas_webserver.projects_nodes import NodePatch +from models_library.basic_types import KeyIDStr from models_library.errors import ErrorDict from models_library.products import ProductName from models_library.projects import Project, ProjectID, ProjectIDStr +from models_library.projects_access import Owner from models_library.projects_nodes import Node, OutputsDict from models_library.projects_nodes_io import NodeID, NodeIDStr, PortLink from models_library.projects_state import ( - Owner, ProjectLocked, ProjectRunningState, ProjectState, @@ -472,7 +473,7 @@ async def _check_project_node_has_all_required_inputs( unset_required_inputs: list[str] = [] unset_outputs_in_upstream: list[tuple[str, str]] = [] - def _check_required_input(required_input_key: str) -> None: + def _check_required_input(required_input_key: KeyIDStr) -> None: input_entry: PortLink | None = None if node.inputs: input_entry = node.inputs.get(required_input_key, None) diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/registry.py b/services/web/server/src/simcore_service_webserver/resource_manager/registry.py index 68bb1527881..9ba7c7fcee8 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/registry.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/registry.py @@ -94,7 +94,7 @@ async def set_resource( ) -> None: hash_key = f"{self._hash_key(key)}:{_RESOURCE_SUFFIX}" field, value = resource - await self.client.hset(hash_key, mapping={field: value}) + await self.client.hset(hash_key, mapping={field: value}) # type: ignore[misc] async def get_resources(self, key: UserSessionDict) -> ResourcesDict: hash_key = f"{self._hash_key(key)}:{_RESOURCE_SUFFIX}" diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py index f83cf2d5855..c79058fe791 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py @@ -56,8 +56,8 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _RequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] - product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] + user_id: UserID = Field(..., alias=RQT_USERID_KEY) + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_handlers.py b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_handlers.py index f25fc00c1ec..191abec399a 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_handlers.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_handlers.py @@ -35,8 +35,8 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _RequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] - product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] + user_id: UserID = Field(..., alias=RQT_USERID_KEY) + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py b/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py index 3f0120178de..8d1d25742b8 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py @@ -61,8 +61,8 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _RequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] - product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] + user_id: UserID = Field(..., alias=RQT_USERID_KEY) + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) ORDER_BY_DESCRIPTION = "Order by field (wallet_id|wallet_name|user_id|project_id|project_name|node_id|node_name|service_key|service_version|service_type|started_at|stopped_at|service_run_status|credit_cost|transaction_status) and direction (asc|desc). The default sorting order is ascending." @@ -75,9 +75,10 @@ class _ListServicesResourceUsagesQueryParams(BaseModel): description=ORDER_BY_DESCRIPTION, example='{"field": "started_at", "direction": "desc"}', ) - filters: Json[ # pylint: disable=unsubscriptable-object - ServiceResourceUsagesFilters - ] | None = Field( + filters: ( + Json[ServiceResourceUsagesFilters] # pylint: disable=unsubscriptable-object + | None + ) = Field( default=None, description="Filters to process on the resource usages list, encoded as JSON. Currently supports the filtering of 'started_at' field with 'from' and 'until' parameters in ISO 8601 format. The date range specified is inclusive.", example='{"started_at": {"from": "yyyy-mm-dd", "until": "yyyy-mm-dd"}}', @@ -154,8 +155,10 @@ class Config: @_handle_resource_usage_exceptions async def list_resource_usage_services(request: web.Request): req_ctx = _RequestContext.parse_obj(request) - query_params = parse_request_query_parameters_as( - _ListServicesResourceUsagesQueryParamsWithPagination, request + query_params: _ListServicesResourceUsagesQueryParamsWithPagination = ( + parse_request_query_parameters_as( + _ListServicesResourceUsagesQueryParamsWithPagination, request + ) ) services: ServiceRunPage = await api.list_usage_services( @@ -193,8 +196,10 @@ async def list_resource_usage_services(request: web.Request): @_handle_resource_usage_exceptions async def list_osparc_credits_aggregated_usages(request: web.Request): req_ctx = _RequestContext.parse_obj(request) - query_params = parse_request_query_parameters_as( - _ListServicesAggregatedUsagesQueryParams, request + query_params: _ListServicesAggregatedUsagesQueryParams = ( + parse_request_query_parameters_as( + _ListServicesAggregatedUsagesQueryParams, request + ) ) aggregated_services: OsparcCreditsAggregatedUsagesPage = ( @@ -231,8 +236,10 @@ async def list_osparc_credits_aggregated_usages(request: web.Request): @_handle_resource_usage_exceptions async def export_resource_usage_services(request: web.Request): req_ctx = _RequestContext.parse_obj(request) - query_params = parse_request_query_parameters_as( - _ListServicesResourceUsagesQueryParams, request + query_params: _ListServicesResourceUsagesQueryParams = ( + parse_request_query_parameters_as( + _ListServicesResourceUsagesQueryParams, request + ) ) download_url = await api.export_usage_services( app=request.app, diff --git a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py index fd46a6d5c3d..d6ac87f2c7c 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py @@ -15,9 +15,7 @@ from servicelib.aiohttp.observer import emit from servicelib.logging_utils import get_log_record_extra, log_context from servicelib.request_keys import RQT_USERID_KEY -from socketio.exceptions import ( - ConnectionRefusedError as SocketIOConnectionError, # type: ignore[import-untyped] -) +from socketio.exceptions import ConnectionRefusedError as SocketIOConnectionError from ..groups.api import list_user_groups_with_read_access from ..login.decorators import login_required diff --git a/services/web/server/src/simcore_service_webserver/statics/_events.py b/services/web/server/src/simcore_service_webserver/statics/_events.py index d97ecac822f..cfd1e044701 100644 --- a/services/web/server/src/simcore_service_webserver/statics/_events.py +++ b/services/web/server/src/simcore_service_webserver/statics/_events.py @@ -1,6 +1,7 @@ import logging import re from copy import deepcopy +from typing import Any, Final from aiohttp import web from aiohttp.client import ClientSession @@ -28,7 +29,7 @@ get_plugin_settings, ) -_RE_PRODUCTION_RELEASE_VERSION = r"^v\d+\.\d+\.\d+$" +_RE_PRODUCTION_RELEASE_VERSION: Final[re.Pattern] = re.compile(r"^v\d+\.\d+\.\d+$") _logger = logging.getLogger(__name__) @@ -40,7 +41,7 @@ # which might still not be ready. # # -_STATIC_WEBSERVER_RETRY_ON_STARTUP_POLICY = { +_STATIC_WEBSERVER_RETRY_ON_STARTUP_POLICY: Final[dict[str, Any]] = { "stop": stop_after_attempt(5), "wait": wait_fixed(1.5), "before": before_log(_logger, logging.WARNING), diff --git a/services/web/server/src/simcore_service_webserver/storage/api.py b/services/web/server/src/simcore_service_webserver/storage/api.py index 933893ff9db..2ddf66d8907 100644 --- a/services/web/server/src/simcore_service_webserver/storage/api.py +++ b/services/web/server/src/simcore_service_webserver/storage/api.py @@ -1,6 +1,7 @@ """ Storage subsystem's API: responsible of communication with storage service """ + import asyncio import logging import urllib.parse @@ -202,9 +203,10 @@ async def get_download_link( async with session.get(f"{url}") as response: response.raise_for_status() - download: PresignedLink = ( + download: PresignedLink | None = ( Envelope[PresignedLink].parse_obj(await response.json()).data ) + assert download is not None # nosec link: HttpUrl = parse_obj_as(HttpUrl, download.link) return link diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py index 55ecc1f5303..4c1f0de2d06 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py @@ -5,15 +5,17 @@ - TMP: add_new_project includes to projects and director_v2 app modules! """ + import json import logging from pathlib import Path from typing import NamedTuple from aiohttp import web -from models_library.projects import AccessRights, Project, ProjectID -from models_library.projects_nodes import Node, NodeID -from models_library.projects_nodes_io import DownloadLink, PortLink +from models_library.projects import Project, ProjectID +from models_library.projects_access import AccessRights +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import DownloadLink, NodeID, PortLink from models_library.projects_ui import StudyUI from models_library.services import ServiceKey, ServiceVersion from pydantic import AnyUrl, HttpUrl, parse_obj_as diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py index 00217822f5f..8ed9d777c65 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py @@ -80,7 +80,7 @@ def _create_service_info_from(service: ValidService) -> ServiceInfo: ) if service.thumbnail: values_map["thumbnail"] = service.thumbnail - return ServiceInfo.construct(_fields_set=set(values_map.keys()), **values_map) # type: ignore + return ServiceInfo.construct(_fields_set=set(values_map.keys()), **values_map) def _handle_errors_with_error_page(handler: Handler): diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py index 1e826f971d7..54ae3a2d912 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py @@ -14,6 +14,7 @@ from contextlib import suppress from datetime import datetime +import asyncpg import redis.asyncio as aioredis from aiohttp import web from models_library.emails import LowerCaseEmailStr @@ -111,7 +112,7 @@ async def create_temporary_guest_user(request: web.Request): # # (1) read details above - usr = {} + usr: asyncpg.Record | None = None try: async with redis_locks_client.lock( GUEST_USER_RC_LOCK_FORMAT.format(user_id=random_user_name), @@ -148,7 +149,7 @@ async def create_temporary_guest_user(request: web.Request): # stop creating GUEST users. # NOTE: here we cleanup but if any trace is left it will be deleted by gc - if usr.get("id"): + if usr is not None and usr.get("id"): async def _cleanup(draft_user): with suppress(Exception): diff --git a/services/web/server/src/simcore_service_webserver/tags/_handlers.py b/services/web/server/src/simcore_service_webserver/tags/_handlers.py index bd1acd631e1..8603cce3d28 100644 --- a/services/web/server/src/simcore_service_webserver/tags/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/tags/_handlers.py @@ -47,7 +47,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _RequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore + user_id: UserID = Field(..., alias=RQT_USERID_KEY) class _InputSchema(BaseModel): diff --git a/services/web/server/src/simcore_service_webserver/users/_handlers.py b/services/web/server/src/simcore_service_webserver/users/_handlers.py index 9067b00d15b..3b9f40ea84d 100644 --- a/services/web/server/src/simcore_service_webserver/users/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/users/_handlers.py @@ -30,8 +30,8 @@ class UsersRequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] - product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] + user_id: UserID = Field(..., alias=RQT_USERID_KEY) + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) def _handle_users_exceptions(handler: Handler): @@ -90,7 +90,9 @@ async def search_users(request: web.Request) -> web.Response: req_ctx = UsersRequestContext.parse_obj(request) assert req_ctx.product_name # nosec - query_params = parse_request_query_parameters_as(_SearchQueryParams, request) + query_params: _SearchQueryParams = parse_request_query_parameters_as( + _SearchQueryParams, request + ) found = await _api.search_users( request.app, email_glob=query_params.email, include_products=True diff --git a/services/web/server/src/simcore_service_webserver/users/_preferences_handlers.py b/services/web/server/src/simcore_service_webserver/users/_preferences_handlers.py index 6043730c9ba..cae7c4c5485 100644 --- a/services/web/server/src/simcore_service_webserver/users/_preferences_handlers.py +++ b/services/web/server/src/simcore_service_webserver/users/_preferences_handlers.py @@ -29,8 +29,8 @@ class _RequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] - product_name: ProductName = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] + user_id: UserID = Field(..., alias=RQT_USERID_KEY) + product_name: ProductName = Field(..., alias=RQ_PRODUCT_KEY) def _handle_users_exceptions(handler: Handler): diff --git a/services/web/server/src/simcore_service_webserver/users/api.py b/services/web/server/src/simcore_service_webserver/users/api.py index a37384ed6ec..ea52df70f20 100644 --- a/services/web/server/src/simcore_service_webserver/users/api.py +++ b/services/web/server/src/simcore_service_webserver/users/api.py @@ -1,3 +1,4 @@ +# mypy: disable-error-code=truthy-function """ This should be the interface other modules should use to get information from user module diff --git a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py index 8031872c0ea..ae35a58ee6f 100644 --- a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py +++ b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py @@ -21,8 +21,8 @@ def rename_routes_as_handler_function(routes: RouteTableDef, *, prefix: str): - route: RouteDef for route in routes: + assert isinstance(route, RouteDef) # nosec route.kwargs["name"] = f"{prefix}.{route.handler.__name__}" diff --git a/services/web/server/src/simcore_service_webserver/version_control/_handlers.py b/services/web/server/src/simcore_service_webserver/version_control/_handlers.py index 625dbc31c10..0cf849effb0 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/version_control/_handlers.py @@ -69,7 +69,9 @@ async def _list_repos_handler(request: web.Request): url_for = create_url_for_function(request) vc_repo = VersionControlRepository.create_from_request(request) - query_params = parse_request_query_parameters_as(PageQueryParameters, request) + query_params: PageQueryParameters = parse_request_query_parameters_as( + PageQueryParameters, request + ) repos_rows, total_number_of_repos = await list_repos( vc_repo, offset=query_params.offset, limit=query_params.limit @@ -146,7 +148,9 @@ async def _list_checkpoints_handler(request: web.Request): vc_repo = VersionControlRepository.create_from_request(request) path_params = parse_request_path_parameters_as(_ProjectPathParam, request) - query_params = parse_request_query_parameters_as(PageQueryParameters, request) + query_params: PageQueryParameters = parse_request_query_parameters_as( + PageQueryParameters, request + ) checkpoints: list[Checkpoint] diff --git a/services/web/server/src/simcore_service_webserver/version_control/db.py b/services/web/server/src/simcore_service_webserver/version_control/db.py index 0f485fb2857..87b4237babf 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/db.py +++ b/services/web/server/src/simcore_service_webserver/version_control/db.py @@ -137,7 +137,7 @@ async def _fetch_workcopy_project_id( ) for tag in found: if workcopy_project_id := parse_workcopy_project_tag_name(tag.name): - return str(workcopy_project_id) + return ProjectIDStr(workcopy_project_id) repo = await self.ReposOrm(conn).set_filter(id=repo_id).fetch("project_uuid") assert repo # nosec diff --git a/services/web/server/src/simcore_service_webserver/wallets/_groups_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_groups_handlers.py index 4d811f4d475..723ea2655c6 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_groups_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_groups_handlers.py @@ -31,8 +31,8 @@ class _RequestContext(BaseModel): - user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[pydantic-alias] - product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[pydantic-alias] + user_id: UserID = Field(..., alias=RQT_USERID_KEY) + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) def _handle_wallets_groups_exceptions(handler: Handler): diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 7ceed719c9b..27060372abd 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -114,7 +114,9 @@ async def _list_all_payments(request: web.Request): """ req_ctx = WalletsRequestContext.parse_obj(request) - query_params = parse_request_query_parameters_as(PageQueryParameters, request) + query_params: PageQueryParameters = parse_request_query_parameters_as( + PageQueryParameters, request + ) payments, total_number_of_items = await list_user_payments_page( request.app, diff --git a/services/web/server/tests/unit/with_dbs/03/test_email.py b/services/web/server/tests/unit/with_dbs/03/test_email.py index bb1ef162001..e2164071c16 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_email.py +++ b/services/web/server/tests/unit/with_dbs/03/test_email.py @@ -74,18 +74,24 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc @pytest.fixture -def mocked_send_email(mocker: MockerFixture, app_environment: EnvVarsDict) -> MagicMock: - # Overrides services/web/server/tests/unit/with_dbs/conftest.py::mocked_send_email +async def mocked_aiosmtplib(mocker: MockerFixture) -> MagicMock: + mocked_lib = mocker.patch("aiosmtplib.SMTP", autospec=True) settings = SMTPSettings.create_from_envs() + mocked_lib.return_value.__aenter__.return_value.hostname = settings.SMTP_HOST + mocked_lib.return_value.__aenter__.return_value.port = settings.SMTP_PORT + mocked_lib.return_value.__aenter__.return_value.timeout = 100 + mocked_lib.return_value.__aenter__.return_value.use_tls = ( + settings.SMTP_PROTOCOL == EmailProtocol.TLS + ) + return mocked_lib - mock = mocker.patch("aiosmtplib.SMTP") - smtp_instance = mock.return_value.__aenter__.return_value - smtp_instance.hostname = settings.SMTP_HOST - smtp_instance.port = settings.SMTP_PORT - smtp_instance.timeout = 100 - smtp_instance.use_tls = settings.SMTP_PROTOCOL == EmailProtocol.TLS - return smtp_instance +@pytest.fixture +async def mocked_send_email( + mocked_aiosmtplib: MagicMock, mocker: MockerFixture, app_environment: EnvVarsDict +) -> MagicMock: + # Overrides services/web/server/tests/unit/with_dbs/conftest.py::mocked_send_email + return mocked_aiosmtplib.return_value @pytest.mark.parametrize( @@ -104,6 +110,7 @@ async def test_email_handlers( logged_user: UserInfoDict, user_role: UserRole, expected_response_cls: type[web.Response], + mocked_aiosmtplib: MagicMock, mocked_send_email: MagicMock, ): assert logged_user["role"] == user_role.name