pydantic2 migration made integration tests green (#6719)

Co-authored-by: Andrei Neagu <[email protected]>
Co-authored-by: Giancarlo Romeo <[email protected]>

packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py

@@ -1,7 +1,7 @@
 import json
 from contextlib import suppress
 from pathlib import Path
-from typing import Any, TypeAlias, Union
+from typing import Annotated, Any, TypeAlias
 
 from models_library.basic_regex import MIME_TYPE_RE
 from models_library.generics import DictModel
@@ -84,15 +84,16 @@ class FileUrl(BaseModel):
     )
 
 
-PortValue: TypeAlias = Union[
-    StrictBool,
-    StrictInt,
-    StrictFloat,
-    StrictStr,
-    FileUrl,
-    list[Any],
-    dict[str, Any],
-    None,
+PortValue: TypeAlias = Annotated[
+    StrictBool
+    | StrictInt
+    | StrictFloat
+    | StrictStr
+    | FileUrl
+    | list[Any]
+    | dict[str, Any]
+    | None,
+    Field(union_mode="left_to_right"),
 ]
 
 
@@ -112,7 +113,9 @@ class TaskInputData(DictModel[ServicePortKey, PortValue]):
     )
 
 
-PortSchemaValue: TypeAlias = Union[PortSchema, FilePortSchema]
+PortSchemaValue: TypeAlias = Annotated[
+    PortSchema | FilePortSchema, Field(union_mode="left_to_right")
+]
 
 
 class TaskOutputDataSchema(DictModel[ServicePortKey, PortSchemaValue]):

packages/models-library/src/models_library/api_schemas_catalog/services.py

@@ -205,7 +205,9 @@ class ServiceUpdate(ServiceMetaDataEditable, ServiceAccessRights):
 class ServiceGet(
     ServiceMetaDataPublished, ServiceAccessRights, ServiceMetaDataEditable
 ):  # pylint: disable=too-many-ancestors
-    owner: LowerCaseEmailStr | None
+    owner: LowerCaseEmailStr | None = Field(
+        description="None when the owner email cannot be found in the database"
+    )
 
     model_config = ConfigDict(
         extra="ignore",
@@ -230,7 +232,9 @@ class ServiceGetV2(BaseModel):
 
     contact: LowerCaseEmailStr | None
     authors: list[Author] = Field(..., min_length=1)
-    owner: LowerCaseEmailStr | None
+    owner: LowerCaseEmailStr | None = Field(
+        description="None when the owner email cannot be found in the database"
+    )
 
     inputs: ServiceInputsDict
     outputs: ServiceOutputsDict

packages/models-library/src/models_library/clusters.py

@@ -1,6 +1,6 @@
 from enum import auto
 from pathlib import Path
-from typing import Final, Literal, Self, TypeAlias
+from typing import Annotated, Final, Literal, Self, TypeAlias
 
 from pydantic import (
     AnyUrl,
@@ -142,7 +142,9 @@ class BaseCluster(BaseModel):
     authentication: ClusterAuthentication = Field(
         ..., description="Dask gateway authentication", discriminator="type"
     )
-    access_rights: dict[GroupID, ClusterAccessRights] = Field(default_factory=dict)
+    access_rights: Annotated[
+        dict[GroupID, ClusterAccessRights], Field(default_factory=dict)
+    ]
 
     _from_equivalent_enums = field_validator("type", mode="before")(
         create_enums_pre_validator(ClusterTypeInModel)

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -94,13 +94,37 @@ def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict:
     # ensure we do not use the bucket of simcore or so
     env_devel["S3_BUCKET_NAME"] = "pytestbucket"
 
+    # ensure OpenTelemetry is not enabled
+    env_devel |= {
+        tracing_setting: "null"
+        for tracing_setting in (
+            "AGENT_TRACING",
+            "API_SERVER_TRACING",
+            "AUTOSCALING_TRACING",
+            "CATALOG_TRACING",
+            "CLUSTERS_KEEPER_TRACING",
+            "DATCORE_ADAPTER_TRACING",
+            "DIRECTOR_TRACING",
+            "DIRECTOR_V2_TRACING",
+            "DYNAMIC_SCHEDULER_TRACING",
+            "EFS_GUARDIAN_TRACING",
+            "INVITATIONS_TRACING",
+            "PAYMENTS_TRACING",
+            "RESOURCE_USAGE_TRACKER_TRACING",
+            "STORAGE_TRACING",
+            "WB_DB_EL_TRACING",
+            "WB_GC_TRACING",
+            "WEBSERVER_TRACING",
+        )
+    }
+
     return {key: value for key, value in env_devel.items() if value is not None}
 
 
 @pytest.fixture(scope="module")
 def env_file_for_testing(
     temp_folder: Path,
-    testing_environ_vars: dict[str, str],
+    testing_environ_vars: EnvVarsDict,
     osparc_simcore_root_dir: Path,
 ) -> Iterator[Path]:
     """Dumps all the environment variables into an $(temp_folder)/.env.test file

packages/pytest-simcore/src/pytest_simcore/docker_registry.py

@@ -261,7 +261,7 @@ def jupyter_service(docker_registry: str, node_meta_schema: dict) -> dict[str, A
     )
 
 
-@pytest.fixture(scope="session", params=["2.0.4"])
+@pytest.fixture(scope="session", params=["2.0.7"])
 def dy_static_file_server_version(request: pytest.FixtureRequest):
     return request.param
 

packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py

@@ -67,9 +67,19 @@ def regex_pattern(self) -> str:
             msg = "Current version cannot compute regex patterns in case of oneOf. Please go ahead and implement it yourself."
             raise NotImplementedError(msg)
         if self.anyOf:
-            return "|".join([elm.regex_pattern for elm in self.anyOf])
+            return "|".join(
+                [
+                    elm.regex_pattern
+                    for elm in self.anyOf  # pylint:disable=not-an-iterable
+                ]
+            )
         if self.allOf:
-            return "&".join([elm.regex_pattern for elm in self.allOf])
+            return "&".join(
+                [
+                    elm.regex_pattern
+                    for elm in self.allOf  # pylint:disable=not-an-iterable
+                ]
+            )
 
         # now deal with non-recursive cases
         pattern: str | None = None

packages/pytest-simcore/src/pytest_simcore/helpers/storage.py

@@ -0,0 +1,11 @@
+from collections.abc import Callable
+
+from yarl import URL
+
+
+def replace_storage_endpoint(host: str, port: int) -> Callable[[str], str]:
+    def _(url: str) -> str:
+        url_obj = URL(url).with_host(host).with_port(port)
+        return f"{url_obj}"
+
+    return _

packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py

@@ -27,9 +27,10 @@ async def dask_scheduler_service(
     )
     # override the port
     monkeypatch.setenv("DASK_SCHEDULER_PORT", f"{dask_scheduler_api_port}")
-    return AnyUrl.build(
-        scheme="tls", host=get_localhost_ip(), port=dask_scheduler_api_port
+    url = AnyUrl.build(
+        scheme="tls", host=get_localhost_ip(), port=int(dask_scheduler_api_port)
     )
+    return f"{url}"
 
 
 @pytest.fixture

packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py

@@ -10,13 +10,14 @@
 import tenacity
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID, SimcoreS3FileID
-from pydantic import AnyUrl, TypeAdapter
+from pydantic import TypeAdapter
 from pytest_mock import MockerFixture
 from servicelib.minio_utils import ServiceRetryPolicyUponInitialization
 from yarl import URL
 
 from .helpers.docker import get_service_published_port
 from .helpers.host import get_localhost_ip
+from .helpers.storage import replace_storage_endpoint
 
 
 @pytest.fixture(scope="module")
@@ -45,22 +46,12 @@ async def storage_service(
 ) -> URL:
     await wait_till_storage_responsive(storage_endpoint)
 
-    def correct_ip(url: AnyUrl):
-        assert storage_endpoint.host is not None
-        assert storage_endpoint.port is not None
-
-        return AnyUrl.build(
-            scheme=url.scheme,
-            host=storage_endpoint.host,
-            port=f"{storage_endpoint.port}",
-            path=url.path,
-            query=url.query,
-        )
-
     # NOTE: Mock to ensure container IP agrees with host IP when testing
+    assert storage_endpoint.host is not None
+    assert storage_endpoint.port is not None
     mocker.patch(
         "simcore_sdk.node_ports_common._filemanager._get_https_link_if_storage_secure",
-        correct_ip,
+        replace_storage_endpoint(storage_endpoint.host, storage_endpoint.port),
     )
 
     return storage_endpoint

packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py

@@ -6,8 +6,7 @@
 
 from aiohttp import web
 from common_library.json_serialization import json_dumps
-from common_library.pydantic_networks_extension import AnyHttpUrlLegacy
-from pydantic import PositiveFloat, TypeAdapter
+from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter
 
 from ...aiohttp import status
 from ...long_running_tasks._models import TaskGet
@@ -68,13 +67,13 @@ async def start_long_running_task(
         ip_addr, port = request_.transport.get_extra_info(
             "sockname"
         )  # https://docs.python.org/3/library/asyncio-protocol.html#asyncio.BaseTransport.get_extra_info
-        status_url = TypeAdapter(AnyHttpUrlLegacy).validate_python(
+        status_url = TypeAdapter(AnyHttpUrl).validate_python(
             f"http://{ip_addr}:{port}{request_.app.router['get_task_status'].url_for(task_id=task_id)}"  # NOSONAR
         )
-        result_url = TypeAdapter(AnyHttpUrlLegacy).validate_python(
+        result_url = TypeAdapter(AnyHttpUrl).validate_python(
             f"http://{ip_addr}:{port}{request_.app.router['get_task_result'].url_for(task_id=task_id)}"  # NOSONAR
         )
-        abort_url = TypeAdapter(AnyHttpUrlLegacy).validate_python(
+        abort_url = TypeAdapter(AnyHttpUrl).validate_python(
             f"http://{ip_addr}:{port}{request_.app.router['cancel_and_delete_task'].url_for(task_id=task_id)}"  # NOSONAR
         )
         task_get = TaskGet(

packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py

@@ -4,10 +4,9 @@
 import warnings
 from typing import Any, Awaitable, Callable, Final
 
-from common_library.pydantic_networks_extension import AnyHttpUrlLegacy
 from fastapi import FastAPI, status
 from httpx import AsyncClient, HTTPError
-from pydantic import PositiveFloat, TypeAdapter
+from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter
 from tenacity import RetryCallState
 from tenacity.asyncio import AsyncRetrying
 from tenacity.retry import retry_if_exception_type
@@ -106,6 +105,9 @@ async def request_wrapper(zelf: "Client", *args, **kwargs) -> Any:
             with attempt:
                 return await request_func(zelf, *args, **kwargs)
 
+        msg = "Unexpected"
+        raise RuntimeError(msg)
+
     return request_wrapper
 
 
@@ -131,12 +133,13 @@ def _client_configuration(self) -> ClientConfiguration:
         return output
 
     def _get_url(self, path: str) -> str:
-        url = f"{self._base_url}{self._client_configuration.router_prefix}{path}"
-        return f"{TypeAdapter(AnyHttpUrlLegacy).validate_python(url)}"
+        url_path = f"{self._client_configuration.router_prefix}{path}".lstrip("/")
+        url = TypeAdapter(AnyHttpUrl).validate_python(f"{self._base_url}{url_path}")
+        return f"{url}"
 
     @retry_on_http_errors
     async def get_task_status(
-        self, task_id: TaskId, *, timeout: PositiveFloat | None = None
+        self, task_id: TaskId, *, timeout: PositiveFloat | None = None  # noqa: ASYNC109
     ) -> TaskStatus:
         timeout = timeout or self._client_configuration.default_timeout
         result = await self._async_client.get(
@@ -155,7 +158,7 @@ async def get_task_status(
 
     @retry_on_http_errors
     async def get_task_result(
-        self, task_id: TaskId, *, timeout: PositiveFloat | None = None
+        self, task_id: TaskId, *, timeout: PositiveFloat | None = None  # noqa: ASYNC109
     ) -> Any | None:
         timeout = timeout or self._client_configuration.default_timeout
         result = await self._async_client.get(
@@ -177,7 +180,7 @@ async def get_task_result(
 
     @retry_on_http_errors
     async def cancel_and_delete_task(
-        self, task_id: TaskId, *, timeout: PositiveFloat | None = None
+        self, task_id: TaskId, *, timeout: PositiveFloat | None = None  # noqa: ASYNC109
     ) -> None:
         timeout = timeout or self._client_configuration.default_timeout
         result = await self._async_client.delete(

packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py

@@ -4,12 +4,11 @@
 import asyncio
 from typing import AsyncIterable, Final
 
-from common_library.pydantic_networks_extension import AnyHttpUrlLegacy
 import pytest
 from asgi_lifespan import LifespanManager
 from fastapi import APIRouter, Depends, FastAPI, status
 from httpx import AsyncClient
-from pydantic import PositiveFloat, TypeAdapter
+from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter
 from servicelib.fastapi.long_running_tasks._context_manager import _ProgressManager
 from servicelib.fastapi.long_running_tasks.client import (
     Client,
@@ -50,7 +49,8 @@ async def a_test_task(task_progress: TaskProgress) -> int:
 
 async def a_failing_test_task(task_progress: TaskProgress) -> None:
     await asyncio.sleep(TASK_SLEEP_INTERVAL)
-    raise RuntimeError("I am failing as requested")
+    msg = "I am failing as requested"
+    raise RuntimeError(msg)
 
 
 @pytest.fixture
@@ -101,7 +101,7 @@ async def test_task_result(
     assert result.status_code == status.HTTP_200_OK, result.text
     task_id = result.json()
 
-    url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://backgroud.testserver.io")
+    url = TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io/")
     client = Client(app=bg_task_app, async_client=async_client, base_url=url)
     async with periodic_task_result(
         client,
@@ -121,7 +121,7 @@ async def test_task_result_times_out(
     assert result.status_code == status.HTTP_200_OK, result.text
     task_id = result.json()
 
-    url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://backgroud.testserver.io")
+    url = TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io/")
     client = Client(app=bg_task_app, async_client=async_client, base_url=url)
     timeout = TASK_SLEEP_INTERVAL / 10
     with pytest.raises(TaskClientTimeoutError) as exec_info:
@@ -147,7 +147,7 @@ async def test_task_result_task_result_is_an_error(
     assert result.status_code == status.HTTP_200_OK, result.text
     task_id = result.json()
 
-    url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://backgroud.testserver.io")
+    url = TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io/")
     client = Client(app=bg_task_app, async_client=async_client, base_url=url)
     with pytest.raises(TaskClientResultError) as exec_info:
         async with periodic_task_result(
@@ -158,7 +158,7 @@ async def test_task_result_task_result_is_an_error(
         ):
             pass
     assert f"{exec_info.value}".startswith(f"Task {task_id} finished with exception:")
-    assert 'raise RuntimeError("I am failing as requested")' in f"{exec_info.value}"
+    assert "I am failing as requested" in f"{exec_info.value}"
     await _assert_task_removed(async_client, task_id, router_prefix)
 
 
@@ -186,13 +186,17 @@ async def progress_update(
         assert received == ("", None)
 
     for _ in range(repeat):
-        await progress_updater.update(mock_task_id, percent=TypeAdapter(ProgressPercent).validate_python(0.0))
+        await progress_updater.update(
+            mock_task_id, percent=TypeAdapter(ProgressPercent).validate_python(0.0)
+        )
         assert counter == 2
         assert received == ("", 0.0)
 
     for _ in range(repeat):
         await progress_updater.update(
-            mock_task_id, percent=TypeAdapter(ProgressPercent).validate_python(1.0), message="done"
+            mock_task_id,
+            percent=TypeAdapter(ProgressPercent).validate_python(1.0),
+            message="done",
         )
         assert counter == 3
         assert received == ("done", 1.0)