From 2f3c8d92a5b1c68a71b7c59540a44d29598608f9 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Dec 2024 10:36:56 +0100 Subject: [PATCH 1/5] fix docstring --- .../director_v2/_core_dynamic_services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py b/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py index bd4c03f31fe..463bec27147 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py @@ -67,11 +67,11 @@ async def request_retrieve_dyn_service( @log_decorator(logger=_log) async def restart_dynamic_service(app: web.Application, node_uuid: str) -> None: - """User restart the dynamic dynamic service started in the node_uuid + """Restarts the user service(s) started by the the node_uuid's sidecar NOTE that this operation will NOT restart all sidecar services (``simcore-service-dynamic-sidecar`` or ``reverse-proxy caddy`` services) but - ONLY those containers in the compose-spec (i.e. the ones exposed to the user) + ONLY user services. """ settings: DirectorV2Settings = get_plugin_settings(app) await request_director_v2( From 62f0089c18d4f23b2c5e71a29ad53e9e4dea2918 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Dec 2024 10:38:42 +0100 Subject: [PATCH 2/5] docstring --- .../director_v2/_core_dynamic_services.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py b/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py index 463bec27147..f1be2dc4e3d 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py @@ -69,9 +69,9 @@ async def request_retrieve_dyn_service( async def restart_dynamic_service(app: web.Application, node_uuid: str) -> None: """Restarts the user service(s) started by the the node_uuid's sidecar - NOTE that this operation will NOT restart all sidecar services - (``simcore-service-dynamic-sidecar`` or ``reverse-proxy caddy`` services) but - ONLY user services. + NOTE: this operation will NOT restart + sidecar services (``dy-sidecar`` or ``dy-proxy`` services), + but ONLY user services (the ones defined by the compose spec). """ settings: DirectorV2Settings = get_plugin_settings(app) await request_director_v2( From 7a74839522610722a34ffe7739712c89cd85779b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Dec 2024 13:12:57 +0100 Subject: [PATCH 3/5] rerouted restart user services --- .../dynamic_scheduler/services.py | 16 ++++++++++++++ .../api/rpc/_services.py | 5 +++++ .../services/director_v2/_public_client.py | 3 +++ .../services/director_v2/_thin_client.py | 5 +++++ .../services/scheduler_interface.py | 9 ++++++++ .../unit/api_rpc/test_api_rpc__services.py | 22 +++++++++++++++++++ .../director_v2/_core_dynamic_services.py | 18 --------------- .../director_v2/api.py | 2 -- .../director_v2/settings.py | 8 ------- .../dynamic_scheduler/api.py | 17 ++++++++++++++ .../dynamic_scheduler/settings.py | 8 +++++++ .../projects/_nodes_handlers.py | 4 +++- 12 files changed, 88 insertions(+), 29 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/services.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/services.py index d941f889bd7..32645a7685a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/services.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/services.py @@ -93,3 +93,19 @@ async def stop_dynamic_service( timeout_s=timeout_s, ) assert result is None # nosec + + +@log_decorator(_logger, level=logging.DEBUG) +async def restart_user_services( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + node_id: NodeID, + timeout_s: NonNegativeInt, +) -> None: + result = await rabbitmq_rpc_client.request( + DYNAMIC_SCHEDULER_RPC_NAMESPACE, + _RPC_METHOD_NAME_ADAPTER.validate_python("restart_user_services"), + node_id=node_id, + timeout_s=timeout_s, + ) + assert result is None # nosec diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rpc/_services.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rpc/_services.py index 8cd90ddb8f0..aa20449b92e 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rpc/_services.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rpc/_services.py @@ -56,3 +56,8 @@ async def stop_dynamic_service( return await scheduler_interface.stop_dynamic_service( app, dynamic_service_stop=dynamic_service_stop ) + + +@router.expose() +async def restart_user_services(app: FastAPI, *, node_id: NodeID) -> None: + await scheduler_interface.restart_user_services(app, node_id=node_id) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py index 4b49618d6df..1c0fb88deaf 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py @@ -108,6 +108,9 @@ async def list_tracked_dynamic_services( ) return TypeAdapter(list[DynamicServiceGet]).validate_python(response.json()) + async def restart_user_services(self, *, node_id: NodeID) -> None: + await self.thin_client.post_restart(node_id=node_id) + def setup_director_v2(app: FastAPI) -> None: public_client = DirectorV2Client(app) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py index bfb64e8839a..cf400416032 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py @@ -124,3 +124,8 @@ async def get_dynamic_services( "/dynamic_services", params=as_dict_exclude_unset(user_id=user_id, project_id=project_id), ) + + @retry_on_errors() + @expect_status(status.HTTP_204_NO_CONTENT) + async def post_restart(self, *, node_id: NodeID) -> Response: + return await self.client.post(f"/dynamic_services/{node_id}:restart") diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/scheduler_interface.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/scheduler_interface.py index 6f655b544e2..0d26ab39186 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/scheduler_interface.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/scheduler_interface.py @@ -73,3 +73,12 @@ async def stop_dynamic_service( ) await set_request_as_stopped(app, dynamic_service_stop) + + +async def restart_user_services(app: FastAPI, *, node_id: NodeID) -> None: + settings: ApplicationSettings = app.state.settings + if settings.DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: + raise NotImplementedError + + director_v2_client = DirectorV2Client.get_from_app_state(app) + await director_v2_client.restart_user_services(node_id=node_id) diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py index 7c1665065ae..cb4e38b39e5 100644 --- a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py +++ b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py @@ -490,3 +490,25 @@ async def test_stop_dynamic_service_serializes_generic_errors( ), timeout_s=5, ) + + +@pytest.fixture +def mock_director_v2_restart_user_services(node_id: NodeID) -> Iterator[None]: + with respx.mock( + base_url="http://director-v2:8000/v2", + assert_all_called=False, + assert_all_mocked=True, # IMPORTANT: KEEP always True! + ) as mock: + mock.post(f"/dynamic_services/{node_id}:restart").respond( + status.HTTP_204_NO_CONTENT + ) + + yield None + + +async def test_restart_user_services( + mock_director_v2_restart_user_services: None, + rpc_client: RabbitMQRPCClient, + node_id: NodeID, +): + await services.restart_user_services(rpc_client, node_id=node_id, timeout_s=5) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py b/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py index f1be2dc4e3d..7fd9e4d0a83 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py @@ -65,24 +65,6 @@ async def request_retrieve_dyn_service( ) -@log_decorator(logger=_log) -async def restart_dynamic_service(app: web.Application, node_uuid: str) -> None: - """Restarts the user service(s) started by the the node_uuid's sidecar - - NOTE: this operation will NOT restart - sidecar services (``dy-sidecar`` or ``dy-proxy`` services), - but ONLY user services (the ones defined by the compose spec). - """ - settings: DirectorV2Settings = get_plugin_settings(app) - await request_director_v2( - app, - "POST", - url=settings.base_url / f"dynamic_services/{node_uuid}:restart", - expected_status=web.HTTPOk, - timeout=settings.DIRECTOR_V2_RESTART_DYNAMIC_SERVICE_TIMEOUT, - ) - - @log_decorator(logger=_log) async def update_dynamic_service_networks_in_project( app: web.Application, project_id: ProjectID diff --git a/services/web/server/src/simcore_service_webserver/director_v2/api.py b/services/web/server/src/simcore_service_webserver/director_v2/api.py index f56de16b543..2d05efc5bfe 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/api.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/api.py @@ -19,7 +19,6 @@ from ._core_dynamic_services import ( get_project_inactivity, request_retrieve_dyn_service, - restart_dynamic_service, retrieve, update_dynamic_service_networks_in_project, ) @@ -39,7 +38,6 @@ "is_healthy", "is_pipeline_running", "request_retrieve_dyn_service", - "restart_dynamic_service", "retrieve", "set_project_run_policy", "stop_pipeline", diff --git a/services/web/server/src/simcore_service_webserver/director_v2/settings.py b/services/web/server/src/simcore_service_webserver/director_v2/settings.py index 21cb368ff50..31fc096a5dd 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/settings.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/settings.py @@ -33,14 +33,6 @@ def base_url(self) -> URL: # - Mostly in floats (aiohttp.Client/) but sometimes in ints # - Typically in seconds but occasionally in ms - DIRECTOR_V2_RESTART_DYNAMIC_SERVICE_TIMEOUT: PositiveInt = Field( - 1 * _MINUTE, - description="timeout of containers restart", - validation_alias=AliasChoices( - "DIRECTOR_V2_RESTART_DYNAMIC_SERVICE_TIMEOUT", - ), - ) - DIRECTOR_V2_STORAGE_SERVICE_UPLOAD_DOWNLOAD_TIMEOUT: PositiveInt = Field( _HOUR, description=( diff --git a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py index ef8f2b1f703..0b0ff9419f3 100644 --- a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py +++ b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py @@ -148,3 +148,20 @@ async def stop_dynamic_services_in_project( ] await logged_gather(*services_to_stop) + + +async def restart_user_services(app: web.Application, *, node_id: NodeID) -> None: + """Restarts the user service(s) started by the the node_uuid's sidecar + + NOTE: this operation will NOT restart + sidecar services (``dy-sidecar`` or ``dy-proxy`` services), + but ONLY user services (the ones defined by the compose spec). + """ + settings: DynamicSchedulerSettings = get_plugin_settings(app) + await services.restart_user_services( + get_rabbitmq_rpc_client(app), + node_id=node_id, + timeout_s=int( + settings.DYNAMIC_SCHEDULER_RESTART_DYNAMIC_SERVICE_TIMEOUT.total_seconds() + ), + ) diff --git a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py index 91dac1317b6..93ea94d4921 100644 --- a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py +++ b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py @@ -26,6 +26,14 @@ class DynamicSchedulerSettings(BaseCustomSettings, MixinServiceSettings): ), ) + DYNAMIC_SCHEDULER_RESTART_DYNAMIC_SERVICE_TIMEOUT: datetime.timedelta = Field( + datetime.timedelta(minutes=1), + description="timeout of containers restart", + validation_alias=AliasChoices( + "DIRECTOR_V2_RESTART_DYNAMIC_SERVICE_TIMEOUT", + ), + ) + def get_plugin_settings(app: web.Application) -> DynamicSchedulerSettings: settings = app[APP_SETTINGS_KEY].WEBSERVER_DYNAMIC_SCHEDULER diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index d5978f794d2..2888b2731ee 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -412,7 +412,9 @@ async def restart_node(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(NodePathParams, request) - await director_v2_api.restart_dynamic_service(request.app, f"{path_params.node_id}") + await dynamic_scheduler_api.restart_user_services( + request.app, node_id=path_params.node_id + ) return web.json_response(status=status.HTTP_204_NO_CONTENT) From 81bf68fd2c1d44e709f2990b57b8ca3f04d67b17 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Dec 2024 13:17:49 +0100 Subject: [PATCH 4/5] rename and docstring --- .../simcore_service_webserver/dynamic_scheduler/api.py | 2 +- .../dynamic_scheduler/settings.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py index 0b0ff9419f3..748754294a0 100644 --- a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py +++ b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py @@ -162,6 +162,6 @@ async def restart_user_services(app: web.Application, *, node_id: NodeID) -> Non get_rabbitmq_rpc_client(app), node_id=node_id, timeout_s=int( - settings.DYNAMIC_SCHEDULER_RESTART_DYNAMIC_SERVICE_TIMEOUT.total_seconds() + settings.DYNAMIC_SCHEDULER_RESTART_USER_SERVICES_TIMEOUT.total_seconds() ), ) diff --git a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py index 93ea94d4921..f73878a5643 100644 --- a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py +++ b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py @@ -26,12 +26,8 @@ class DynamicSchedulerSettings(BaseCustomSettings, MixinServiceSettings): ), ) - DYNAMIC_SCHEDULER_RESTART_DYNAMIC_SERVICE_TIMEOUT: datetime.timedelta = Field( - datetime.timedelta(minutes=1), - description="timeout of containers restart", - validation_alias=AliasChoices( - "DIRECTOR_V2_RESTART_DYNAMIC_SERVICE_TIMEOUT", - ), + DYNAMIC_SCHEDULER_RESTART_USER_SERVICES_TIMEOUT: datetime.timedelta = Field( + datetime.timedelta(minutes=1), description="timeout for user services restart" ) From c7e7956a594549cc6bd0da0ee5be0dab6fd6d515 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 11 Dec 2024 13:32:18 +0100 Subject: [PATCH 5/5] fixed failing specs --- .../server/src/simcore_service_webserver/api/v0/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index d27cbd9537e..ce36e2e6e93 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -10636,7 +10636,7 @@ components: read: true write: false description: Open to all users - gid: '0' + gid: 1 label: All me: accessRights: