diff --git a/src/neptune/common/warnings.py b/src/neptune/common/warnings.py index dbcc509bf..8ec22942b 100644 --- a/src/neptune/common/warnings.py +++ b/src/neptune/common/warnings.py @@ -55,16 +55,19 @@ def get_user_code_stack_level(): def warn_once(message: str, *, exception: type(Exception) = None): - if exception is None: - exception = NeptuneDeprecationWarning - - if message not in warned_once: - warnings.warn( - message=message, - category=exception, - stacklevel=get_user_code_stack_level(), - ) - warned_once.add(message) + if len(warned_once) < 1_000: + if exception is None: + exception = NeptuneDeprecationWarning + + message_hash = hash(message) + + if message_hash not in warned_once: + warnings.warn( + message=message, + category=exception, + stacklevel=get_user_code_stack_level(), + ) + warned_once.add(message_hash) def warn_about_unsupported_type(type_str: str): diff --git a/src/neptune/envs.py b/src/neptune/envs.py index 31bc2e144..9cec19ac5 100644 --- a/src/neptune/envs.py +++ b/src/neptune/envs.py @@ -19,6 +19,7 @@ "PROJECT_ENV_NAME", "CUSTOM_RUN_ID_ENV_NAME", "MONITORING_NAMESPACE", + "NEPTUNE_SAFETY_MODE", "NEPTUNE_ALLOW_SELF_SIGNED_CERTIFICATE", "NEPTUNE_NOTEBOOK_ID", "NEPTUNE_NOTEBOOK_PATH", @@ -48,6 +49,8 @@ MONITORING_NAMESPACE = "NEPTUNE_MONITORING_NAMESPACE" +NEPTUNE_SAFETY_MODE = "NEPTUNE_SAFETY_MODE" + NEPTUNE_ALLOW_SELF_SIGNED_CERTIFICATE = "NEPTUNE_ALLOW_SELF_SIGNED_CERTIFICATE" NEPTUNE_NOTEBOOK_ID = "NEPTUNE_NOTEBOOK_ID" diff --git a/src/neptune/metadata_containers/model.py b/src/neptune/metadata_containers/model.py index 69a17bce4..53f7827c6 100644 --- a/src/neptune/metadata_containers/model.py +++ b/src/neptune/metadata_containers/model.py @@ -55,9 +55,11 @@ from neptune.metadata_containers import MetadataContainer from neptune.metadata_containers.abstract import NeptuneObjectCallback from neptune.metadata_containers.metadata_containers_table import Table +from neptune.metadata_containers.safe_container import safety_decorator from neptune.types.mode import Mode +@safety_decorator class Model(MetadataContainer): """Class for registering a model to neptune.ai and retrieving information from it.""" diff --git a/src/neptune/metadata_containers/model_version.py b/src/neptune/metadata_containers/model_version.py index 2ac34c846..f071a56f4 100644 --- a/src/neptune/metadata_containers/model_version.py +++ b/src/neptune/metadata_containers/model_version.py @@ -47,10 +47,12 @@ from neptune.internal.utils.ping_background_job import PingBackgroundJob from neptune.metadata_containers import MetadataContainer from neptune.metadata_containers.abstract import NeptuneObjectCallback +from neptune.metadata_containers.safe_container import safety_decorator from neptune.types.mode import Mode from neptune.types.model_version_stage import ModelVersionStage +@safety_decorator class ModelVersion(MetadataContainer): """Class for managing a version of a neptune.ai model and retrieving information from it.""" diff --git a/src/neptune/metadata_containers/project.py b/src/neptune/metadata_containers/project.py index 055394921..c52ea1047 100644 --- a/src/neptune/metadata_containers/project.py +++ b/src/neptune/metadata_containers/project.py @@ -49,9 +49,11 @@ from neptune.metadata_containers import MetadataContainer from neptune.metadata_containers.abstract import NeptuneObjectCallback from neptune.metadata_containers.metadata_containers_table import Table +from neptune.metadata_containers.safe_container import safety_decorator from neptune.types.mode import Mode +@safety_decorator class Project(MetadataContainer): """Class for tracking and retrieving project-level metadata of a neptune.ai project.""" diff --git a/src/neptune/metadata_containers/run.py b/src/neptune/metadata_containers/run.py index 6033a3b24..dc1fa69d5 100644 --- a/src/neptune/metadata_containers/run.py +++ b/src/neptune/metadata_containers/run.py @@ -92,6 +92,7 @@ from neptune.internal.websockets.websocket_signals_background_job import WebsocketSignalsBackgroundJob from neptune.metadata_containers import MetadataContainer from neptune.metadata_containers.abstract import NeptuneObjectCallback +from neptune.metadata_containers.safe_container import safety_decorator from neptune.types import ( GitRef, StringSeries, @@ -100,6 +101,7 @@ from neptune.types.mode import Mode +@safety_decorator class Run(MetadataContainer): """Starts a tracked run that logs ML model-building metadata to neptune.ai.""" diff --git a/src/neptune/metadata_containers/safe_container.py b/src/neptune/metadata_containers/safe_container.py new file mode 100644 index 000000000..4df9734b8 --- /dev/null +++ b/src/neptune/metadata_containers/safe_container.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2022, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__all__ = ["safety_decorator"] + +import functools +import inspect +import os +from typing import Any + +from neptune.common.warnings import warn_once +from neptune.envs import NEPTUNE_SAFETY_MODE +from neptune.internal.utils.logger import logger + +_SAFETY_MODE = os.getenv(NEPTUNE_SAFETY_MODE, "false").lower() in ("true", "1", "t") + + +def _safe_function(func: Any) -> Any: + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + try: + return func(*args, **kwargs) + except Exception as ex: + try: + warn_once(f"Exception in method {func}: {ex.__class__.__name__}") + logger.debug("In safe mode exception is ignored", exc_info=True) + except Exception: + pass + + return wrapper + + +def safety_decorator(cls: Any) -> Any: + if _SAFETY_MODE: + for name, method in inspect.getmembers(cls): + if (not inspect.ismethod(method) and not inspect.isfunction(method)) or inspect.isbuiltin(method): + continue + setattr(cls, name, _safe_function(method)) + return cls + else: + return cls