From 6bcfb4fdbf2c4f403f7326dfe3efea918abcba2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20=C5=81usakowski?= Date: Tue, 31 Mar 2020 16:46:18 +0200 Subject: [PATCH] Remove double warning about missing psutil on experiment start --- neptune/internal/abort.py | 51 +++++++------------ .../internal/execution/execution_context.py | 22 +++++--- .../hardware/system/system_monitor.py | 36 ++++++------- neptune/utils.py | 1 + 4 files changed, 48 insertions(+), 62 deletions(-) diff --git a/neptune/internal/abort.py b/neptune/internal/abort.py index e5628b616..2e50f1ef3 100644 --- a/neptune/internal/abort.py +++ b/neptune/internal/abort.py @@ -13,9 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import logging -_logger = logging.getLogger(__name__) +try: + import psutil + PSUTIL_INSTALLED = True +except ImportError: + PSUTIL_INSTALLED = False class CustomAbortImpl(object): @@ -34,56 +37,36 @@ def __init__(self, pid): @staticmethod def requirements_installed(): - # pylint:disable=unused-import,unused-variable,bad-option-value,import-outside-toplevel - try: - import psutil - return True - except ImportError: - _logger.warning('psutil is not installed. The experiment will not be abortable.') - return False + return PSUTIL_INSTALLED def abort(self): - # pylint:disable=bad-option-value,import-outside-toplevel - import psutil - - process = None try: - process = psutil.Process(self._pid) + processes = self._get_process_with_children(psutil.Process(self._pid)) except psutil.NoSuchProcess: - pass + processes = [] - if process is not None: - processes = self._get_processes(process) - for p in processes: - self._abort(p) - _, alive = psutil.wait_procs(processes, timeout=self.KILL_TIMEOUT) - for p in alive: - self._kill(p) + for p in processes: + self._abort(p) + _, alive = psutil.wait_procs(processes, timeout=self.KILL_TIMEOUT) + for p in alive: + self._kill(p) @staticmethod - def _get_processes(process): - # pylint:disable=bad-option-value,import-outside-toplevel - import psutil - + def _get_process_with_children(process): try: return [process] + process.children(recursive=True) except psutil.NoSuchProcess: return [] - def _abort(self, process): - # pylint:disable=bad-option-value,import-outside-toplevel - import psutil - + @staticmethod + def _abort(process): try: process.terminate() except psutil.NoSuchProcess: pass def _kill(self, process): - # pylint:disable=bad-option-value,import-outside-toplevel - import psutil - - for process in self._get_processes(process): + for process in self._get_process_with_children(process): try: if process.is_running(): process.kill() diff --git a/neptune/internal/execution/execution_context.py b/neptune/internal/execution/execution_context.py index c1a72c334..72453686e 100644 --- a/neptune/internal/execution/execution_context.py +++ b/neptune/internal/execution/execution_context.py @@ -17,6 +17,7 @@ import sys import time import traceback +import logging from logging import StreamHandler from neptune.internal.abort import DefaultAbortImpl, CustomAbortImpl @@ -33,6 +34,9 @@ from neptune.utils import is_notebook, in_docker +_logger = logging.getLogger(__name__) + + class ExecutionContext(object): def __init__(self, backend, experiment): @@ -61,11 +65,6 @@ def start(self, if handle_uncaught_exceptions: self._set_uncaught_exception_handler() - abortable = abort_callback is not None or DefaultAbortImpl.requirements_installed() - - if abortable: - self._run_aborting_thread(abort_callback) - if logger: # pylint: disable=protected-access channel = self._experiment._get_channel('logger', 'text', ChannelNamespace.SYSTEM) @@ -80,11 +79,20 @@ def start(self, if upload_stderr and not is_notebook(): self._stderr_uploader = StdErrWithUpload(self._experiment) + abortable = abort_callback is not None or DefaultAbortImpl.requirements_installed() + if abortable: + self._run_aborting_thread(abort_callback) + else: + _logger.warning('psutil is not installed. You will not be able to abort this experiment from the UI.') + if run_monitoring_thread: self._run_monitoring_thread() - if send_hardware_metrics and SystemMonitor.requirements_installed(): - self._run_hardware_metrics_reporting_thread() + if send_hardware_metrics: + if SystemMonitor.requirements_installed(): + self._run_hardware_metrics_reporting_thread() + else: + _logger.warning('psutil is not installed. Hardware metrics will not be collected.') def stop(self): if self._ping_thread: diff --git a/neptune/internal/hardware/system/system_monitor.py b/neptune/internal/hardware/system/system_monitor.py index a972b66dc..5c22a84d9 100644 --- a/neptune/internal/hardware/system/system_monitor.py +++ b/neptune/internal/hardware/system/system_monitor.py @@ -13,33 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import logging -_logger = logging.getLogger(__name__) +try: + import psutil + PSUTIL_INSTALLED = True +except ImportError: + PSUTIL_INSTALLED = False class SystemMonitor(object): @staticmethod def requirements_installed(): - # pylint:disable=unused-import,unused-variable,bad-option-value,import-outside-toplevel - try: - import psutil - return True - except ImportError: - _logger.warning('psutil is not installed. Hardware metrics will not be collected.') - return False + return PSUTIL_INSTALLED - def cpu_count(self): - return self._psutil().cpu_count() - - def cpu_percent(self): - return self._psutil().cpu_percent() + @staticmethod + def cpu_count(): + return psutil.cpu_count() - def virtual_memory(self): - return self._psutil().virtual_memory() + @staticmethod + def cpu_percent(): + return psutil.cpu_percent() - @classmethod - def _psutil(cls): - # pylint:disable=bad-option-value,import-outside-toplevel - import psutil - return psutil + @staticmethod + def virtual_memory(): + return psutil.virtual_memory() diff --git a/neptune/utils.py b/neptune/utils.py index ea315385b..16baabd2e 100644 --- a/neptune/utils.py +++ b/neptune/utils.py @@ -257,6 +257,7 @@ def glob(pathname): else: return globlib.glob(pathname, recursive=True) + def is_ipython(): try: # pylint:disable=bad-option-value,import-outside-toplevel