Print warning if gpu metrics cannot be reported (#219)

neptune/internal/hardware/gpu/gpu_monitor.py

@@ -14,11 +14,19 @@
 # limitations under the License.
 #
 
+import logging
+import time
+
 from py3nvml.py3nvml import NVMLError, nvmlDeviceGetCount, nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo, \
     nvmlDeviceGetUtilizationRates, nvmlInit
 
+_logger = logging.getLogger(__name__)
 
 class GPUMonitor(object):
+
+    nvml_error_time = 0
+    nvml_error_period = 30
+
     def get_card_count(self):
         return self.__nvml_get_or_else(nvmlDeviceGetCount, default=0)
 
@@ -51,5 +59,9 @@ def __nvml_get_or_else(self, getter, default=None):
         try:
             nvmlInit()
             return getter()
-        except NVMLError:
+        except NVMLError as e:
+            timestamp = time.time()
+            if timestamp - GPUMonitor.nvml_error_time > GPUMonitor.nvml_error_period:
+                _logger.warning("NVMLError: %s - GPU usage metrics may not be reported.", e)
+                GPUMonitor.nvml_error_time = timestamp
             return default

neptune/internal/hardware/metrics/reports/metric_reporter.py

@@ -29,15 +29,16 @@ def report(self, timestamp):
         return [
             MetricReport(
                 metric=metric,
-                values=[self.__metric_value_for_gauge(gauge, timestamp) for gauge in metric.gauges]
+                values=[x for x in [self.__metric_value_for_gauge(gauge, timestamp) for gauge in metric.gauges] if x]
             )
             for metric in self.__metrics
         ]
 
     def __metric_value_for_gauge(self, gauge, timestamp):
+        value = gauge.value()
         return MetricValue(
             timestamp=timestamp,
             running_time=timestamp - self.__reference_timestamp,
             gauge_name=gauge.name(),
-            value=gauge.value()
-        )
+            value=value
+        ) if value else None