Monitor sensor value out of critical thresholds

Summary:
Only publishing sensors violation of known critical thresholds.
Otherwise, we would publish them with value 0 (i.e. not violating) when in fact they're violating.

Reviewed By: somasun

Differential Revision: D66833343

fbshipit-source-id: 6167014c863c488f204829e8c2d4e89f32987678

fboss/platform/sensor_service/SensorServiceImpl.cpp

@@ -30,6 +30,30 @@ DEFINE_int32(
     "Interval at which stats subscriptions are served");
 
 namespace facebook::fboss::platform::sensor_service {
+namespace {
+void monitorSensorValue(const SensorData& sensorData) {
+  // Don't monitor if thresholds aren't defined to prevent false data.
+  if (!sensorData.thresholds()->upperCriticalVal() &&
+      !sensorData.thresholds()->lowerCriticalVal()) {
+    return;
+  }
+  // Skip reporting if there's no sensor value.
+  if (!sensorData.value()) {
+    return;
+  }
+  // At least one of upperCriticalVal or lowerCriticalVal exist.
+  bool thresholdViolation = *sensorData.value() >
+          sensorData.thresholds()->upperCriticalVal().value_or(INT_MAX) ||
+      *sensorData.value() <
+          sensorData.thresholds()->lowerCriticalVal().value_or(INT_MIN);
+  fb303::fbData->setCounter(
+      fmt::format(
+          SensorServiceImpl::kCriticalThresholdViolation,
+          *sensorData.name(),
+          apache::thrift::util::enumNameSafe(*sensorData.sensorType())),
+      thresholdViolation);
+}
+} // namespace
 
 SensorServiceImpl::SensorServiceImpl(const SensorConfig& sensorConfig)
     : sensorConfig_(sensorConfig) {
@@ -168,6 +192,7 @@ SensorData SensorServiceImpl::fetchSensorDataImpl(
   }
   sensorData.thresholds() = thresholds ? *thresholds : Thresholds();
   sensorData.sensorType() = sensorType;
+  monitorSensorValue(sensorData);
   return sensorData;
 }
 

fboss/platform/sensor_service/SensorServiceImpl.h

@@ -33,6 +33,8 @@ class SensorServiceImpl {
   auto static constexpr kReadTotal = "sensor_read.total";
   auto static constexpr kTotalReadFailure = "sensor_read.total.failures";
   auto static constexpr kHasReadFailure = "sensor_read.has.failures";
+  auto static constexpr kCriticalThresholdViolation =
+      "sensor_read.sensor_{}.type_{}.critical_threshold_violation";
 
   explicit SensorServiceImpl(const SensorConfig& sensorConfig);
   ~SensorServiceImpl();