Support transient SMART failures

As discussed in [1] some SMART errors are transient and should not be treated as permanent. This commit adds support for a configurable list of ATA SMART attribute IDs for which failures will be treated as transient. Drive health history is still recorded and notifications are sent, but the device itself is not marked as failed. Fixes AnalogJ#374. [1] AnalogJ#374
dcelasun · Sep 24, 2022 · 81920c5 · 81920c5
1 parent 578a262
commit 81920c5
Show file tree

Hide file tree

Showing 12 changed files with 152 additions and 46 deletions.
diff --git a/collector/pkg/config/interface.go b/collector/pkg/config/interface.go
@@ -20,6 +20,7 @@ type Interface interface {
 	GetInt(key string) int
 	GetString(key string) string
 	GetStringSlice(key string) []string
+	GetIntSlice(key string) []int
 	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
 
 	GetDeviceOverrides() []models.ScanOverride

diff --git a/collector/pkg/config/mock/mock_config.go b/collector/pkg/config/mock/mock_config.go
diff --git a/example.scrutiny.yaml b/example.scrutiny.yaml
@@ -52,6 +52,10 @@ log:
   file: '' #absolute or relative paths allowed, eg. web.log
   level: INFO
 
+failures:
+  transient:
+    ata:
+      - 195 # Hardware_ECC_Recovered, see https://superuser.com/a/1511916/169872
 
 # Notification "urls" look like the following. For more information about service specific configuration see
 # Shoutrrr's documentation: https://containrrr.dev/shoutrrr/services/overview/

diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go
@@ -1,12 +1,13 @@
 package config
 
 import (
-	"github.com/analogj/go-util/utils"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
-	"github.com/spf13/viper"
 	"log"
 	"os"
 	"strings"
+
+	"github.com/analogj/go-util/utils"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
+	"github.com/spf13/viper"
 )
 
 const DB_USER_SETTINGS_SUBKEY = "user"
@@ -51,6 +52,8 @@ func (c *configuration) Init() error {
 	c.SetDefault("web.influxdb.token", "scrutiny-default-admin-token")
 	c.SetDefault("web.influxdb.retention_policy", true)
 
+	c.SetDefault("failures.transient.ata", []int{195})
+
 	//c.SetDefault("disks.include", []string{})
 	//c.SetDefault("disks.exclude", []string{})
 

diff --git a/webapp/backend/pkg/config/interface.go b/webapp/backend/pkg/config/interface.go
@@ -25,5 +25,6 @@ type Interface interface {
 	GetInt64(key string) int64
 	GetString(key string) string
 	GetStringSlice(key string) []string
+	GetIntSlice(key string) []int
 	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
 }
diff --git a/webapp/backend/pkg/config/mock/mock_config.go b/webapp/backend/pkg/config/mock/mock_config.go
diff --git a/webapp/backend/pkg/constants.go b/webapp/backend/pkg/constants.go
@@ -4,8 +4,9 @@ const DeviceProtocolAta = "ATA"
 const DeviceProtocolScsi = "SCSI"
 const DeviceProtocolNvme = "NVMe"
 
-//go:generate stringer -type=AttributeStatus
 // AttributeStatus bitwise flag, 1,2,4,8,16,32,etc
+//
+//go:generate stringer -type=AttributeStatus
 type AttributeStatus uint8
 
 const (
@@ -23,8 +24,9 @@ func AttributeStatusClear(b, flag AttributeStatus) AttributeStatus  { return b &
 func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ flag }
 func AttributeStatusHas(b, flag AttributeStatus) bool               { return b&flag != 0 }
 
-//go:generate stringer -type=DeviceStatus
 // DeviceStatus bitwise flag, 1,2,4,8,16,32,etc
+//
+//go:generate stringer -type=DeviceStatus
 type DeviceStatus uint8
 
 const (

diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go
@@ -3,21 +3,22 @@ package database
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
 	"github.com/influxdata/influxdb-client-go/v2/api"
 	log "github.com/sirupsen/logrus"
-	"strings"
-	"time"
 )
 
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // SMART
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
 	deviceSmartData := measurements.Smart{}
-	err := deviceSmartData.FromCollectorSmartInfo(wwn, collectorSmartData)
+	err := deviceSmartData.FromCollectorSmartInfo(sr.appConfig, wwn, collectorSmartData)
 	if err != nil {
 		sr.logger.Errorln("Could not process SMART metrics", err)
 		return measurements.Smart{}, err

diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go
@@ -4,6 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strconv"
+	"time"
+
 	"github.com/analogj/scrutiny/webapp/backend/pkg"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
@@ -17,8 +20,6 @@ import (
 	"github.com/influxdata/influxdb-client-go/v2/api/http"
 	log "github.com/sirupsen/logrus"
 	"gorm.io/gorm"
-	"strconv"
-	"time"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -384,8 +385,8 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 
 // helpers
 
-//When adding data to influxdb, an error may be returned if the data point is outside the range of the retention policy.
-//This function will ignore retention policy errors, and allow the migration to continue.
+// When adding data to influxdb, an error may be returned if the data point is outside the range of the retention policy.
+// This function will ignore retention policy errors, and allow the migration to continue.
 func ignorePastRetentionPolicyError(err error) error {
 	var influxDbWriteError *http.Error
 	if errors.As(err, &influxDbWriteError) {
@@ -468,7 +469,7 @@ func m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(d
 			})
 		}
 
-		postDeviceSmartData.ProcessAtaSmartInfo(preAtaSmartAttributesTable)
+		postDeviceSmartData.ProcessAtaSmartInfo(nil, preAtaSmartAttributesTable)
 
 	} else if preDevice.IsNvme() {
 		//info collector.SmartInfo

diff --git a/webapp/backend/pkg/models/measurements/smart.go b/webapp/backend/pkg/models/measurements/smart.go
@@ -2,13 +2,15 @@ package measurements
 
 import (
 	"fmt"
-	"github.com/analogj/scrutiny/webapp/backend/pkg"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
-	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 	"log"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/analogj/scrutiny/webapp/backend/pkg"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
+	"github.com/analogj/scrutiny/webapp/backend/pkg/thresholds"
 )
 
 type Smart struct {
@@ -100,8 +102,8 @@ func NewSmartFromInfluxDB(attrs map[string]interface{}) (*Smart, error) {
 	return &sm, nil
 }
 
-//Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries)
-func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error {
+// Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries)
+func (sm *Smart) FromCollectorSmartInfo(cfg config.Interface, wwn string, info collector.SmartInfo) error {
 	sm.DeviceWWN = wwn
 	sm.Date = time.Unix(info.LocalTime.TimeT, 0)
 
@@ -117,7 +119,7 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er
 	// process ATA/NVME/SCSI protocol data
 	sm.Attributes = map[string]SmartAttribute{}
 	if sm.DeviceProtocol == pkg.DeviceProtocolAta {
-		sm.ProcessAtaSmartInfo(info.AtaSmartAttributes.Table)
+		sm.ProcessAtaSmartInfo(cfg, info.AtaSmartAttributes.Table)
 	} else if sm.DeviceProtocol == pkg.DeviceProtocolNvme {
 		sm.ProcessNvmeSmartInfo(info.NvmeSmartHealthInformationLog)
 	} else if sm.DeviceProtocol == pkg.DeviceProtocolScsi {
@@ -127,8 +129,8 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er
 	return nil
 }
 
-//generate SmartAtaAttribute entries from Scrutiny Collector Smart data.
-func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTableItem) {
+// generate SmartAtaAttribute entries from Scrutiny Collector Smart data.
+func (sm *Smart) ProcessAtaSmartInfo(cfg config.Interface, tableItems []collector.AtaSmartAttributesTableItem) {
 	for _, collectorAttr := range tableItems {
 		attrModel := SmartAtaAttribute{
 			AttributeId: collectorAttr.ID,
@@ -149,13 +151,27 @@ func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTa
 		attrModel.PopulateAttributeStatus()
 		sm.Attributes[strconv.Itoa(collectorAttr.ID)] = &attrModel
 
+		var transient bool
+
+		if cfg != nil {
+			transients := cfg.GetIntSlice("failures.transient.ata")
+			for i := range transients {
+				if collectorAttr.ID == transients[i] {
+					transient = true
+					break
+				}
+			}
+		}
+
 		if pkg.AttributeStatusHas(attrModel.Status, pkg.AttributeStatusFailedScrutiny) {
-			sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
+			if !transient {
+				sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny)
+			}
 		}
 	}
 }
 
-//generate SmartNvmeAttribute entries from Scrutiny Collector Smart data.
+// generate SmartNvmeAttribute entries from Scrutiny Collector Smart data.
 func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.NvmeSmartHealthInformationLog) {
 
 	sm.Attributes = map[string]SmartAttribute{
@@ -185,7 +201,7 @@ func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.Nv
 	}
 }
 
-//generate SmartScsiAttribute entries from Scrutiny Collector Smart data.
+// generate SmartScsiAttribute entries from Scrutiny Collector Smart data.
 func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog collector.ScsiErrorCounterLog) {
 	sm.Attributes = map[string]SmartAttribute{
 		"scsi_grown_defect_list":                     (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: defectGrownList, Threshold: 0}).PopulateAttributeStatus(),