From 7babc280a05588ea3cd3fbceb25c5eed3c5a27ed Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Mon, 20 Jun 2022 08:15:06 -0700 Subject: [PATCH] ensure that users can filter their notifications by: - failing attribute type (Critical vs All) - failure reason (Smart, Scrutiny, Both) fixes #300 --- example.scrutiny.yaml | 2 + webapp/backend/pkg/config/config.go | 3 + webapp/backend/pkg/constants.go | 7 + webapp/backend/pkg/notify/notify.go | 135 +++++++++++++-- webapp/backend/pkg/notify/notify_test.go | 161 ++++++++++++++++++ .../pkg/web/handler/send_test_notification.go | 15 +- .../pkg/web/handler/upload_device_metrics.go | 22 +-- 7 files changed, 311 insertions(+), 34 deletions(-) create mode 100644 webapp/backend/pkg/notify/notify_test.go diff --git a/example.scrutiny.yaml b/example.scrutiny.yaml index c93725e..b73b711 100644 --- a/example.scrutiny.yaml +++ b/example.scrutiny.yaml @@ -73,6 +73,8 @@ log: # - "join://shoutrrr:api-key@join/?devices=device1[,device2, ...][&icon=icon][&title=title]" # - "script:///file/path/on/disk" # - "https://www.example.com/path" +# filter_attributes: 'all' # options: 'all' or 'critical' +# level: 'fail' # options: 'fail', 'fail_scrutiny', 'fail_smart' ######################################################################################################################## # FEATURES COMING SOON diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go index dbff261..5f5d4cc 100644 --- a/webapp/backend/pkg/config/config.go +++ b/webapp/backend/pkg/config/config.go @@ -2,6 +2,7 @@ package config import ( "github.com/analogj/go-util/utils" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/errors" "github.com/spf13/viper" "log" @@ -38,6 +39,8 @@ func (c *configuration) Init() error { c.SetDefault("log.file", "") c.SetDefault("notify.urls", []string{}) + c.SetDefault("notify.filter_attributes", pkg.NotifyFilterAttributesAll) + c.SetDefault("notify.level", pkg.NotifyLevelFail) c.SetDefault("web.influxdb.scheme", "http") c.SetDefault("web.influxdb.host", "localhost") diff --git a/webapp/backend/pkg/constants.go b/webapp/backend/pkg/constants.go index fce37f8..0de6750 100644 --- a/webapp/backend/pkg/constants.go +++ b/webapp/backend/pkg/constants.go @@ -4,6 +4,13 @@ const DeviceProtocolAta = "ATA" const DeviceProtocolScsi = "SCSI" const DeviceProtocolNvme = "NVMe" +const NotifyFilterAttributesAll = "all" +const NotifyFilterAttributesCritical = "critical" + +const NotifyLevelFail = "fail" +const NotifyLevelFailScrutiny = "fail_scrutiny" +const NotifyLevelFailSmart = "fail_smart" + type AttributeStatus uint8 const ( diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index 5b81464..bfc6510 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -6,7 +6,11 @@ import ( "errors" "fmt" "github.com/analogj/go-util/utils" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/config" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" + "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" "github.com/containrrr/shoutrrr" shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types" "github.com/sirupsen/logrus" @@ -14,28 +18,130 @@ import ( "net/http" "net/url" "os" + "strconv" "strings" "time" ) const NotifyFailureTypeEmailTest = "EmailTest" -const NotifyFailureTypeSmartPrefail = "SmartPreFailure" +const NotifyFailureTypeBothFailure = "SmartFailure" //SmartFailure always takes precedence when Scrutiny & Smart failed. const NotifyFailureTypeSmartFailure = "SmartFailure" -const NotifyFailureTypeSmartErrorLog = "SmartErrorLog" -const NotifyFailureTypeSmartSelfTest = "SmartSelfTestLog" +const NotifyFailureTypeScrutinyFailure = "ScrutinyFailure" + +// ShouldNotify check if the error Message should be filtered (level mismatch or filtered_attributes) +func ShouldNotify(device models.Device, smartAttrs measurements.Smart, notifyLevel string, notifyFilterAttributes string) bool { + // 1. check if the device is healthy + if device.DeviceStatus == pkg.DeviceStatusPassed { + return false + } + + // setup constants for comparison + var requiredDeviceStatus pkg.DeviceStatus + var requiredAttrStatus pkg.AttributeStatus + if notifyLevel == pkg.NotifyLevelFail { + // either scrutiny or smart failures should trigger an email + requiredDeviceStatus = pkg.DeviceStatusSet(pkg.DeviceStatusFailedSmart, pkg.DeviceStatusFailedScrutiny) + requiredAttrStatus = pkg.AttributeStatusSet(pkg.AttributeStatusFailedSmart, pkg.AttributeStatusFailedScrutiny) + } else if notifyLevel == pkg.NotifyLevelFailSmart { + //only smart failures + requiredDeviceStatus = pkg.DeviceStatusFailedSmart + requiredAttrStatus = pkg.AttributeStatusFailedSmart + } else { + requiredDeviceStatus = pkg.DeviceStatusFailedScrutiny + requiredAttrStatus = pkg.AttributeStatusFailedScrutiny + } + + // 2. check if the attributes that are failing should be filtered (non-critical) + // 3. for any unfiltered attribute, store the failure reason (Smart or Scrutiny) + if notifyFilterAttributes == pkg.NotifyFilterAttributesCritical { + hasFailingCriticalAttr := false + var statusFailingCrtiticalAttr pkg.AttributeStatus + + for attrId, attrData := range smartAttrs.Attributes { + //find failing attribute + if attrData.GetStatus() == pkg.AttributeStatusPassed { + continue //skip all passing attributes + } + + // merge the status's of all critical attributes + statusFailingCrtiticalAttr = pkg.AttributeStatusSet(statusFailingCrtiticalAttr, attrData.GetStatus()) + + //found a failing attribute, see if its critical + if device.IsScsi() && thresholds.ScsiMetadata[attrId].Critical { + hasFailingCriticalAttr = true + } else if device.IsNvme() && thresholds.NmveMetadata[attrId].Critical { + hasFailingCriticalAttr = true + } else { + //this is ATA + attrIdInt, err := strconv.Atoi(attrId) + if err != nil { + continue + } + if thresholds.AtaMetadata[attrIdInt].Critical { + hasFailingCriticalAttr = true + } + } + + } + + if !hasFailingCriticalAttr { + //no critical attributes are failing, and notifyFilterAttributes == "critical" + return false + } else { + // check if any of the critical attributes have a status that we're looking for + return pkg.AttributeStatusHas(statusFailingCrtiticalAttr, requiredAttrStatus) + } + + } else { + // 2. SKIP - we are processing every attribute. + // 3. check if the device failure level matches the wanted failure level. + return pkg.DeviceStatusHas(device.DeviceStatus, requiredDeviceStatus) + } +} // TODO: include host and/or user label for device. type Payload struct { - Date string `json:"date"` //populated by Send function. - FailureType string `json:"failure_type"` //EmailTest, SmartFail, ScrutinyFail DeviceType string `json:"device_type"` //ATA/SCSI/NVMe DeviceName string `json:"device_name"` //dev/sda DeviceSerial string `json:"device_serial"` //WDDJ324KSO Test bool `json:"test"` // false - //should not be populated - Subject string `json:"subject"` - Message string `json:"message"` + //private, populated during init (marked as Public for JSON serialization) + Date string `json:"date"` //populated by Send function. + FailureType string `json:"failure_type"` //EmailTest, BothFail, SmartFail, ScrutinyFail + Subject string `json:"subject"` + Message string `json:"message"` +} + +func NewPayload(device models.Device, test bool) Payload { + payload := Payload{ + DeviceType: device.DeviceType, + DeviceName: device.DeviceName, + DeviceSerial: device.SerialNumber, + Test: test, + } + + //validate that the Payload is populated + sendDate := time.Now() + payload.Date = sendDate.Format(time.RFC3339) + payload.FailureType = payload.GenerateFailureType(device.DeviceStatus) + payload.Subject = payload.GenerateSubject() + payload.Message = payload.GenerateMessage() + return payload +} + +func (p *Payload) GenerateFailureType(deviceStatus pkg.DeviceStatus) string { + //generate a failure type, given Test and DeviceStatus + if p.Test { + return NotifyFailureTypeEmailTest // must be an email test if "Test" is true + } + if pkg.DeviceStatusHas(deviceStatus, pkg.DeviceStatusFailedSmart) && pkg.DeviceStatusHas(deviceStatus, pkg.DeviceStatusFailedScrutiny) { + return NotifyFailureTypeBothFailure //both failed + } else if pkg.DeviceStatusHas(deviceStatus, pkg.DeviceStatusFailedSmart) { + return NotifyFailureTypeSmartFailure //only SMART failed + } else { + return NotifyFailureTypeScrutinyFailure //only Scrutiny failed + } } func (p *Payload) GenerateSubject() string { @@ -61,6 +167,14 @@ Date: %s`, p.DeviceName, p.FailureType, p.DeviceName, p.DeviceSerial, p.DeviceTy return message } +func New(logger logrus.FieldLogger, appconfig config.Interface, device models.Device, test bool) Notify { + return Notify{ + Logger: logger, + Config: appconfig, + Payload: NewPayload(device, test), + } +} + type Notify struct { Logger logrus.FieldLogger Config config.Interface @@ -68,11 +182,6 @@ type Notify struct { } func (n *Notify) Send() error { - //validate that the Payload is populated - sendDate := time.Now() - n.Payload.Date = sendDate.Format(time.RFC3339) - n.Payload.Subject = n.Payload.GenerateSubject() - n.Payload.Message = n.Payload.GenerateMessage() //retrieve list of notification endpoints from config file configUrls := n.Config.GetStringSlice("notify.urls") diff --git a/webapp/backend/pkg/notify/notify_test.go b/webapp/backend/pkg/notify/notify_test.go new file mode 100644 index 0000000..aadb5f9 --- /dev/null +++ b/webapp/backend/pkg/notify/notify_test.go @@ -0,0 +1,161 @@ +package notify + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" + "github.com/stretchr/testify/require" + "testing" +) + +func TestShouldNotify_MustSkipPassingDevices(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusPassed, + } + smartAttrs := measurements.Smart{} + notifyLevel := pkg.NotifyLevelFail + notifyFilterAttributes := pkg.NotifyFilterAttributesAll + + //assert + require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) +} + +func TestShouldNotify_NotifyLevelFail_FailingSmartDevice(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedSmart, + } + smartAttrs := measurements.Smart{} + notifyLevel := pkg.NotifyLevelFail + notifyFilterAttributes := pkg.NotifyFilterAttributesAll + + //assert + require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) +} + +func TestShouldNotify_NotifyLevelFailSmart_FailingSmartDevice(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedSmart, + } + smartAttrs := measurements.Smart{} + notifyLevel := pkg.NotifyLevelFailSmart + notifyFilterAttributes := pkg.NotifyFilterAttributesAll + + //assert + require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) +} + +func TestShouldNotify_NotifyLevelFailScrutiny_FailingSmartDevice(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedSmart, + } + smartAttrs := measurements.Smart{} + notifyLevel := pkg.NotifyLevelFailScrutiny + notifyFilterAttributes := pkg.NotifyFilterAttributesAll + + //assert + require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) +} + +func TestShouldNotify_NotifyFilterAttributesCritical_WithCriticalAttrs(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedSmart, + } + smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{ + "5": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusFailedSmart, + }, + }} + notifyLevel := pkg.NotifyLevelFail + notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + + //assert + require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) +} + +func TestShouldNotify_NotifyFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedSmart, + } + smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{ + "5": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusPassed, + }, + "10": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusFailedScrutiny, + }, + }} + notifyLevel := pkg.NotifyLevelFail + notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + + //assert + require.True(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) +} + +func TestShouldNotify_NotifyFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedSmart, + } + smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{ + "1": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusFailedSmart, + }, + }} + notifyLevel := pkg.NotifyLevelFail + notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + + //assert + require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) +} + +func TestShouldNotify_NotifyFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedSmart, + } + smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{ + "5": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusPassed, + }, + }} + notifyLevel := pkg.NotifyLevelFail + notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + + //assert + require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) +} + +func TestShouldNotify_NotifyFilterAttributesCritical_NotifyLevelFailSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedSmart, + } + smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{ + "5": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusPassed, + }, + "10": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusFailedScrutiny, + }, + }} + notifyLevel := pkg.NotifyLevelFailSmart + notifyFilterAttributes := pkg.NotifyFilterAttributesCritical + + //assert + require.False(t, ShouldNotify(device, smartAttrs, notifyLevel, notifyFilterAttributes)) +} diff --git a/webapp/backend/pkg/web/handler/send_test_notification.go b/webapp/backend/pkg/web/handler/send_test_notification.go index 07a1d6c..dae55ab 100644 --- a/webapp/backend/pkg/web/handler/send_test_notification.go +++ b/webapp/backend/pkg/web/handler/send_test_notification.go @@ -15,17 +15,16 @@ func SendTestNotification(c *gin.Context) { appConfig := c.MustGet("CONFIG").(config.Interface) logger := c.MustGet("LOGGER").(logrus.FieldLogger) - testNotify := notify.Notify{ - Logger: logger, - Config: appConfig, - Payload: notify.Payload{ - FailureType: "EmailTest", - DeviceSerial: "FAKEWDDJ324KSO", + testNotify := notify.New( + logger, + appConfig, + models.Device{ + SerialNumber: "FAKEWDDJ324KSO", DeviceType: pkg.DeviceProtocolAta, DeviceName: "/dev/sda", - Test: true, }, - } + true, + ) err := testNotify.Send() if err != nil { logger.Errorln("An error occurred while sending test notification", err) diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index 7a971bd..e893366 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -63,20 +63,16 @@ func UploadDeviceMetrics(c *gin.Context) { } //check for error - if updatedDevice.DeviceStatus != pkg.DeviceStatusPassed { + if notify.ShouldNotify(updatedDevice, smartData, appConfig.GetString("notify.level"), appConfig.GetString("notify.filter_attributes")) { //send notifications - testNotify := notify.Notify{ - Config: appConfig, - Payload: notify.Payload{ - FailureType: notify.NotifyFailureTypeSmartFailure, - DeviceName: updatedDevice.DeviceName, - DeviceType: updatedDevice.DeviceProtocol, - DeviceSerial: updatedDevice.SerialNumber, - Test: false, - }, - Logger: logger, - } - _ = testNotify.Send() //we ignore error message when sending notifications. + + liveNotify := notify.New( + logger, + appConfig, + updatedDevice, + false, + ) + _ = liveNotify.Send() //we ignore error message when sending notifications. } c.JSON(http.StatusOK, gin.H{"success": true})