From 6377a258f64f6a429de473371b82eeff1ae01310 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 1 Oct 2020 18:59:08 -0600 Subject: [PATCH] sending notifications on failure. --- example.scrutiny.yaml | 18 ++- webapp/backend/pkg/notify/notify.go | 146 ++++++++++++++---- .../pkg/web/handler/send_test_notification.go | 13 +- .../pkg/web/handler/upload_device_metrics.go | 19 +++ 4 files changed, 152 insertions(+), 44 deletions(-) diff --git a/example.scrutiny.yaml b/example.scrutiny.yaml index 8518d56..c2461eb 100644 --- a/example.scrutiny.yaml +++ b/example.scrutiny.yaml @@ -32,19 +32,12 @@ log: file: '' #absolute or relative paths allowed, eg. web.log level: INFO -# The following commented out sections are a preview of additional configuration options that will be available soon. - -#disks: -# include: -# # - /dev/sda -# exclude: -# # - /dev/sdb #notify: # urls: # - "discord://token@channel" # - "telegram://token@telegram?channels=channel-1[,channel-2,...]" -# - "pushover://shoutrrr:apiToken@userKey/?devices=device1[,device2, ...]" +# - "pushover://shoutrrr:apiToken@userKey/?priority=1&devices=device1[,device2, ...]" # - "slack://[botname@]token-a/token-b/token-c" # - "smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]" # - "teams://token-a/token-b/token-c" @@ -58,6 +51,15 @@ log: # - "script:///file/path/on/disk" # - "https://www.example.com/path" + +# The following commented out sections are a preview of additional configuration options that will be available soon. + +#disks: +# include: +# # - /dev/sda +# exclude: +# # - /dev/sdb + #limits: # ata: # critical: diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index e2c4876..01b5306 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -7,26 +7,57 @@ import ( "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/containrrr/shoutrrr" - log "github.com/sirupsen/logrus" + shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types" + "github.com/sirupsen/logrus" "net/http" + "net/url" "os" "strings" "sync" "time" ) +const NotifyFailureTypeEmailTest = "EmailTest" +const NotifyFailureTypeSmartPrefail = "SmartPreFailure" +const NotifyFailureTypeSmartFailure = "SmartFailure" +const NotifyFailureTypeSmartErrorLog = "SmartErrorLog" +const NotifyFailureTypeSmartSelfTest = "SmartSelfTestLog" + +// TODO: include host and/or user label for device. type Payload struct { - Mailer string `json:"mailer"` - Subject string `json:"subject"` - Date string `json:"date"` - FailureType string `json:"failure_type"` - Device string `json:"device"` - DeviceType string `json:"device_type"` - DeviceString string `json:"device_string"` - Message string `json:"message"` + Date string `json:"date"` //populated by Send function. + FailureType string `json:"failure_type"` //EmailTest, SmartFail, ScrutinyFail + DeviceType string `json:"device_type"` //ATA/SCSI/NVMe + DeviceName string `json:"device_string"` //dev/sda + DeviceSerial string `json:"device"` //WDDJ324KSO + Test bool `json:"-"` // false +} + +func (p *Payload) GenerateMessage() string { + //generate a detailed failure message + return fmt.Sprintf("Scrutiny SMART error (%s) detected on device: %s", p.FailureType, p.DeviceName) +} + +func (p *Payload) GenerateSubject() string { + //generate a detailed failure message + message := fmt.Sprintf( + `Scrutiny SMART error notification for device: %s +Failure Type: %s +Device Name: %s +Device Serial: %s +Device Type: %s + +Date: %s`, p.DeviceName, p.FailureType, p.DeviceName, p.DeviceSerial, p.DeviceType, p.Date) + + if p.Test { + message = "TEST NOTIFICATION:\n" + message + } + + return message } type Notify struct { + Logger logrus.FieldLogger Config config.Interface Payload Payload } @@ -38,6 +69,7 @@ func (n *Notify) Send() error { //retrieve list of notification endpoints from config file configUrls := n.Config.GetStringSlice("notify.urls") + n.Logger.Debugf("Configured notification services: %v", configUrls) //remove http:// https:// and script:// prefixed urls notifyWebhooks := []string{} @@ -54,6 +86,10 @@ func (n *Notify) Send() error { } } + n.Logger.Debugf("Configured scripts: %v", notifyScripts) + n.Logger.Debugf("Configured webhooks: %v", notifyWebhooks) + n.Logger.Debugf("Configured shoutrrr: %v", notifyShoutrrr) + //run all scripts, webhooks and shoutrr commands in parallel var wg sync.WaitGroup @@ -67,12 +103,14 @@ func (n *Notify) Send() error { wg.Add(1) go n.SendScriptNotification(&wg, notifyScript) } - if len(notifyScripts) > 0 { + for _, shoutrrrUrl := range notifyShoutrrr { wg.Add(1) - go n.SendShoutrrrNotification(&wg, notifyShoutrrr) + go n.SendShoutrrrNotification(&wg, shoutrrrUrl) } //and wait for completion, error or timeout. + n.Logger.Debugf("Main: waiting for notifications to complete.") + //wg.Wait() if waitTimeout(&wg, time.Minute) { //wait for 1 minute fmt.Println("Timed out while sending notifications") } else { @@ -83,16 +121,16 @@ func (n *Notify) Send() error { func (n *Notify) SendWebhookNotification(wg *sync.WaitGroup, webhookUrl string) { defer wg.Done() - log.Infof("Sending Webhook to %s", webhookUrl) + n.Logger.Infof("Sending Webhook to %s", webhookUrl) requestBody, err := json.Marshal(n.Payload) if err != nil { - log.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) + n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) return } resp, err := http.Post(webhookUrl, "application/json", bytes.NewBuffer(requestBody)) if err != nil { - log.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) + n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) return } defer resp.Body.Close() @@ -104,45 +142,97 @@ func (n *Notify) SendScriptNotification(wg *sync.WaitGroup, scriptUrl string) { //check if the script exists. scriptPath := strings.TrimPrefix(scriptUrl, "script://") - log.Infof("Executing Script %s", scriptPath) + n.Logger.Infof("Executing Script %s", scriptPath) if !utils.FileExists(scriptPath) { - log.Errorf("Script does not exist: %s", scriptPath) + n.Logger.Errorf("Script does not exist: %s", scriptPath) return } copyEnv := os.Environ() - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MAILER=%s", n.Payload.Mailer)) - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.Subject)) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.GenerateSubject())) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DATE=%s", n.Payload.Date)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_FAILURE_TYPE=%s", n.Payload.FailureType)) - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE=%s", n.Payload.Device)) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_NAME=%s", n.Payload.DeviceName)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_TYPE=%s", n.Payload.DeviceType)) - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_STRING=%s", n.Payload.DeviceString)) - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.Message)) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_SERIAL=%s", n.Payload.DeviceSerial)) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.GenerateMessage())) err := utils.CmdExec(scriptPath, []string{}, "", copyEnv, "") if err != nil { - log.Errorf("An error occurred while executing script %s: %v", scriptPath, err) + n.Logger.Errorf("An error occurred while executing script %s: %v", scriptPath, err) } return } -func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrls []string) { - log.Infof("Sending notifications to %v", shoutrrrUrls) +func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrl string) { + + fmt.Printf("Sending Notifications to %v", shoutrrrUrl) + n.Logger.Infof("Sending notifications to %v", shoutrrrUrl) defer wg.Done() - sender, err := shoutrrr.CreateSender(shoutrrrUrls...) + sender, err := shoutrrr.CreateSender(shoutrrrUrl) if err != nil { - log.Errorf("An error occurred while sending notifications %v: %v", shoutrrrUrls, err) + n.Logger.Errorf("An error occurred while sending notifications %v: %v", shoutrrrUrl, err) return } - errs := sender.Send(n.Payload.Subject, nil) //structs.Map(n.Payload).()) + //sender.SetLogger(n.Logger.) + serviceName, params, err := n.GenShoutrrrNotificationParams(shoutrrrUrl) + n.Logger.Debug("notification data for %s: (%s)\n%v", serviceName, shoutrrrUrl, params) + + if err != nil { + n.Logger.Errorf("An error occurred occurred while generating notification payload for %s:\n %v", serviceName, shoutrrrUrl, err) + } + + errs := sender.Send(n.Payload.GenerateMessage(), params) if len(errs) > 0 { - log.Errorf("One or more errors occurred occurred while sending notifications %v:\n %v", shoutrrrUrls, errs) + n.Logger.Errorf("One or more errors occurred occurred while sending notifications for %s:\n %v", shoutrrrUrl, errs) + for _, err := range errs { + n.Logger.Error(err) + } } } +func (n *Notify) GenShoutrrrNotificationParams(shoutrrrUrl string) (string, *shoutrrrTypes.Params, error) { + serviceURL, err := url.Parse(shoutrrrUrl) + if err != nil { + return "", nil, err + } + + serviceName := serviceURL.Scheme + params := &shoutrrrTypes.Params{} + + logoUrl := "https://raw.githubusercontent.com/AnalogJ/scrutiny/master/webapp/frontend/src/ms-icon-144x144.png" + subject := n.Payload.GenerateSubject() + switch serviceName { + // no params supported for these services + case "discord", "hangouts", "ifttt", "mattermost", "teams": + break + case "gotify": + (*params)["title"] = subject + case "join": + (*params)["title"] = subject + (*params)["icon"] = logoUrl + case "pushbullet": + (*params)["title"] = subject + case "pushover": + (*params)["subject"] = subject + case "slack": + (*params)["title"] = subject + (*params)["thumb_url"] = logoUrl + case "smtp": + (*params)["subject"] = subject + case "standard": + (*params)["subject"] = subject + case "telegram": + (*params)["subject"] = subject + case "zulip": + (*params)["topic"] = subject + } + + return serviceName, params, nil +} + //utility functions // waitTimeout waits for the waitgroup for the specified max timeout. // Returns true if waiting timed out. diff --git a/webapp/backend/pkg/web/handler/send_test_notification.go b/webapp/backend/pkg/web/handler/send_test_notification.go index 1091f72..153c013 100644 --- a/webapp/backend/pkg/web/handler/send_test_notification.go +++ b/webapp/backend/pkg/web/handler/send_test_notification.go @@ -1,14 +1,12 @@ package handler import ( - "fmt" "github.com/analogj/scrutiny/webapp/backend/pkg/config" dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" "github.com/analogj/scrutiny/webapp/backend/pkg/notify" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "net/http" - "os" ) // Send test notification @@ -17,15 +15,14 @@ func SendTestNotification(c *gin.Context) { logger := c.MustGet("LOGGER").(logrus.FieldLogger) testNotify := notify.Notify{ + Logger: logger, Config: appConfig, Payload: notify.Payload{ - Mailer: os.Args[0], - Subject: fmt.Sprintf("Scrutiny SMART error (EmailTest) detected on disk: XXXXX"), FailureType: "EmailTest", - Device: "/dev/sda", - DeviceType: "ata", - DeviceString: "/dev/sda", - Message: "TEST EMAIL from smartd for device: /dev/sda", + DeviceSerial: "FAKEWDDJ324KSO", + DeviceType: dbModels.DeviceProtocolAta, + DeviceName: "/dev/sda", + Test: true, }, } err := testNotify.Send() diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index f3bb442..d94bd05 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -1,8 +1,10 @@ package handler import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" + "github.com/analogj/scrutiny/webapp/backend/pkg/notify" "github.com/gin-gonic/gin" "github.com/jinzhu/gorm" "github.com/sirupsen/logrus" @@ -12,6 +14,7 @@ import ( func UploadDeviceMetrics(c *gin.Context) { db := c.MustGet("DB").(*gorm.DB) logger := c.MustGet("LOGGER").(logrus.FieldLogger) + appConfig := c.MustGet("CONFIG").(config.Interface) var collectorSmartData collector.SmartInfo err := c.BindJSON(&collectorSmartData) @@ -45,5 +48,21 @@ func UploadDeviceMetrics(c *gin.Context) { return } + //check for error + if deviceSmartData.SmartStatus == dbModels.SmartStatusFailed { + //send notifications + testNotify := notify.Notify{ + Config: appConfig, + Payload: notify.Payload{ + FailureType: notify.NotifyFailureTypeSmartFailure, + DeviceName: device.DeviceName, + DeviceType: device.DeviceProtocol, + DeviceSerial: device.SerialNumber, + Test: false, + }, + } + _ = testNotify.Send() //we ignore error message when sending notifications. + } + c.JSON(http.StatusOK, gin.H{"success": true}) }