From 1246f5bba99a6da2ad75126b71ae81018f030ec3 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 30 Sep 2020 08:50:10 -0600 Subject: [PATCH 01/10] started working on notifications. --- webapp/backend/pkg/models/db/smart.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/webapp/backend/pkg/models/db/smart.go b/webapp/backend/pkg/models/db/smart.go index 889b6d1..7e11a45 100644 --- a/webapp/backend/pkg/models/db/smart.go +++ b/webapp/backend/pkg/models/db/smart.go @@ -10,6 +10,9 @@ import ( const SmartWhenFailedFailingNow = "FAILING_NOW" const SmartWhenFailedInThePast = "IN_THE_PAST" +const SmartStatusPassed = "passed" +const SmartStatusFailed = "failed" + type Smart struct { gorm.Model @@ -17,7 +20,7 @@ type Smart struct { Device Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key TestDate time.Time `json:"date"` - SmartStatus string `json:"smart_status"` + SmartStatus string `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed //Metrics Temp int64 `json:"temp"` @@ -49,9 +52,9 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er } if info.SmartStatus.Passed { - sm.SmartStatus = "passed" + sm.SmartStatus = SmartStatusPassed } else { - sm.SmartStatus = "failed" + sm.SmartStatus = SmartStatusFailed } return nil } From 6377a258f64f6a429de473371b82eeff1ae01310 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Thu, 1 Oct 2020 18:59:08 -0600 Subject: [PATCH 02/10] sending notifications on failure. --- example.scrutiny.yaml | 18 ++- webapp/backend/pkg/notify/notify.go | 146 ++++++++++++++---- .../pkg/web/handler/send_test_notification.go | 13 +- .../pkg/web/handler/upload_device_metrics.go | 19 +++ 4 files changed, 152 insertions(+), 44 deletions(-) diff --git a/example.scrutiny.yaml b/example.scrutiny.yaml index 8518d56..c2461eb 100644 --- a/example.scrutiny.yaml +++ b/example.scrutiny.yaml @@ -32,19 +32,12 @@ log: file: '' #absolute or relative paths allowed, eg. web.log level: INFO -# The following commented out sections are a preview of additional configuration options that will be available soon. - -#disks: -# include: -# # - /dev/sda -# exclude: -# # - /dev/sdb #notify: # urls: # - "discord://token@channel" # - "telegram://token@telegram?channels=channel-1[,channel-2,...]" -# - "pushover://shoutrrr:apiToken@userKey/?devices=device1[,device2, ...]" +# - "pushover://shoutrrr:apiToken@userKey/?priority=1&devices=device1[,device2, ...]" # - "slack://[botname@]token-a/token-b/token-c" # - "smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]" # - "teams://token-a/token-b/token-c" @@ -58,6 +51,15 @@ log: # - "script:///file/path/on/disk" # - "https://www.example.com/path" + +# The following commented out sections are a preview of additional configuration options that will be available soon. + +#disks: +# include: +# # - /dev/sda +# exclude: +# # - /dev/sdb + #limits: # ata: # critical: diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index e2c4876..01b5306 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -7,26 +7,57 @@ import ( "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/containrrr/shoutrrr" - log "github.com/sirupsen/logrus" + shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types" + "github.com/sirupsen/logrus" "net/http" + "net/url" "os" "strings" "sync" "time" ) +const NotifyFailureTypeEmailTest = "EmailTest" +const NotifyFailureTypeSmartPrefail = "SmartPreFailure" +const NotifyFailureTypeSmartFailure = "SmartFailure" +const NotifyFailureTypeSmartErrorLog = "SmartErrorLog" +const NotifyFailureTypeSmartSelfTest = "SmartSelfTestLog" + +// TODO: include host and/or user label for device. type Payload struct { - Mailer string `json:"mailer"` - Subject string `json:"subject"` - Date string `json:"date"` - FailureType string `json:"failure_type"` - Device string `json:"device"` - DeviceType string `json:"device_type"` - DeviceString string `json:"device_string"` - Message string `json:"message"` + Date string `json:"date"` //populated by Send function. + FailureType string `json:"failure_type"` //EmailTest, SmartFail, ScrutinyFail + DeviceType string `json:"device_type"` //ATA/SCSI/NVMe + DeviceName string `json:"device_string"` //dev/sda + DeviceSerial string `json:"device"` //WDDJ324KSO + Test bool `json:"-"` // false +} + +func (p *Payload) GenerateMessage() string { + //generate a detailed failure message + return fmt.Sprintf("Scrutiny SMART error (%s) detected on device: %s", p.FailureType, p.DeviceName) +} + +func (p *Payload) GenerateSubject() string { + //generate a detailed failure message + message := fmt.Sprintf( + `Scrutiny SMART error notification for device: %s +Failure Type: %s +Device Name: %s +Device Serial: %s +Device Type: %s + +Date: %s`, p.DeviceName, p.FailureType, p.DeviceName, p.DeviceSerial, p.DeviceType, p.Date) + + if p.Test { + message = "TEST NOTIFICATION:\n" + message + } + + return message } type Notify struct { + Logger logrus.FieldLogger Config config.Interface Payload Payload } @@ -38,6 +69,7 @@ func (n *Notify) Send() error { //retrieve list of notification endpoints from config file configUrls := n.Config.GetStringSlice("notify.urls") + n.Logger.Debugf("Configured notification services: %v", configUrls) //remove http:// https:// and script:// prefixed urls notifyWebhooks := []string{} @@ -54,6 +86,10 @@ func (n *Notify) Send() error { } } + n.Logger.Debugf("Configured scripts: %v", notifyScripts) + n.Logger.Debugf("Configured webhooks: %v", notifyWebhooks) + n.Logger.Debugf("Configured shoutrrr: %v", notifyShoutrrr) + //run all scripts, webhooks and shoutrr commands in parallel var wg sync.WaitGroup @@ -67,12 +103,14 @@ func (n *Notify) Send() error { wg.Add(1) go n.SendScriptNotification(&wg, notifyScript) } - if len(notifyScripts) > 0 { + for _, shoutrrrUrl := range notifyShoutrrr { wg.Add(1) - go n.SendShoutrrrNotification(&wg, notifyShoutrrr) + go n.SendShoutrrrNotification(&wg, shoutrrrUrl) } //and wait for completion, error or timeout. + n.Logger.Debugf("Main: waiting for notifications to complete.") + //wg.Wait() if waitTimeout(&wg, time.Minute) { //wait for 1 minute fmt.Println("Timed out while sending notifications") } else { @@ -83,16 +121,16 @@ func (n *Notify) Send() error { func (n *Notify) SendWebhookNotification(wg *sync.WaitGroup, webhookUrl string) { defer wg.Done() - log.Infof("Sending Webhook to %s", webhookUrl) + n.Logger.Infof("Sending Webhook to %s", webhookUrl) requestBody, err := json.Marshal(n.Payload) if err != nil { - log.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) + n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) return } resp, err := http.Post(webhookUrl, "application/json", bytes.NewBuffer(requestBody)) if err != nil { - log.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) + n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) return } defer resp.Body.Close() @@ -104,45 +142,97 @@ func (n *Notify) SendScriptNotification(wg *sync.WaitGroup, scriptUrl string) { //check if the script exists. scriptPath := strings.TrimPrefix(scriptUrl, "script://") - log.Infof("Executing Script %s", scriptPath) + n.Logger.Infof("Executing Script %s", scriptPath) if !utils.FileExists(scriptPath) { - log.Errorf("Script does not exist: %s", scriptPath) + n.Logger.Errorf("Script does not exist: %s", scriptPath) return } copyEnv := os.Environ() - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MAILER=%s", n.Payload.Mailer)) - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.Subject)) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.GenerateSubject())) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DATE=%s", n.Payload.Date)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_FAILURE_TYPE=%s", n.Payload.FailureType)) - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE=%s", n.Payload.Device)) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_NAME=%s", n.Payload.DeviceName)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_TYPE=%s", n.Payload.DeviceType)) - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_STRING=%s", n.Payload.DeviceString)) - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.Message)) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_SERIAL=%s", n.Payload.DeviceSerial)) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.GenerateMessage())) err := utils.CmdExec(scriptPath, []string{}, "", copyEnv, "") if err != nil { - log.Errorf("An error occurred while executing script %s: %v", scriptPath, err) + n.Logger.Errorf("An error occurred while executing script %s: %v", scriptPath, err) } return } -func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrls []string) { - log.Infof("Sending notifications to %v", shoutrrrUrls) +func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrl string) { + + fmt.Printf("Sending Notifications to %v", shoutrrrUrl) + n.Logger.Infof("Sending notifications to %v", shoutrrrUrl) defer wg.Done() - sender, err := shoutrrr.CreateSender(shoutrrrUrls...) + sender, err := shoutrrr.CreateSender(shoutrrrUrl) if err != nil { - log.Errorf("An error occurred while sending notifications %v: %v", shoutrrrUrls, err) + n.Logger.Errorf("An error occurred while sending notifications %v: %v", shoutrrrUrl, err) return } - errs := sender.Send(n.Payload.Subject, nil) //structs.Map(n.Payload).()) + //sender.SetLogger(n.Logger.) + serviceName, params, err := n.GenShoutrrrNotificationParams(shoutrrrUrl) + n.Logger.Debug("notification data for %s: (%s)\n%v", serviceName, shoutrrrUrl, params) + + if err != nil { + n.Logger.Errorf("An error occurred occurred while generating notification payload for %s:\n %v", serviceName, shoutrrrUrl, err) + } + + errs := sender.Send(n.Payload.GenerateMessage(), params) if len(errs) > 0 { - log.Errorf("One or more errors occurred occurred while sending notifications %v:\n %v", shoutrrrUrls, errs) + n.Logger.Errorf("One or more errors occurred occurred while sending notifications for %s:\n %v", shoutrrrUrl, errs) + for _, err := range errs { + n.Logger.Error(err) + } } } +func (n *Notify) GenShoutrrrNotificationParams(shoutrrrUrl string) (string, *shoutrrrTypes.Params, error) { + serviceURL, err := url.Parse(shoutrrrUrl) + if err != nil { + return "", nil, err + } + + serviceName := serviceURL.Scheme + params := &shoutrrrTypes.Params{} + + logoUrl := "https://raw.githubusercontent.com/AnalogJ/scrutiny/master/webapp/frontend/src/ms-icon-144x144.png" + subject := n.Payload.GenerateSubject() + switch serviceName { + // no params supported for these services + case "discord", "hangouts", "ifttt", "mattermost", "teams": + break + case "gotify": + (*params)["title"] = subject + case "join": + (*params)["title"] = subject + (*params)["icon"] = logoUrl + case "pushbullet": + (*params)["title"] = subject + case "pushover": + (*params)["subject"] = subject + case "slack": + (*params)["title"] = subject + (*params)["thumb_url"] = logoUrl + case "smtp": + (*params)["subject"] = subject + case "standard": + (*params)["subject"] = subject + case "telegram": + (*params)["subject"] = subject + case "zulip": + (*params)["topic"] = subject + } + + return serviceName, params, nil +} + //utility functions // waitTimeout waits for the waitgroup for the specified max timeout. // Returns true if waiting timed out. diff --git a/webapp/backend/pkg/web/handler/send_test_notification.go b/webapp/backend/pkg/web/handler/send_test_notification.go index 1091f72..153c013 100644 --- a/webapp/backend/pkg/web/handler/send_test_notification.go +++ b/webapp/backend/pkg/web/handler/send_test_notification.go @@ -1,14 +1,12 @@ package handler import ( - "fmt" "github.com/analogj/scrutiny/webapp/backend/pkg/config" dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" "github.com/analogj/scrutiny/webapp/backend/pkg/notify" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "net/http" - "os" ) // Send test notification @@ -17,15 +15,14 @@ func SendTestNotification(c *gin.Context) { logger := c.MustGet("LOGGER").(logrus.FieldLogger) testNotify := notify.Notify{ + Logger: logger, Config: appConfig, Payload: notify.Payload{ - Mailer: os.Args[0], - Subject: fmt.Sprintf("Scrutiny SMART error (EmailTest) detected on disk: XXXXX"), FailureType: "EmailTest", - Device: "/dev/sda", - DeviceType: "ata", - DeviceString: "/dev/sda", - Message: "TEST EMAIL from smartd for device: /dev/sda", + DeviceSerial: "FAKEWDDJ324KSO", + DeviceType: dbModels.DeviceProtocolAta, + DeviceName: "/dev/sda", + Test: true, }, } err := testNotify.Send() diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index f3bb442..d94bd05 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -1,8 +1,10 @@ package handler import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" + "github.com/analogj/scrutiny/webapp/backend/pkg/notify" "github.com/gin-gonic/gin" "github.com/jinzhu/gorm" "github.com/sirupsen/logrus" @@ -12,6 +14,7 @@ import ( func UploadDeviceMetrics(c *gin.Context) { db := c.MustGet("DB").(*gorm.DB) logger := c.MustGet("LOGGER").(logrus.FieldLogger) + appConfig := c.MustGet("CONFIG").(config.Interface) var collectorSmartData collector.SmartInfo err := c.BindJSON(&collectorSmartData) @@ -45,5 +48,21 @@ func UploadDeviceMetrics(c *gin.Context) { return } + //check for error + if deviceSmartData.SmartStatus == dbModels.SmartStatusFailed { + //send notifications + testNotify := notify.Notify{ + Config: appConfig, + Payload: notify.Payload{ + FailureType: notify.NotifyFailureTypeSmartFailure, + DeviceName: device.DeviceName, + DeviceType: device.DeviceProtocol, + DeviceSerial: device.SerialNumber, + Test: false, + }, + } + _ = testNotify.Send() //we ignore error message when sending notifications. + } + c.JSON(http.StatusOK, gin.H{"success": true}) } From d339e967a619fce7f8447f8bb79ef30bbcc10256 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Fri, 2 Oct 2020 21:05:04 -0600 Subject: [PATCH 03/10] fixing notifications. --- webapp/backend/pkg/config/config.go | 2 ++ webapp/backend/pkg/config/interface.go | 2 +- webapp/backend/pkg/notify/notify.go | 5 +++++ webapp/backend/pkg/web/handler/upload_device_metrics.go | 1 + webapp/backend/pkg/web/server_test.go | 1 + 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go index 95111b0..6349b55 100644 --- a/webapp/backend/pkg/config/config.go +++ b/webapp/backend/pkg/config/config.go @@ -35,6 +35,8 @@ func (c *configuration) Init() error { c.SetDefault("log.level", "INFO") c.SetDefault("log.file", "") + c.SetDefault("notify.urls", []string{}) + //c.SetDefault("disks.include", []string{}) //c.SetDefault("disks.exclude", []string{}) diff --git a/webapp/backend/pkg/config/interface.go b/webapp/backend/pkg/config/interface.go index 479142c..fabe8af 100644 --- a/webapp/backend/pkg/config/interface.go +++ b/webapp/backend/pkg/config/interface.go @@ -5,7 +5,7 @@ import ( ) // Create mock using: -// mockgen -source=pkg/config/interface.go -destination=pkg/config/mock/mock_config.go +// mockgen -source=webapp/backend/pkg/config/interface.go -destination=webapp/backend/pkg/config/mock/mock_config.go type Interface interface { Init() error ReadConfig(configFilePath string) error diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index 01b5306..f2ccedc 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -71,6 +71,11 @@ func (n *Notify) Send() error { configUrls := n.Config.GetStringSlice("notify.urls") n.Logger.Debugf("Configured notification services: %v", configUrls) + if len(configUrls) == 0 { + n.Logger.Infof("No notification endpoints configured. Skipping failure notification.") + return nil + } + //remove http:// https:// and script:// prefixed urls notifyWebhooks := []string{} notifyScripts := []string{} diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index d94bd05..2024b07 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -60,6 +60,7 @@ func UploadDeviceMetrics(c *gin.Context) { DeviceSerial: device.SerialNumber, Test: false, }, + Logger: logger, } _ = testNotify.Send() //we ignore error message when sending notifications. } diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index b4821db..9892424 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -109,6 +109,7 @@ func TestPopulateMultiple(t *testing.T) { defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) //fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db") + fakeConfig.EXPECT().GetStringSlice("notify.urls").Return([]string{}).AnyTimes() fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) ae := web.AppEngine{ From 6ca4ce39dead813f606bb7d5e525a7416c6b877a Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 3 Oct 2020 09:53:06 -0600 Subject: [PATCH 04/10] Adding documenation for notifications. --- README.md | 28 ++++++++++++++++++++++++++++ example.scrutiny.yaml | 9 ++++++++- webapp/backend/pkg/notify/notify.go | 24 +++++++++++++++--------- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 4e09f87..770eacb 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,34 @@ We support a global YAML configuration file that must be located at /scrutiny/co Check the [example.scrutiny.yml](example.scrutiny.yaml) file for a fully commented version. +## Notifications + +Scrutiny supports sending SMART device failure notifications via the following services: +- Email +- Webhooks +- Discord +- Gotify +- Hangouts +- IFTTT +- Join +- Mattermost +- Pushbullet +- Pushover +- Slack +- Teams +- Telegram +- Tulip + +Check the `notify.urls` section of [example.scrutiny.yml](example.scrutiny.yaml) for more information and documentation for service specific setup. + +### Testing Notifications + +You can test that your notifications are configured correctly by posting an empty payload to the notifications health check API. + +``` +curl -X POST http://localhost:8080/api/health/notify +``` + # Contributing Please see the [CONTRIBUTING.md](CONTRIBUTING.md) for instructions for how to develop and contribute to the scrutiny codebase. diff --git a/example.scrutiny.yaml b/example.scrutiny.yaml index c2461eb..2846f46 100644 --- a/example.scrutiny.yaml +++ b/example.scrutiny.yaml @@ -33,6 +33,9 @@ log: level: INFO +# Notification "urls" look like the following. For more information about service specific configuration see +# Shoutrrr's documentation: https://containrrr.dev/shoutrrr/services/overview/ + #notify: # urls: # - "discord://token@channel" @@ -51,8 +54,12 @@ log: # - "script:///file/path/on/disk" # - "https://www.example.com/path" - +######################################################################################################################## +# FEATURES COMING SOON +# # The following commented out sections are a preview of additional configuration options that will be available soon. +# +######################################################################################################################## #disks: # include: diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index f2ccedc..e70334d 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -28,17 +28,21 @@ type Payload struct { Date string `json:"date"` //populated by Send function. FailureType string `json:"failure_type"` //EmailTest, SmartFail, ScrutinyFail DeviceType string `json:"device_type"` //ATA/SCSI/NVMe - DeviceName string `json:"device_string"` //dev/sda - DeviceSerial string `json:"device"` //WDDJ324KSO - Test bool `json:"-"` // false + DeviceName string `json:"device_name"` //dev/sda + DeviceSerial string `json:"device_serial"` //WDDJ324KSO + Test bool `json:"test"` // false + + //should not be populated + Subject string `json:"subject"` + Message string `json:"message"` } -func (p *Payload) GenerateMessage() string { +func (p *Payload) GenerateSubject() string { //generate a detailed failure message return fmt.Sprintf("Scrutiny SMART error (%s) detected on device: %s", p.FailureType, p.DeviceName) } -func (p *Payload) GenerateSubject() string { +func (p *Payload) GenerateMessage() string { //generate a detailed failure message message := fmt.Sprintf( `Scrutiny SMART error notification for device: %s @@ -66,6 +70,8 @@ func (n *Notify) Send() error { //validate that the Payload is populated sendDate := time.Now() n.Payload.Date = sendDate.Format(time.RFC3339) + n.Payload.Subject = n.Payload.GenerateSubject() + n.Payload.Message = n.Payload.GenerateMessage() //retrieve list of notification endpoints from config file configUrls := n.Config.GetStringSlice("notify.urls") @@ -155,13 +161,13 @@ func (n *Notify) SendScriptNotification(wg *sync.WaitGroup, scriptUrl string) { } copyEnv := os.Environ() - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.GenerateSubject())) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.Subject)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DATE=%s", n.Payload.Date)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_FAILURE_TYPE=%s", n.Payload.FailureType)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_NAME=%s", n.Payload.DeviceName)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_TYPE=%s", n.Payload.DeviceType)) copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_SERIAL=%s", n.Payload.DeviceSerial)) - copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.GenerateMessage())) + copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.Message)) err := utils.CmdExec(scriptPath, []string{}, "", copyEnv, "") if err != nil { n.Logger.Errorf("An error occurred while executing script %s: %v", scriptPath, err) @@ -189,7 +195,7 @@ func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrl string n.Logger.Errorf("An error occurred occurred while generating notification payload for %s:\n %v", serviceName, shoutrrrUrl, err) } - errs := sender.Send(n.Payload.GenerateMessage(), params) + errs := sender.Send(n.Payload.Message, params) if len(errs) > 0 { n.Logger.Errorf("One or more errors occurred occurred while sending notifications for %s:\n %v", shoutrrrUrl, errs) for _, err := range errs { @@ -208,7 +214,7 @@ func (n *Notify) GenShoutrrrNotificationParams(shoutrrrUrl string) (string, *sho params := &shoutrrrTypes.Params{} logoUrl := "https://raw.githubusercontent.com/AnalogJ/scrutiny/master/webapp/frontend/src/ms-icon-144x144.png" - subject := n.Payload.GenerateSubject() + subject := n.Payload.Subject switch serviceName { // no params supported for these services case "discord", "hangouts", "ifttt", "mattermost", "teams": From 2d903453d5628a55af41533445636f1a00d7e4cd Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 3 Oct 2020 10:01:40 -0600 Subject: [PATCH 05/10] updated Debug mode documenation. --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index 770eacb..95ed9da 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,47 @@ You can test that your notifications are configured correctly by posting an empt curl -X POST http://localhost:8080/api/health/notify ``` +# Debug mode & Log Files +Scrutiny provides various methods to change the log level to debug and generate log files. + +## Web Server/API + +You can use environmental variables to enable debug logging and/or log files for the web server: + +``` +DEBUG=true +SCRUTINY_LOG_FILE=/tmp/web.log +``` + +You can configure the log level and log file in the config file: + +``` +log: + file: '/tmp/web.log' + level: DEBUG +``` + +Or if you're not using docker, you can pass CLI arguments to the web server during startup: + +``` +scrutiny start --debug --log-file /tmp/web.log +``` + +## Collector + +You can use environmental variables to enable debug logging and/or log files for the collector: + +``` +DEBUG=true +COLLECTOR_LOG_FILE=/tmp/collector.log +``` + +Or if you're not using docker, you can pass CLI arguments to the collector during startup: + +``` +scrutiny-collector-metrics run --debug --log-file /tmp/collector.log +``` + # Contributing Please see the [CONTRIBUTING.md](CONTRIBUTING.md) for instructions for how to develop and contribute to the scrutiny codebase. From 732eb039da420a98aeb4bad8c66049022dfbebf8 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 3 Oct 2020 10:05:44 -0600 Subject: [PATCH 06/10] added custom script notification type. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 95ed9da..57220fa 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ Check the [example.scrutiny.yml](example.scrutiny.yaml) file for a fully comment ## Notifications Scrutiny supports sending SMART device failure notifications via the following services: +- Custom Script (data provided via environmental variables) - Email - Webhooks - Discord From a3438297e68284da3fa3980d6a15ab555790066e Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 3 Oct 2020 10:40:27 -0600 Subject: [PATCH 07/10] removeedd waitgroup, using sync/errgroup instead (to pass error messages back). returning errors in test notifications endpoint payload. Adding failure tests for webhooks, scripts & shoutrr. --- go.mod | 1 + go.sum | 1 + webapp/backend/pkg/notify/notify.go | 81 +++++++++---------- .../pkg/web/handler/send_test_notification.go | 1 + webapp/backend/pkg/web/server_test.go | 72 +++++++++++++++++ 5 files changed, 112 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 4b9bbb7..c1027e5 100644 --- a/go.mod +++ b/go.mod @@ -18,5 +18,6 @@ require ( github.com/stretchr/testify v1.5.1 github.com/urfave/cli/v2 v2.2.0 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect + golang.org/x/sync v0.0.0-20190423024810-112230192c58 gopkg.in/yaml.v2 v2.3.0 // indirect ) diff --git a/go.sum b/go.sum index e32dd06..df4c220 100644 --- a/go.sum +++ b/go.sum @@ -347,6 +347,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index e70334d..dffece3 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -3,17 +3,18 @@ package notify import ( "bytes" "encoding/json" + "errors" "fmt" "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/containrrr/shoutrrr" shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types" "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" "net/http" "net/url" "os" "strings" - "sync" "time" ) @@ -102,62 +103,65 @@ func (n *Notify) Send() error { n.Logger.Debugf("Configured shoutrrr: %v", notifyShoutrrr) //run all scripts, webhooks and shoutrr commands in parallel - var wg sync.WaitGroup + //var wg sync.WaitGroup + var eg errgroup.Group for _, notifyWebhook := range notifyWebhooks { // execute collection in parallel go-routines - wg.Add(1) - go n.SendWebhookNotification(&wg, notifyWebhook) + eg.Go(func() error { return n.SendWebhookNotification(notifyWebhook) }) } for _, notifyScript := range notifyScripts { // execute collection in parallel go-routines - wg.Add(1) - go n.SendScriptNotification(&wg, notifyScript) + eg.Go(func() error { return n.SendScriptNotification(notifyScript) }) } for _, shoutrrrUrl := range notifyShoutrrr { - wg.Add(1) - go n.SendShoutrrrNotification(&wg, shoutrrrUrl) + eg.Go(func() error { return n.SendShoutrrrNotification(shoutrrrUrl) }) } //and wait for completion, error or timeout. n.Logger.Debugf("Main: waiting for notifications to complete.") - //wg.Wait() - if waitTimeout(&wg, time.Minute) { //wait for 1 minute - fmt.Println("Timed out while sending notifications") + + if err := eg.Wait(); err == nil { + n.Logger.Info("Successfully sent notifications. Check logs for more information.") + return nil } else { - fmt.Println("Sent notifications. Check logs for more information.") - } - return nil + n.Logger.Error("One or more notifications failed to send successfully. See logs for more information.") + return err + } + ////wg.Wait() + //if waitTimeout(&wg, time.Minute) { //wait for 1 minute + // fmt.Println("Timed out while sending notifications") + //} else { + //} + //return nil } -func (n *Notify) SendWebhookNotification(wg *sync.WaitGroup, webhookUrl string) { - defer wg.Done() +func (n *Notify) SendWebhookNotification(webhookUrl string) error { n.Logger.Infof("Sending Webhook to %s", webhookUrl) requestBody, err := json.Marshal(n.Payload) if err != nil { n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) - return + return err } resp, err := http.Post(webhookUrl, "application/json", bytes.NewBuffer(requestBody)) if err != nil { n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err) - return + return err } defer resp.Body.Close() //we don't care about resp body content, but maybe we should log it? + return nil } -func (n *Notify) SendScriptNotification(wg *sync.WaitGroup, scriptUrl string) { - defer wg.Done() - +func (n *Notify) SendScriptNotification(scriptUrl string) error { //check if the script exists. scriptPath := strings.TrimPrefix(scriptUrl, "script://") n.Logger.Infof("Executing Script %s", scriptPath) if !utils.FileExists(scriptPath) { n.Logger.Errorf("Script does not exist: %s", scriptPath) - return + return errors.New(fmt.Sprintf("custom script path does not exist: %s", scriptPath)) } copyEnv := os.Environ() @@ -171,20 +175,20 @@ func (n *Notify) SendScriptNotification(wg *sync.WaitGroup, scriptUrl string) { err := utils.CmdExec(scriptPath, []string{}, "", copyEnv, "") if err != nil { n.Logger.Errorf("An error occurred while executing script %s: %v", scriptPath, err) + return err } - return + return nil } -func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrl string) { +func (n *Notify) SendShoutrrrNotification(shoutrrrUrl string) error { fmt.Printf("Sending Notifications to %v", shoutrrrUrl) n.Logger.Infof("Sending notifications to %v", shoutrrrUrl) - defer wg.Done() sender, err := shoutrrr.CreateSender(shoutrrrUrl) if err != nil { n.Logger.Errorf("An error occurred while sending notifications %v: %v", shoutrrrUrl, err) - return + return err } //sender.SetLogger(n.Logger.) @@ -193,15 +197,21 @@ func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrl string if err != nil { n.Logger.Errorf("An error occurred occurred while generating notification payload for %s:\n %v", serviceName, shoutrrrUrl, err) + return err } errs := sender.Send(n.Payload.Message, params) if len(errs) > 0 { - n.Logger.Errorf("One or more errors occurred occurred while sending notifications for %s:\n %v", shoutrrrUrl, errs) + n.Logger.Errorf("One or more errors occurred occurred while sending notifications for %s:", shoutrrrUrl) + var errstrings []string + for _, err := range errs { n.Logger.Error(err) + errstrings = append(errstrings, err.Error()) } + return errors.New(strings.Join(errstrings, "\n")) } + return nil } func (n *Notify) GenShoutrrrNotificationParams(shoutrrrUrl string) (string, *shoutrrrTypes.Params, error) { @@ -243,20 +253,3 @@ func (n *Notify) GenShoutrrrNotificationParams(shoutrrrUrl string) (string, *sho return serviceName, params, nil } - -//utility functions -// waitTimeout waits for the waitgroup for the specified max timeout. -// Returns true if waiting timed out. -func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { - c := make(chan struct{}) - go func() { - defer close(c) - wg.Wait() - }() - select { - case <-c: - return false // completed normally - case <-time.After(timeout): - return true // timed out - } -} diff --git a/webapp/backend/pkg/web/handler/send_test_notification.go b/webapp/backend/pkg/web/handler/send_test_notification.go index 153c013..52ba8f5 100644 --- a/webapp/backend/pkg/web/handler/send_test_notification.go +++ b/webapp/backend/pkg/web/handler/send_test_notification.go @@ -30,6 +30,7 @@ func SendTestNotification(c *gin.Context) { logger.Errorln("An error occurred while sending test notification", err) c.JSON(http.StatusInternalServerError, gin.H{ "success": false, + "errors": []string{err.Error()}, }) } else { c.JSON(http.StatusOK, dbModels.DeviceWrapper{ diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index 9892424..e121a2d 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -188,6 +188,78 @@ func TestSendTestNotificationRoute(t *testing.T) { require.Equal(t, 200, wr.Code) } +func TestSendTestNotificationRoute_WebhookFailure(t *testing.T) { + //setup + parentPath, _ := ioutil.TempDir("", "") + defer os.RemoveAll(parentPath) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) + fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) + fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://unroutable.domain.example.asdfghj"}) + ae := web.AppEngine{ + Config: fakeConfig, + } + router := ae.Setup(logrus.New()) + + //test + wr := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/api/health/notify", strings.NewReader("{}")) + router.ServeHTTP(wr, req) + + //assert + require.Equal(t, 500, wr.Code) +} + +func TestSendTestNotificationRoute_ScriptFailure(t *testing.T) { + //setup + parentPath, _ := ioutil.TempDir("", "") + defer os.RemoveAll(parentPath) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) + fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) + fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///missing/path/on/disk"}) + ae := web.AppEngine{ + Config: fakeConfig, + } + router := ae.Setup(logrus.New()) + + //test + wr := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/api/health/notify", strings.NewReader("{}")) + router.ServeHTTP(wr, req) + + //assert + require.Equal(t, 500, wr.Code) +} + +func TestSendTestNotificationRoute_ShoutrrrFailure(t *testing.T) { + //setup + parentPath, _ := ioutil.TempDir("", "") + defer os.RemoveAll(parentPath) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) + fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) + fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"discord://invalidtoken@channel"}) + ae := web.AppEngine{ + Config: fakeConfig, + } + router := ae.Setup(logrus.New()) + + //test + wr := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/api/health/notify", strings.NewReader("{}")) + router.ServeHTTP(wr, req) + + //assert + require.Equal(t, 500, wr.Code) +} + func TestGetDevicesSummaryRoute_Nvme(t *testing.T) { //setup parentPath, _ := ioutil.TempDir("", "") From 80a382d098e4029acda3fc150f4f5ee46cc06176 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sun, 4 Oct 2020 13:55:59 -0600 Subject: [PATCH 08/10] slight tweaks to notifications error handling. --- webapp/backend/pkg/notify/notify.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index dffece3..c2ae507 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -206,10 +206,19 @@ func (n *Notify) SendShoutrrrNotification(shoutrrrUrl string) error { var errstrings []string for _, err := range errs { + if err == nil || err.Error() == "" { + continue + } n.Logger.Error(err) errstrings = append(errstrings, err.Error()) } - return errors.New(strings.Join(errstrings, "\n")) + //sometimes there are empty errs, we're going to skip them. + if len(errstrings) == 0 { + return nil + } else { + return errors.New(strings.Join(errstrings, "\n")) + + } } return nil } From c8a4be6b07013adad7aa7f31a424aee89449684b Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sun, 4 Oct 2020 14:30:05 -0600 Subject: [PATCH 09/10] fix x509: certificate signed by unknown authority error for notifications. --- docker/Dockerfile | 3 +-- docker/Dockerfile.collector | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 90bd7d4..19746c4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,13 +34,12 @@ ENV PATH="/scrutiny/bin:${PATH}" ADD https://github.com/dshearer/jobber/releases/download/v1.4.4/jobber_1.4.4-1_amd64.deb /tmp/ RUN apt install /tmp/jobber_1.4.4-1_amd64.deb -RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1 +RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates && update-ca-certificates ADD https://github.com/just-containers/s6-overlay/releases/download/v1.21.8.0/s6-overlay-amd64.tar.gz /tmp/ RUN tar xzf /tmp/s6-overlay-amd64.tar.gz -C / COPY /rootfs / - COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny /scrutiny/bin/ COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /scrutiny/bin/ COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /scrutiny/bin/ diff --git a/docker/Dockerfile.collector b/docker/Dockerfile.collector index e783d02..6899899 100644 --- a/docker/Dockerfile.collector +++ b/docker/Dockerfile.collector @@ -18,11 +18,10 @@ ENV PATH="/scrutiny/bin:${PATH}" ADD https://github.com/dshearer/jobber/releases/download/v1.4.4/jobber_1.4.4-1_amd64.deb /tmp/ RUN apt install /tmp/jobber_1.4.4-1_amd64.deb -RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1 +RUN apt-get update && apt-get install -y smartmontools=7.0-0ubuntu1~ubuntu18.04.1 ca-certificates && update-ca-certificates COPY /rootfs/scrutiny /scrutiny - COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-selftest /scrutiny/bin/ COPY --from=backendbuild /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /scrutiny/bin/ RUN chmod +x /scrutiny/bin/scrutiny-collector-selftest && \ From ac7c1f28cfaedaf45c8859d3654576fa7a5795af Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Mon, 5 Oct 2020 08:56:43 -0600 Subject: [PATCH 10/10] ignore empty ("") errors. --- webapp/backend/pkg/notify/notify.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index c2ae507..3c7a924 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -202,22 +202,21 @@ func (n *Notify) SendShoutrrrNotification(shoutrrrUrl string) error { errs := sender.Send(n.Payload.Message, params) if len(errs) > 0 { - n.Logger.Errorf("One or more errors occurred occurred while sending notifications for %s:", shoutrrrUrl) var errstrings []string for _, err := range errs { if err == nil || err.Error() == "" { continue } - n.Logger.Error(err) errstrings = append(errstrings, err.Error()) } //sometimes there are empty errs, we're going to skip them. if len(errstrings) == 0 { return nil } else { + n.Logger.Errorf("One or more errors occurred while sending notifications for %s:", shoutrrrUrl) + n.Logger.Error(errs) return errors.New(strings.Join(errstrings, "\n")) - } } return nil