open-source-mirrors
/
scrutiny
mirror of https://github.com/AnalogJ/scrutiny
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

sending notifications on failure.

Browse Source
pull/75/head
Jason Kulatunga 4 years ago
parent 1246f5bba9
commit 6377a258f6
4 changed files with 152 additions and 44 deletions
Show Stats Download Patch File Download Diff File

18
example.scrutiny.yaml
Unescape View File

@ -32,19 +32,12 @@ log:
file: '' #absolute or relative paths allowed, eg. web.log
level: INFO
# The following commented out sections are a preview of additional configuration options that will be available soon.
#disks:
# include:
# # - /dev/sda
# exclude:
# # - /dev/sdb
#notify:
# urls:
# - "discord://token@channel"
# - "telegram://token@telegram?channels=channel-1[,channel-2,...]"
# - "pushover://shoutrrr:apiToken@userKey/?devices=device1[,device2, ...]"
# - "pushover://shoutrrr:apiToken@userKey/?priority=1&devices=device1[,device2, ...]"
# - "slack://[botname@]token-a/token-b/token-c"
# - "smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]"
# - "teams://token-a/token-b/token-c"
@ -58,6 +51,15 @@ log:
# - "script:///file/path/on/disk"
# - "https://www.example.com/path"
# The following commented out sections are a preview of additional configuration options that will be available soon.
#disks:
# include:
# # - /dev/sda
# exclude:
# # - /dev/sdb
#limits:
# ata:
# critical:

146
webapp/backend/pkg/notify/notify.go
Unescape View File

@ -7,26 +7,57 @@ import (
"github.com/analogj/go-util/utils"
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/containrrr/shoutrrr"
log "github.com/sirupsen/logrus"
shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types"
"github.com/sirupsen/logrus"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
)
const NotifyFailureTypeEmailTest = "EmailTest"
const NotifyFailureTypeSmartPrefail = "SmartPreFailure"
const NotifyFailureTypeSmartFailure = "SmartFailure"
const NotifyFailureTypeSmartErrorLog = "SmartErrorLog"
const NotifyFailureTypeSmartSelfTest = "SmartSelfTestLog"
// TODO: include host and/or user label for device.
type Payload struct {
Mailer string `json:"mailer"`
Subject string `json:"subject"`
Date string `json:"date"`
FailureType string `json:"failure_type"`
Device string `json:"device"`
DeviceType string `json:"device_type"`
DeviceString string `json:"device_string"`
Message string `json:"message"`
Date string `json:"date"` //populated by Send function.
FailureType string `json:"failure_type"` //EmailTest, SmartFail, ScrutinyFail
DeviceType string `json:"device_type"` //ATA/SCSI/NVMe
DeviceName string `json:"device_string"` //dev/sda
DeviceSerial string `json:"device"` //WDDJ324KSO
Test bool `json:"-"` // false
}
func (p *Payload) GenerateMessage() string {
//generate a detailed failure message
return fmt.Sprintf("Scrutiny SMART error (%s) detected on device: %s", p.FailureType, p.DeviceName)
}
func (p *Payload) GenerateSubject() string {
//generate a detailed failure message
message := fmt.Sprintf(
`Scrutiny SMART error notification for device: %s
Failure Type: %s
Device Name: %s
Device Serial: %s
Device Type: %s
Date: %s`, p.DeviceName, p.FailureType, p.DeviceName, p.DeviceSerial, p.DeviceType, p.Date)
if p.Test {
message = "TEST NOTIFICATION:\n" + message
}
return message
}
type Notify struct {
Logger logrus.FieldLogger
Config config.Interface
Payload Payload
}
@ -38,6 +69,7 @@ func (n *Notify) Send() error {
//retrieve list of notification endpoints from config file
configUrls := n.Config.GetStringSlice("notify.urls")
n.Logger.Debugf("Configured notification services: %v", configUrls)
//remove http:// https:// and script:// prefixed urls
notifyWebhooks := []string{}
@ -54,6 +86,10 @@ func (n *Notify) Send() error {
}
}
n.Logger.Debugf("Configured scripts: %v", notifyScripts)
n.Logger.Debugf("Configured webhooks: %v", notifyWebhooks)
n.Logger.Debugf("Configured shoutrrr: %v", notifyShoutrrr)
//run all scripts, webhooks and shoutrr commands in parallel
var wg sync.WaitGroup
@ -67,12 +103,14 @@ func (n *Notify) Send() error {
wg.Add(1)
go n.SendScriptNotification(&wg, notifyScript)
}
if len(notifyScripts) > 0 {
for _, shoutrrrUrl := range notifyShoutrrr {
wg.Add(1)
go n.SendShoutrrrNotification(&wg, notifyShoutrrr)
go n.SendShoutrrrNotification(&wg, shoutrrrUrl)
}
//and wait for completion, error or timeout.
n.Logger.Debugf("Main: waiting for notifications to complete.")
//wg.Wait()
if waitTimeout(&wg, time.Minute) { //wait for 1 minute
fmt.Println("Timed out while sending notifications")
} else {
@ -83,16 +121,16 @@ func (n *Notify) Send() error {
func (n *Notify) SendWebhookNotification(wg *sync.WaitGroup, webhookUrl string) {
defer wg.Done()
log.Infof("Sending Webhook to %s", webhookUrl)
n.Logger.Infof("Sending Webhook to %s", webhookUrl)
requestBody, err := json.Marshal(n.Payload)
if err != nil {
log.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err)
n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err)
return
}
resp, err := http.Post(webhookUrl, "application/json", bytes.NewBuffer(requestBody))
if err != nil {
log.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err)
n.Logger.Errorf("An error occurred while sending Webhook to %s: %v", webhookUrl, err)
return
}
defer resp.Body.Close()
@ -104,45 +142,97 @@ func (n *Notify) SendScriptNotification(wg *sync.WaitGroup, scriptUrl string) {
//check if the script exists.
scriptPath := strings.TrimPrefix(scriptUrl, "script://")
log.Infof("Executing Script %s", scriptPath)
n.Logger.Infof("Executing Script %s", scriptPath)
if !utils.FileExists(scriptPath) {
log.Errorf("Script does not exist: %s", scriptPath)
n.Logger.Errorf("Script does not exist: %s", scriptPath)
return
}
copyEnv := os.Environ()
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MAILER=%s", n.Payload.Mailer))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.Subject))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_SUBJECT=%s", n.Payload.GenerateSubject()))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DATE=%s", n.Payload.Date))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_FAILURE_TYPE=%s", n.Payload.FailureType))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE=%s", n.Payload.Device))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_NAME=%s", n.Payload.DeviceName))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_TYPE=%s", n.Payload.DeviceType))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_STRING=%s", n.Payload.DeviceString))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.Message))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_DEVICE_SERIAL=%s", n.Payload.DeviceSerial))
copyEnv = append(copyEnv, fmt.Sprintf("SCRUTINY_MESSAGE=%s", n.Payload.GenerateMessage()))
err := utils.CmdExec(scriptPath, []string{}, "", copyEnv, "")
if err != nil {
log.Errorf("An error occurred while executing script %s: %v", scriptPath, err)
n.Logger.Errorf("An error occurred while executing script %s: %v", scriptPath, err)
}
return
}
func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrls []string) {
log.Infof("Sending notifications to %v", shoutrrrUrls)
func (n *Notify) SendShoutrrrNotification(wg *sync.WaitGroup, shoutrrrUrl string) {
fmt.Printf("Sending Notifications to %v", shoutrrrUrl)
n.Logger.Infof("Sending notifications to %v", shoutrrrUrl)
defer wg.Done()
sender, err := shoutrrr.CreateSender(shoutrrrUrls...)
sender, err := shoutrrr.CreateSender(shoutrrrUrl)
if err != nil {
log.Errorf("An error occurred while sending notifications %v: %v", shoutrrrUrls, err)
n.Logger.Errorf("An error occurred while sending notifications %v: %v", shoutrrrUrl, err)
return
}
errs := sender.Send(n.Payload.Subject, nil) //structs.Map(n.Payload).())
//sender.SetLogger(n.Logger.)
serviceName, params, err := n.GenShoutrrrNotificationParams(shoutrrrUrl)
n.Logger.Debug("notification data for %s: (%s)\n%v", serviceName, shoutrrrUrl, params)
if err != nil {
n.Logger.Errorf("An error occurred occurred while generating notification payload for %s:\n %v", serviceName, shoutrrrUrl, err)
}
errs := sender.Send(n.Payload.GenerateMessage(), params)
if len(errs) > 0 {
log.Errorf("One or more errors occurred occurred while sending notifications %v:\n %v", shoutrrrUrls, errs)
n.Logger.Errorf("One or more errors occurred occurred while sending notifications for %s:\n %v", shoutrrrUrl, errs)
for _, err := range errs {
n.Logger.Error(err)
}
}
}
func (n *Notify) GenShoutrrrNotificationParams(shoutrrrUrl string) (string, *shoutrrrTypes.Params, error) {
serviceURL, err := url.Parse(shoutrrrUrl)
if err != nil {
return "", nil, err
}
serviceName := serviceURL.Scheme
params := &shoutrrrTypes.Params{}
logoUrl := "https://raw.githubusercontent.com/AnalogJ/scrutiny/master/webapp/frontend/src/ms-icon-144x144.png"
subject := n.Payload.GenerateSubject()
switch serviceName {
// no params supported for these services
case "discord", "hangouts", "ifttt", "mattermost", "teams":
break
case "gotify":
(*params)["title"] = subject
case "join":
(*params)["title"] = subject
(*params)["icon"] = logoUrl
case "pushbullet":
(*params)["title"] = subject
case "pushover":
(*params)["subject"] = subject
case "slack":
(*params)["title"] = subject
(*params)["thumb_url"] = logoUrl
case "smtp":
(*params)["subject"] = subject
case "standard":
(*params)["subject"] = subject
case "telegram":
(*params)["subject"] = subject
case "zulip":
(*params)["topic"] = subject
}
return serviceName, params, nil
}
//utility functions
// waitTimeout waits for the waitgroup for the specified max timeout.
// Returns true if waiting timed out.

13
webapp/backend/pkg/web/handler/send_test_notification.go
Unescape View File

@ -1,14 +1,12 @@
package handler
import (
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"net/http"
"os"
)
// Send test notification
@ -17,15 +15,14 @@ func SendTestNotification(c *gin.Context) {
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
testNotify := notify.Notify{
Logger: logger,
Config: appConfig,
Payload: notify.Payload{
Mailer: os.Args[0],
Subject: fmt.Sprintf("Scrutiny SMART error (EmailTest) detected on disk: XXXXX"),
FailureType: "EmailTest",
Device: "/dev/sda",
DeviceType: "ata",
DeviceString: "/dev/sda",
Message: "TEST EMAIL from smartd for device: /dev/sda",
DeviceSerial: "FAKEWDDJ324KSO",
DeviceType: dbModels.DeviceProtocolAta,
DeviceName: "/dev/sda",
Test: true,
},
}
err := testNotify.Send()

19
webapp/backend/pkg/web/handler/upload_device_metrics.go
Unescape View File

@ -1,8 +1,10 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"github.com/sirupsen/logrus"
@ -12,6 +14,7 @@ import (
func UploadDeviceMetrics(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
appConfig := c.MustGet("CONFIG").(config.Interface)
var collectorSmartData collector.SmartInfo
err := c.BindJSON(&collectorSmartData)
@ -45,5 +48,21 @@ func UploadDeviceMetrics(c *gin.Context) {
return
}
//check for error
if deviceSmartData.SmartStatus == dbModels.SmartStatusFailed {
//send notifications
testNotify := notify.Notify{
Config: appConfig,
Payload: notify.Payload{
FailureType: notify.NotifyFailureTypeSmartFailure,
DeviceName: device.DeviceName,
DeviceType: device.DeviceProtocol,
DeviceSerial: device.SerialNumber,
Test: false,
},
}
_ = testNotify.Send() //we ignore error message when sending notifications.
}
c.JSON(http.StatusOK, gin.H{"success": true})
}

Write Preview
Loading…
Cancel
Save