WIP settings system.

- updated dbdiagrams schema
- [BREAKING] force failure if `notify.filter_attributes` or `notify.level` is set
- added Settings table (and default values during migration)
- Added Save Settings and Get Settings functions.
- Added web API endpoints for getting and saving settings.
- Deprecated old Notify* constants. Created new MetricsStatus* and MetricsNotifyLevel constants.
pull/338/head
Jason Kulatunga 2 years ago
parent dd0c3e6fba
commit 99af2b8b16

@ -1,62 +1,88 @@
// SQLite Table(s) // SQLite Table(s)
Table device {
created_at timestamp
wwn varchar [pk]
//user provided
label varchar
host_id varchar
// smartctl provided
device_name varchar
manufacturer varchar
model_name varchar
interface_type varchar
interface_speed varchar
serial_number varchar
firmware varchar
rotational_speed varchar
capacity varchar
form_factor varchar
smart_support varchar
device_protocol varchar
device_type varchar
Table Device {
//GORM attributes, see: http://gorm.io/docs/conventions.html
CreatedAt time
UpdatedAt time
DeletedAt time
WWN string
DeviceName string
DeviceUUID string
DeviceSerialID string
DeviceLabel string
Manufacturer string
ModelName string
InterfaceType string
InterfaceSpeed string
SerialNumber string
Firmware string
RotationSpeed int
Capacity int64
FormFactor string
SmartSupport bool
DeviceProtocol string//protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
DeviceType string//device type is used for querying with -d/t flag, should only be used by collector.
// User provided metadata
Label string
HostId string
// Data set by Scrutiny
DeviceStatus enum
} }
Table Setting {
//GORM attributes, see: http://gorm.io/docs/conventions.html
// InfluxDB Tables SettingKeyName string
Table device_temperature { SettingKeyDescription string
//timestamp SettingDataType string
created_at timestamp
//tags (indexed & queryable)
device_wwn varchar [pk]
//fields SettingValueNumeric int64
temp bigint SettingValueString string
} }
Table smart_ata_results { // InfluxDB Tables
//timestamp Table SmartTemperature {
created_at timestamp Date time
DeviceWWN string //(tag)
//tags (indexed & queryable) Temp int64
device_wwn varchar [pk] }
smart_status varchar
scrutiny_status varchar
//fields
temp bigint
power_on_hours bigint
power_cycle_count bigint
Table Smart {
Date time
DeviceWWN string //(tag)
DeviceProtocol string
//Metrics (fields)
Temp int64
PowerOnHours int64
PowerCycleCount int64
//Smart Status
Status enum
//SMART Attributes (fields)
Attr_ID_AttributeId int
Attr_ID_Value int64
Attr_ID_Threshold int64
Attr_ID_Worst int64
Attr_ID_RawValue int64
Attr_ID_RawString string
Attr_ID_WhenFailed string
//Generated data
Attr_ID_TransformedValue int64
Attr_ID_Status enum
Attr_ID_StatusReason string
Attr_ID_FailureRate float64
} }
Ref: device.wwn < smart_ata_results.device_wwn Ref: Device.WWN < Smart.DeviceWWN
Ref: Device.WWN < SmartTemperature.DeviceWWN

@ -2,7 +2,6 @@ package config
import ( import (
"github.com/analogj/go-util/utils" "github.com/analogj/go-util/utils"
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/errors" "github.com/analogj/scrutiny/webapp/backend/pkg/errors"
"github.com/spf13/viper" "github.com/spf13/viper"
"log" "log"
@ -39,8 +38,6 @@ func (c *configuration) Init() error {
c.SetDefault("log.file", "") c.SetDefault("log.file", "")
c.SetDefault("notify.urls", []string{}) c.SetDefault("notify.urls", []string{})
c.SetDefault("notify.filter_attributes", pkg.NotifyFilterAttributesAll)
c.SetDefault("notify.level", pkg.NotifyLevelFail)
c.SetDefault("web.influxdb.scheme", "http") c.SetDefault("web.influxdb.scheme", "http")
c.SetDefault("web.influxdb.host", "localhost") c.SetDefault("web.influxdb.host", "localhost")
@ -55,17 +52,6 @@ func (c *configuration) Init() error {
//c.SetDefault("disks.include", []string{}) //c.SetDefault("disks.include", []string{})
//c.SetDefault("disks.exclude", []string{}) //c.SetDefault("disks.exclude", []string{})
//c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh")
//c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh")
//c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh")
//c.SetDefault("collect.metric.enable", true)
//c.SetDefault("collect.metric.command", "-a -o on -S on")
//c.SetDefault("collect.long.enable", true)
//c.SetDefault("collect.long.command", "-a -o on -S on")
//c.SetDefault("collect.short.enable", true)
//c.SetDefault("collect.short.command", "-a -o on -S on")
//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig //if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
c.SetConfigType("yaml") c.SetConfigType("yaml")
//c.SetConfigName("drawbridge") //c.SetConfigName("drawbridge")
@ -77,7 +63,7 @@ func (c *configuration) Init() error {
c.AutomaticEnv() c.AutomaticEnv()
//CLI options will be added via the `Set()` function //CLI options will be added via the `Set()` function
return nil return c.ValidateConfig()
} }
func (c *configuration) ReadConfig(configFilePath string) error { func (c *configuration) ReadConfig(configFilePath string) error {
@ -120,24 +106,18 @@ func (c *configuration) ReadConfig(configFilePath string) error {
// This function ensures that the merged config works correctly. // This function ensures that the merged config works correctly.
func (c *configuration) ValidateConfig() error { func (c *configuration) ValidateConfig() error {
////deserialize Questions //the following keys are deprecated, and no longer supported
//questionsMap := map[string]Question{} /*
//err := c.UnmarshalKey("questions", &questionsMap) - notify.filter_attributes (replaced by metrics.status.filter_attributes SETTING)
// - notify.level (replaced by metrics.notify.level and metrics.status.threshold SETTING)
//if err != nil { */
// log.Printf("questions could not be deserialized correctly. %v", err) //TODO add docs and upgrade doc.
// return err if c.IsSet("notify.filter_attributes") {
//} return errors.ConfigValidationError("`notify.filter_attributes` configuration option is deprecated. Replaced by option in Dashboard Settings page")
// }
//for _, v := range questionsMap { if c.IsSet("notify.level") {
// return errors.ConfigValidationError("`notify.level` configuration option is deprecated. Replaced by option in Dashboard Settings page")
// typeContent, ok := v.Schema["type"].(string) }
// if !ok || len(typeContent) == 0 {
// return errors.QuestionSyntaxError("`type` is required for questions")
// }
//}
//
//
return nil return nil
} }

@ -4,17 +4,11 @@ const DeviceProtocolAta = "ATA"
const DeviceProtocolScsi = "SCSI" const DeviceProtocolScsi = "SCSI"
const DeviceProtocolNvme = "NVMe" const DeviceProtocolNvme = "NVMe"
const NotifyFilterAttributesAll = "all"
const NotifyFilterAttributesCritical = "critical"
const NotifyLevelFail = "fail"
const NotifyLevelFailScrutiny = "fail_scrutiny"
const NotifyLevelFailSmart = "fail_smart"
//go:generate stringer -type=AttributeStatus //go:generate stringer -type=AttributeStatus
// AttributeStatus bitwise flag, 1,2,4,8,16,32,etc
type AttributeStatus uint8 type AttributeStatus uint8
const ( const (
// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
AttributeStatusPassed AttributeStatus = 0 AttributeStatusPassed AttributeStatus = 0
AttributeStatusFailedSmart AttributeStatus = 1 AttributeStatusFailedSmart AttributeStatus = 1
AttributeStatusWarningScrutiny AttributeStatus = 2 AttributeStatusWarningScrutiny AttributeStatus = 2
@ -30,9 +24,10 @@ func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^
func AttributeStatusHas(b, flag AttributeStatus) bool { return b&flag != 0 } func AttributeStatusHas(b, flag AttributeStatus) bool { return b&flag != 0 }
//go:generate stringer -type=DeviceStatus //go:generate stringer -type=DeviceStatus
// DeviceStatus bitwise flag, 1,2,4,8,16,32,etc
type DeviceStatus uint8 type DeviceStatus uint8
const ( const (
// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
DeviceStatusPassed DeviceStatus = 0 DeviceStatusPassed DeviceStatus = 0
DeviceStatusFailedSmart DeviceStatus = 1 DeviceStatusFailedSmart DeviceStatus = 1
DeviceStatusFailedScrutiny DeviceStatus = 2 DeviceStatusFailedScrutiny DeviceStatus = 2
@ -42,3 +37,44 @@ func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus { return b | flag }
func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus { return b &^ flag } func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus { return b &^ flag }
func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag } func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
func DeviceStatusHas(b, flag DeviceStatus) bool { return b&flag != 0 } func DeviceStatusHas(b, flag DeviceStatus) bool { return b&flag != 0 }
// Metrics Specific Filtering & Threshold Constants
type MetricsNotifyLevel int64
const (
MetricsNotifyLevelWarn MetricsNotifyLevel = 1
MetricsNotifyLevelFail MetricsNotifyLevel = 2
)
type MetricsStatusFilterAttributes int64
const (
MetricsStatusFilterAttributesAll MetricsStatusFilterAttributes = 0
MetricsStatusFilterAttributesCritical MetricsStatusFilterAttributes = 1
)
// MetricsStatusThreshold bitwise flag, 1,2,4,8,16,32,etc
type MetricsStatusThreshold int64
const (
MetricsStatusThresholdSmart MetricsStatusThreshold = 1
MetricsStatusThresholdScrutiny MetricsStatusThreshold = 2
//shortcut
MetricsStatusThresholdBoth MetricsStatusThreshold = 3
)
// Deprecated
const NotifyFilterAttributesAll = "all"
// Deprecated
const NotifyFilterAttributesCritical = "critical"
// Deprecated
const NotifyLevelFail = "fail"
// Deprecated
const NotifyLevelFailScrutiny = "fail_scrutiny"
// Deprecated
const NotifyLevelFailSmart = "fail_smart"

@ -28,4 +28,7 @@ type DeviceRepo interface {
GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)
GetSettings(ctx context.Context) (*models.Settings, error)
SaveSettings(ctx context.Context, settings models.Settings) error
} }

@ -9,6 +9,7 @@ type Setting struct {
gorm.Model gorm.Model
SettingKeyName string `json:"setting_key_name"` SettingKeyName string `json:"setting_key_name"`
SettingKeyDescription string `json:"setting_key_description"`
SettingDataType string `json:"setting_data_type"` SettingDataType string `json:"setting_data_type"`
SettingValueNumeric int64 `json:"setting_value_numeric"` SettingValueNumeric int64 `json:"setting_value_numeric"`

@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100"
@ -281,7 +282,33 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
Migrate: func(tx *gorm.DB) error { Migrate: func(tx *gorm.DB) error {
// adding the settings table. // adding the settings table.
return tx.AutoMigrate(m20220716214900.Setting{}) err := tx.AutoMigrate(m20220716214900.Setting{})
if err != nil {
return err
}
//add defaults.
var defaultSettings = []m20220716214900.Setting{
{
SettingKeyName: "metrics.notify.level",
SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)",
SettingDataType: "numeric",
SettingValueNumeric: int64(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn'
},
{
SettingKeyName: "metrics.status.filter_attributes",
SettingKeyDescription: "Determines which attributes should impact device status",
SettingDataType: "numeric",
SettingValueNumeric: int64(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or 'critical'
},
{
SettingKeyName: "metrics.status.threshold",
SettingKeyDescription: "Determines which threshold should impact device status",
SettingDataType: "string",
SettingValueNumeric: int64(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both'
},
}
return tx.Create(&defaultSettings).Error
}, },
}, },
}) })

@ -0,0 +1,33 @@
package database
import (
"context"
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
)
func (sr *scrutinyRepository) GetSettings(ctx context.Context) (*models.Settings, error) {
settingsEntries := []models.SettingEntry{}
if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
return nil, fmt.Errorf("Could not get settings from DB: %v", err)
}
settings := models.Settings{}
settings.PopulateFromSettingEntries(settingsEntries)
return &settings, nil
}
func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error {
//get current settings
settingsEntries := []models.SettingEntry{}
if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
return fmt.Errorf("Could not get settings from DB: %v", err)
}
// override with values from settings object
settingsEntries = settings.UpdateSettingEntries(settingsEntries)
// store in database.
return sr.gormClient.Updates(&settingsEntries).Error
}

@ -1,16 +0,0 @@
package models
import (
"gorm.io/gorm"
)
type Setting struct {
//GORM attributes, see: http://gorm.io/docs/conventions.html
gorm.Model
SettingKeyName string `json:"setting_key_name"`
SettingDataType string `json:"setting_data_type"`
SettingValueNumeric int64 `json:"setting_value_numeric"`
SettingValueString string `json:"setting_value_string"`
}

@ -0,0 +1,22 @@
package models
import (
"gorm.io/gorm"
)
// SettingEntry matches a setting row in the database
type SettingEntry struct {
//GORM attributes, see: http://gorm.io/docs/conventions.html
gorm.Model
SettingKeyName string `json:"setting_key_name" gorm:"unique;not null"`
SettingKeyDescription string `json:"setting_key_description"`
SettingDataType string `json:"setting_data_type"`
SettingValueNumeric int64 `json:"setting_value_numeric"`
SettingValueString string `json:"setting_value_string"`
}
func (s SettingEntry) TableName() string {
return "settings"
}

@ -0,0 +1,35 @@
package models
import "github.com/analogj/scrutiny/webapp/backend/pkg"
// Settings is made up of parsed SettingEntry objects retrieved from the database
type Settings struct {
MetricsNotifyLevel pkg.MetricsNotifyLevel `json:"metrics_notify_level"`
MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics_status_filter_attributes"`
MetricsStatusThreshold pkg.MetricsStatusThreshold `json:"metrics_status_threshold"`
}
func (s *Settings) PopulateFromSettingEntries(entries []SettingEntry) {
for _, entry := range entries {
if entry.SettingKeyName == "metrics.notify.level" {
s.MetricsNotifyLevel = pkg.MetricsNotifyLevel(entry.SettingValueNumeric)
} else if entry.SettingKeyName == "metrics.status.filter_attributes" {
s.MetricsStatusFilterAttributes = pkg.MetricsStatusFilterAttributes(entry.SettingValueNumeric)
} else if entry.SettingKeyName == "metrics.status.threshold" {
s.MetricsStatusThreshold = pkg.MetricsStatusThreshold(entry.SettingValueNumeric)
}
}
}
func (s *Settings) UpdateSettingEntries(entries []SettingEntry) []SettingEntry {
for _, entry := range entries {
if entry.SettingKeyName == "metrics.notify.level" {
entry.SettingValueNumeric = int64(s.MetricsNotifyLevel)
} else if entry.SettingKeyName == "metrics.status.filter_attributes" {
entry.SettingValueNumeric = int64(s.MetricsStatusFilterAttributes)
} else if entry.SettingKeyName == "metrics.status.threshold" {
entry.SettingValueNumeric = int64(s.MetricsStatusThreshold)
}
}
return entries
}

@ -0,0 +1,25 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"net/http"
)
func GetSettings(c *gin.Context) {
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
settings, err := deviceRepo.GetSettings(c)
if err != nil {
logger.Errorln("An error occurred while retrieving settings", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"settings": settings,
})
}

@ -0,0 +1,33 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"net/http"
)
func SaveSettings(c *gin.Context) {
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
var settings models.Settings
err := c.BindJSON(&settings)
if err != nil {
logger.Errorln("Cannot parse updated settings", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
err = deviceRepo.SaveSettings(c, settings)
if err != nil {
logger.Errorln("An error occurred while saving settings", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
})
}

@ -50,6 +50,8 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details
api.DELETE("/device/:wwn", handler.DeleteDevice) //used by UI to delete device api.DELETE("/device/:wwn", handler.DeleteDevice) //used by UI to delete device
api.GET("/settings", handler.GetSettings) //used to get settings
api.POST("/settings", handler.SaveSettings) //used to save settings
} }
} }

Loading…
Cancel
Save