diff --git a/docs/INSTALL_NAS.md.go b/docs/INSTALL_NAS.md similarity index 100% rename from docs/INSTALL_NAS.md.go rename to docs/INSTALL_NAS.md diff --git a/docs/dbdiagram.io.txt b/docs/dbdiagram.io.txt index 7d23af7..23265ad 100644 --- a/docs/dbdiagram.io.txt +++ b/docs/dbdiagram.io.txt @@ -1,62 +1,88 @@ // SQLite Table(s) -Table device { - created_at timestamp - - wwn varchar [pk] - - //user provided - label varchar - host_id varchar - - // smartctl provided - device_name varchar - manufacturer varchar - model_name varchar - interface_type varchar - interface_speed varchar - serial_number varchar - firmware varchar - rotational_speed varchar - capacity varchar - form_factor varchar - smart_support varchar - device_protocol varchar - device_type varchar +Table Device { + //GORM attributes, see: http://gorm.io/docs/conventions.html + CreatedAt time + UpdatedAt time + DeletedAt time + + WWN string + + DeviceName string + DeviceUUID string + DeviceSerialID string + DeviceLabel string + + Manufacturer string + ModelName string + InterfaceType string + InterfaceSpeed string + SerialNumber string + Firmware string + RotationSpeed int + Capacity int64 + FormFactor string + SmartSupport bool + DeviceProtocol string//protocol determines which smart attribute types are available (ATA, NVMe, SCSI) + DeviceType string//device type is used for querying with -d/t flag, should only be used by collector. + + // User provided metadata + Label string + HostId string + + // Data set by Scrutiny + DeviceStatus enum } +Table Setting { + //GORM attributes, see: http://gorm.io/docs/conventions.html -// InfluxDB Tables -Table device_temperature { - //timestamp - created_at timestamp - - //tags (indexed & queryable) - device_wwn varchar [pk] - - //fields - temp bigint - } - + SettingKeyName string + SettingKeyDescription string + SettingDataType string -Table smart_ata_results { - //timestamp - created_at timestamp - - //tags (indexed & queryable) - device_wwn varchar [pk] - smart_status varchar - scrutiny_status varchar + SettingValueNumeric int64 + SettingValueString string +} +// InfluxDB Tables +Table SmartTemperature { + Date time + DeviceWWN string //(tag) + Temp int64 +} - //fields - temp bigint - power_on_hours bigint - power_cycle_count bigint +Table Smart { + Date time + DeviceWWN string //(tag) + DeviceProtocol string + + //Metrics (fields) + Temp int64 + PowerOnHours int64 + PowerCycleCount int64 + + //Smart Status + Status enum + + //SMART Attributes (fields) + Attr_ID_AttributeId int + Attr_ID_Value int64 + Attr_ID_Threshold int64 + Attr_ID_Worst int64 + Attr_ID_RawValue int64 + Attr_ID_RawString string + Attr_ID_WhenFailed string + //Generated data + Attr_ID_TransformedValue int64 + Attr_ID_Status enum + Attr_ID_StatusReason string + Attr_ID_FailureRate float64 } -Ref: device.wwn < smart_ata_results.device_wwn +Ref: Device.WWN < Smart.DeviceWWN +Ref: Device.WWN < SmartTemperature.DeviceWWN diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go index 5f5d4cc..3a8cf52 100644 --- a/webapp/backend/pkg/config/config.go +++ b/webapp/backend/pkg/config/config.go @@ -2,7 +2,6 @@ package config import ( "github.com/analogj/go-util/utils" - "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/errors" "github.com/spf13/viper" "log" @@ -39,8 +38,6 @@ func (c *configuration) Init() error { c.SetDefault("log.file", "") c.SetDefault("notify.urls", []string{}) - c.SetDefault("notify.filter_attributes", pkg.NotifyFilterAttributesAll) - c.SetDefault("notify.level", pkg.NotifyLevelFail) c.SetDefault("web.influxdb.scheme", "http") c.SetDefault("web.influxdb.host", "localhost") @@ -55,17 +52,6 @@ func (c *configuration) Init() error { //c.SetDefault("disks.include", []string{}) //c.SetDefault("disks.exclude", []string{}) - //c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh") - //c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh") - //c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh") - - //c.SetDefault("collect.metric.enable", true) - //c.SetDefault("collect.metric.command", "-a -o on -S on") - //c.SetDefault("collect.long.enable", true) - //c.SetDefault("collect.long.command", "-a -o on -S on") - //c.SetDefault("collect.short.enable", true) - //c.SetDefault("collect.short.command", "-a -o on -S on") - //if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig c.SetConfigType("yaml") //c.SetConfigName("drawbridge") @@ -77,7 +63,7 @@ func (c *configuration) Init() error { c.AutomaticEnv() //CLI options will be added via the `Set()` function - return nil + return c.ValidateConfig() } func (c *configuration) ReadConfig(configFilePath string) error { @@ -120,24 +106,18 @@ func (c *configuration) ReadConfig(configFilePath string) error { // This function ensures that the merged config works correctly. func (c *configuration) ValidateConfig() error { - ////deserialize Questions - //questionsMap := map[string]Question{} - //err := c.UnmarshalKey("questions", &questionsMap) - // - //if err != nil { - // log.Printf("questions could not be deserialized correctly. %v", err) - // return err - //} - // - //for _, v := range questionsMap { - // - // typeContent, ok := v.Schema["type"].(string) - // if !ok || len(typeContent) == 0 { - // return errors.QuestionSyntaxError("`type` is required for questions") - // } - //} - // - // + //the following keys are deprecated, and no longer supported + /* + - notify.filter_attributes (replaced by metrics.status.filter_attributes SETTING) + - notify.level (replaced by metrics.notify.level and metrics.status.threshold SETTING) + */ + //TODO add docs and upgrade doc. + if c.IsSet("notify.filter_attributes") { + return errors.ConfigValidationError("`notify.filter_attributes` configuration option is deprecated. Replaced by option in Dashboard Settings page") + } + if c.IsSet("notify.level") { + return errors.ConfigValidationError("`notify.level` configuration option is deprecated. Replaced by option in Dashboard Settings page") + } return nil } diff --git a/webapp/backend/pkg/constants.go b/webapp/backend/pkg/constants.go index 9535963..f0a4b9d 100644 --- a/webapp/backend/pkg/constants.go +++ b/webapp/backend/pkg/constants.go @@ -4,17 +4,11 @@ const DeviceProtocolAta = "ATA" const DeviceProtocolScsi = "SCSI" const DeviceProtocolNvme = "NVMe" -const NotifyFilterAttributesAll = "all" -const NotifyFilterAttributesCritical = "critical" - -const NotifyLevelFail = "fail" -const NotifyLevelFailScrutiny = "fail_scrutiny" -const NotifyLevelFailSmart = "fail_smart" - //go:generate stringer -type=AttributeStatus +// AttributeStatus bitwise flag, 1,2,4,8,16,32,etc type AttributeStatus uint8 + const ( - // AttributeStatusPassed binary, 1,2,4,8,16,32,etc AttributeStatusPassed AttributeStatus = 0 AttributeStatusFailedSmart AttributeStatus = 1 AttributeStatusWarningScrutiny AttributeStatus = 2 @@ -30,9 +24,10 @@ func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ func AttributeStatusHas(b, flag AttributeStatus) bool { return b&flag != 0 } //go:generate stringer -type=DeviceStatus +// DeviceStatus bitwise flag, 1,2,4,8,16,32,etc type DeviceStatus uint8 + const ( - // DeviceStatusPassed binary, 1,2,4,8,16,32,etc DeviceStatusPassed DeviceStatus = 0 DeviceStatusFailedSmart DeviceStatus = 1 DeviceStatusFailedScrutiny DeviceStatus = 2 @@ -42,3 +37,44 @@ func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus { return b | flag } func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus { return b &^ flag } func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag } func DeviceStatusHas(b, flag DeviceStatus) bool { return b&flag != 0 } + +// Metrics Specific Filtering & Threshold Constants +type MetricsNotifyLevel int64 + +const ( + MetricsNotifyLevelWarn MetricsNotifyLevel = 1 + MetricsNotifyLevelFail MetricsNotifyLevel = 2 +) + +type MetricsStatusFilterAttributes int64 + +const ( + MetricsStatusFilterAttributesAll MetricsStatusFilterAttributes = 0 + MetricsStatusFilterAttributesCritical MetricsStatusFilterAttributes = 1 +) + +// MetricsStatusThreshold bitwise flag, 1,2,4,8,16,32,etc +type MetricsStatusThreshold int64 + +const ( + MetricsStatusThresholdSmart MetricsStatusThreshold = 1 + MetricsStatusThresholdScrutiny MetricsStatusThreshold = 2 + + //shortcut + MetricsStatusThresholdBoth MetricsStatusThreshold = 3 +) + +// Deprecated +const NotifyFilterAttributesAll = "all" + +// Deprecated +const NotifyFilterAttributesCritical = "critical" + +// Deprecated +const NotifyLevelFail = "fail" + +// Deprecated +const NotifyLevelFailScrutiny = "fail_scrutiny" + +// Deprecated +const NotifyLevelFailSmart = "fail_smart" diff --git a/webapp/backend/pkg/database/interface.go b/webapp/backend/pkg/database/interface.go index 5ab7084..09bd6d5 100644 --- a/webapp/backend/pkg/database/interface.go +++ b/webapp/backend/pkg/database/interface.go @@ -28,4 +28,7 @@ type DeviceRepo interface { GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) + + GetSettings(ctx context.Context) (*models.Settings, error) + SaveSettings(ctx context.Context, settings models.Settings) error } diff --git a/webapp/backend/pkg/database/migrations/m20220716214900/setting.go b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go index ba35a71..8e38845 100644 --- a/webapp/backend/pkg/database/migrations/m20220716214900/setting.go +++ b/webapp/backend/pkg/database/migrations/m20220716214900/setting.go @@ -8,8 +8,9 @@ type Setting struct { //GORM attributes, see: http://gorm.io/docs/conventions.html gorm.Model - SettingKeyName string `json:"setting_key_name"` - SettingDataType string `json:"setting_data_type"` + SettingKeyName string `json:"setting_key_name"` + SettingKeyDescription string `json:"setting_key_description"` + SettingDataType string `json:"setting_data_type"` SettingValueNumeric int64 `json:"setting_value_numeric"` SettingValueString string `json:"setting_value_string"` diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 6fd3596..2ae6388 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100" @@ -281,7 +282,33 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { Migrate: func(tx *gorm.DB) error { // adding the settings table. - return tx.AutoMigrate(m20220716214900.Setting{}) + err := tx.AutoMigrate(m20220716214900.Setting{}) + if err != nil { + return err + } + //add defaults. + + var defaultSettings = []m20220716214900.Setting{ + { + SettingKeyName: "metrics.notify.level", + SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)", + SettingDataType: "numeric", + SettingValueNumeric: int64(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn' + }, + { + SettingKeyName: "metrics.status.filter_attributes", + SettingKeyDescription: "Determines which attributes should impact device status", + SettingDataType: "numeric", + SettingValueNumeric: int64(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or 'critical' + }, + { + SettingKeyName: "metrics.status.threshold", + SettingKeyDescription: "Determines which threshold should impact device status", + SettingDataType: "string", + SettingValueNumeric: int64(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both' + }, + } + return tx.Create(&defaultSettings).Error }, }, }) diff --git a/webapp/backend/pkg/database/scrutiny_repository_settings.go b/webapp/backend/pkg/database/scrutiny_repository_settings.go new file mode 100644 index 0000000..05b7e72 --- /dev/null +++ b/webapp/backend/pkg/database/scrutiny_repository_settings.go @@ -0,0 +1,33 @@ +package database + +import ( + "context" + "fmt" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" +) + +func (sr *scrutinyRepository) GetSettings(ctx context.Context) (*models.Settings, error) { + settingsEntries := []models.SettingEntry{} + if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil { + return nil, fmt.Errorf("Could not get settings from DB: %v", err) + } + + settings := models.Settings{} + settings.PopulateFromSettingEntries(settingsEntries) + + return &settings, nil +} +func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error { + + //get current settings + settingsEntries := []models.SettingEntry{} + if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil { + return fmt.Errorf("Could not get settings from DB: %v", err) + } + + // override with values from settings object + settingsEntries = settings.UpdateSettingEntries(settingsEntries) + + // store in database. + return sr.gormClient.Updates(&settingsEntries).Error +} diff --git a/webapp/backend/pkg/models/setting.go b/webapp/backend/pkg/models/setting.go deleted file mode 100644 index 15238f6..0000000 --- a/webapp/backend/pkg/models/setting.go +++ /dev/null @@ -1,16 +0,0 @@ -package models - -import ( - "gorm.io/gorm" -) - -type Setting struct { - //GORM attributes, see: http://gorm.io/docs/conventions.html - gorm.Model - - SettingKeyName string `json:"setting_key_name"` - SettingDataType string `json:"setting_data_type"` - - SettingValueNumeric int64 `json:"setting_value_numeric"` - SettingValueString string `json:"setting_value_string"` -} diff --git a/webapp/backend/pkg/models/setting_entry.go b/webapp/backend/pkg/models/setting_entry.go new file mode 100644 index 0000000..b0f89b3 --- /dev/null +++ b/webapp/backend/pkg/models/setting_entry.go @@ -0,0 +1,22 @@ +package models + +import ( + "gorm.io/gorm" +) + +// SettingEntry matches a setting row in the database +type SettingEntry struct { + //GORM attributes, see: http://gorm.io/docs/conventions.html + gorm.Model + + SettingKeyName string `json:"setting_key_name" gorm:"unique;not null"` + SettingKeyDescription string `json:"setting_key_description"` + SettingDataType string `json:"setting_data_type"` + + SettingValueNumeric int64 `json:"setting_value_numeric"` + SettingValueString string `json:"setting_value_string"` +} + +func (s SettingEntry) TableName() string { + return "settings" +} diff --git a/webapp/backend/pkg/models/settings.go b/webapp/backend/pkg/models/settings.go new file mode 100644 index 0000000..3cf1431 --- /dev/null +++ b/webapp/backend/pkg/models/settings.go @@ -0,0 +1,35 @@ +package models + +import "github.com/analogj/scrutiny/webapp/backend/pkg" + +// Settings is made up of parsed SettingEntry objects retrieved from the database +type Settings struct { + MetricsNotifyLevel pkg.MetricsNotifyLevel `json:"metrics_notify_level"` + MetricsStatusFilterAttributes pkg.MetricsStatusFilterAttributes `json:"metrics_status_filter_attributes"` + MetricsStatusThreshold pkg.MetricsStatusThreshold `json:"metrics_status_threshold"` +} + +func (s *Settings) PopulateFromSettingEntries(entries []SettingEntry) { + for _, entry := range entries { + if entry.SettingKeyName == "metrics.notify.level" { + s.MetricsNotifyLevel = pkg.MetricsNotifyLevel(entry.SettingValueNumeric) + } else if entry.SettingKeyName == "metrics.status.filter_attributes" { + s.MetricsStatusFilterAttributes = pkg.MetricsStatusFilterAttributes(entry.SettingValueNumeric) + } else if entry.SettingKeyName == "metrics.status.threshold" { + s.MetricsStatusThreshold = pkg.MetricsStatusThreshold(entry.SettingValueNumeric) + } + } +} + +func (s *Settings) UpdateSettingEntries(entries []SettingEntry) []SettingEntry { + for _, entry := range entries { + if entry.SettingKeyName == "metrics.notify.level" { + entry.SettingValueNumeric = int64(s.MetricsNotifyLevel) + } else if entry.SettingKeyName == "metrics.status.filter_attributes" { + entry.SettingValueNumeric = int64(s.MetricsStatusFilterAttributes) + } else if entry.SettingKeyName == "metrics.status.threshold" { + entry.SettingValueNumeric = int64(s.MetricsStatusThreshold) + } + } + return entries +} diff --git a/webapp/backend/pkg/web/handler/get_settings.go b/webapp/backend/pkg/web/handler/get_settings.go new file mode 100644 index 0000000..ca66082 --- /dev/null +++ b/webapp/backend/pkg/web/handler/get_settings.go @@ -0,0 +1,25 @@ +package handler + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/database" + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "net/http" +) + +func GetSettings(c *gin.Context) { + logger := c.MustGet("LOGGER").(logrus.FieldLogger) + deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) + + settings, err := deviceRepo.GetSettings(c) + if err != nil { + logger.Errorln("An error occurred while retrieving settings", err) + c.JSON(http.StatusInternalServerError, gin.H{"success": false}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "settings": settings, + }) +} diff --git a/webapp/backend/pkg/web/handler/save_settings.go b/webapp/backend/pkg/web/handler/save_settings.go new file mode 100644 index 0000000..d466169 --- /dev/null +++ b/webapp/backend/pkg/web/handler/save_settings.go @@ -0,0 +1,33 @@ +package handler + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/database" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "net/http" +) + +func SaveSettings(c *gin.Context) { + logger := c.MustGet("LOGGER").(logrus.FieldLogger) + deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) + + var settings models.Settings + err := c.BindJSON(&settings) + if err != nil { + logger.Errorln("Cannot parse updated settings", err) + c.JSON(http.StatusInternalServerError, gin.H{"success": false}) + return + } + + err = deviceRepo.SaveSettings(c, settings) + if err != nil { + logger.Errorln("An error occurred while saving settings", err) + c.JSON(http.StatusInternalServerError, gin.H{"success": false}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + }) +} diff --git a/webapp/backend/pkg/web/server.go b/webapp/backend/pkg/web/server.go index d2a5cf6..0ef8bc8 100644 --- a/webapp/backend/pkg/web/server.go +++ b/webapp/backend/pkg/web/server.go @@ -50,6 +50,8 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine { api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details api.DELETE("/device/:wwn", handler.DeleteDevice) //used by UI to delete device + api.GET("/settings", handler.GetSettings) //used to get settings + api.POST("/settings", handler.SaveSettings) //used to save settings } }