diff --git a/.gitignore b/.gitignore index 6dad40c..edb4def 100644 --- a/.gitignore +++ b/.gitignore @@ -65,4 +65,6 @@ scrutiny_test.db scrutiny.yaml coverage.txt /config -/influxdb \ No newline at end of file +/influxdb +.angular +web.log \ No newline at end of file diff --git a/webapp/backend/pkg/database/helpers.go b/webapp/backend/pkg/database/helpers.go deleted file mode 100644 index 3706d86..0000000 --- a/webapp/backend/pkg/database/helpers.go +++ /dev/null @@ -1,12 +0,0 @@ -package database - -import ( - "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" - "sort" -) - -func sortSmartMeasurementsDesc(smartResults []measurements.Smart) { - sort.SliceStable(smartResults, func(i, j int) bool { - return smartResults[i].Date.After(smartResults[j].Date) - }) -} diff --git a/webapp/backend/pkg/database/helpers_test.go b/webapp/backend/pkg/database/helpers_test.go deleted file mode 100644 index 38587b1..0000000 --- a/webapp/backend/pkg/database/helpers_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package database - -import ( - "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" - "github.com/stretchr/testify/require" - "testing" - "time" -) - -func Test_sortSmartMeasurementsDesc_LatestFirst(t *testing.T) { - //setup - timeNow := time.Now() - smartResults := []measurements.Smart{ - { - Date: timeNow.AddDate(0, 0, -2), - }, - { - Date: timeNow, - }, - { - Date: timeNow.AddDate(0, 0, -1), - }, - } - - //test - sortSmartMeasurementsDesc(smartResults) - - //assert - require.Equal(t, smartResults[0].Date, timeNow) -} diff --git a/webapp/backend/pkg/database/interface.go b/webapp/backend/pkg/database/interface.go index f5dae30..8613eae 100644 --- a/webapp/backend/pkg/database/interface.go +++ b/webapp/backend/pkg/database/interface.go @@ -2,12 +2,15 @@ package database import ( "context" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" ) +// Create mock using: +// mockgen -source=webapp/backend/pkg/database/interface.go -destination=webapp/backend/pkg/database/mock/mock_database.go type DeviceRepo interface { Close() error HealthCheck(ctx context.Context) error @@ -20,7 +23,7 @@ type DeviceRepo interface { DeleteDevice(ctx context.Context, wwn string) error SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) - GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) + GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error diff --git a/webapp/backend/pkg/database/mock/mock_database.go b/webapp/backend/pkg/database/mock/mock_database.go new file mode 100644 index 0000000..c001d08 --- /dev/null +++ b/webapp/backend/pkg/database/mock/mock_database.go @@ -0,0 +1,258 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: webapp/backend/pkg/database/interface.go + +// Package mock_database is a generated GoMock package. +package mock_database + +import ( + context "context" + reflect "reflect" + + pkg "github.com/analogj/scrutiny/webapp/backend/pkg" + models "github.com/analogj/scrutiny/webapp/backend/pkg/models" + collector "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + measurements "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" + gomock "github.com/golang/mock/gomock" +) + +// MockDeviceRepo is a mock of DeviceRepo interface. +type MockDeviceRepo struct { + ctrl *gomock.Controller + recorder *MockDeviceRepoMockRecorder +} + +// MockDeviceRepoMockRecorder is the mock recorder for MockDeviceRepo. +type MockDeviceRepoMockRecorder struct { + mock *MockDeviceRepo +} + +// NewMockDeviceRepo creates a new mock instance. +func NewMockDeviceRepo(ctrl *gomock.Controller) *MockDeviceRepo { + mock := &MockDeviceRepo{ctrl: ctrl} + mock.recorder = &MockDeviceRepoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDeviceRepo) EXPECT() *MockDeviceRepoMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockDeviceRepo) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockDeviceRepoMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockDeviceRepo)(nil).Close)) +} + +// DeleteDevice mocks base method. +func (m *MockDeviceRepo) DeleteDevice(ctx context.Context, wwn string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteDevice", ctx, wwn) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteDevice indicates an expected call of DeleteDevice. +func (mr *MockDeviceRepoMockRecorder) DeleteDevice(ctx, wwn interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDevice", reflect.TypeOf((*MockDeviceRepo)(nil).DeleteDevice), ctx, wwn) +} + +// GetDeviceDetails mocks base method. +func (m *MockDeviceRepo) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDeviceDetails", ctx, wwn) + ret0, _ := ret[0].(models.Device) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDeviceDetails indicates an expected call of GetDeviceDetails. +func (mr *MockDeviceRepoMockRecorder) GetDeviceDetails(ctx, wwn interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceDetails", reflect.TypeOf((*MockDeviceRepo)(nil).GetDeviceDetails), ctx, wwn) +} + +// GetDevices mocks base method. +func (m *MockDeviceRepo) GetDevices(ctx context.Context) ([]models.Device, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDevices", ctx) + ret0, _ := ret[0].([]models.Device) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDevices indicates an expected call of GetDevices. +func (mr *MockDeviceRepoMockRecorder) GetDevices(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDevices", reflect.TypeOf((*MockDeviceRepo)(nil).GetDevices), ctx) +} + +// GetSmartAttributeHistory mocks base method. +func (m *MockDeviceRepo) GetSmartAttributeHistory(ctx context.Context, wwn, durationKey string, selectEntries, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSmartAttributeHistory", ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes) + ret0, _ := ret[0].([]measurements.Smart) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSmartAttributeHistory indicates an expected call of GetSmartAttributeHistory. +func (mr *MockDeviceRepoMockRecorder) GetSmartAttributeHistory(ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSmartAttributeHistory", reflect.TypeOf((*MockDeviceRepo)(nil).GetSmartAttributeHistory), ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes) +} + +// GetSmartTemperatureHistory mocks base method. +func (m *MockDeviceRepo) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSmartTemperatureHistory", ctx, durationKey) + ret0, _ := ret[0].(map[string][]measurements.SmartTemperature) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSmartTemperatureHistory indicates an expected call of GetSmartTemperatureHistory. +func (mr *MockDeviceRepoMockRecorder) GetSmartTemperatureHistory(ctx, durationKey interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSmartTemperatureHistory", reflect.TypeOf((*MockDeviceRepo)(nil).GetSmartTemperatureHistory), ctx, durationKey) +} + +// GetSummary mocks base method. +func (m *MockDeviceRepo) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSummary", ctx) + ret0, _ := ret[0].(map[string]*models.DeviceSummary) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSummary indicates an expected call of GetSummary. +func (mr *MockDeviceRepoMockRecorder) GetSummary(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSummary", reflect.TypeOf((*MockDeviceRepo)(nil).GetSummary), ctx) +} + +// HealthCheck mocks base method. +func (m *MockDeviceRepo) HealthCheck(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HealthCheck", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// HealthCheck indicates an expected call of HealthCheck. +func (mr *MockDeviceRepoMockRecorder) HealthCheck(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HealthCheck", reflect.TypeOf((*MockDeviceRepo)(nil).HealthCheck), ctx) +} + +// LoadSettings mocks base method. +func (m *MockDeviceRepo) LoadSettings(ctx context.Context) (*models.Settings, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadSettings", ctx) + ret0, _ := ret[0].(*models.Settings) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadSettings indicates an expected call of LoadSettings. +func (mr *MockDeviceRepoMockRecorder) LoadSettings(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadSettings", reflect.TypeOf((*MockDeviceRepo)(nil).LoadSettings), ctx) +} + +// RegisterDevice mocks base method. +func (m *MockDeviceRepo) RegisterDevice(ctx context.Context, dev models.Device) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RegisterDevice", ctx, dev) + ret0, _ := ret[0].(error) + return ret0 +} + +// RegisterDevice indicates an expected call of RegisterDevice. +func (mr *MockDeviceRepoMockRecorder) RegisterDevice(ctx, dev interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterDevice", reflect.TypeOf((*MockDeviceRepo)(nil).RegisterDevice), ctx, dev) +} + +// SaveSettings mocks base method. +func (m *MockDeviceRepo) SaveSettings(ctx context.Context, settings models.Settings) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveSettings", ctx, settings) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveSettings indicates an expected call of SaveSettings. +func (mr *MockDeviceRepoMockRecorder) SaveSettings(ctx, settings interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSettings", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSettings), ctx, settings) +} + +// SaveSmartAttributes mocks base method. +func (m *MockDeviceRepo) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveSmartAttributes", ctx, wwn, collectorSmartData) + ret0, _ := ret[0].(measurements.Smart) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SaveSmartAttributes indicates an expected call of SaveSmartAttributes. +func (mr *MockDeviceRepoMockRecorder) SaveSmartAttributes(ctx, wwn, collectorSmartData interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartAttributes", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartAttributes), ctx, wwn, collectorSmartData) +} + +// SaveSmartTemperature mocks base method. +func (m *MockDeviceRepo) SaveSmartTemperature(ctx context.Context, wwn, deviceProtocol string, collectorSmartData collector.SmartInfo) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveSmartTemperature", ctx, wwn, deviceProtocol, collectorSmartData) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveSmartTemperature indicates an expected call of SaveSmartTemperature. +func (mr *MockDeviceRepoMockRecorder) SaveSmartTemperature(ctx, wwn, deviceProtocol, collectorSmartData interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartTemperature", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartTemperature), ctx, wwn, deviceProtocol, collectorSmartData) +} + +// UpdateDevice mocks base method. +func (m *MockDeviceRepo) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateDevice", ctx, wwn, collectorSmartData) + ret0, _ := ret[0].(models.Device) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateDevice indicates an expected call of UpdateDevice. +func (mr *MockDeviceRepoMockRecorder) UpdateDevice(ctx, wwn, collectorSmartData interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDevice", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDevice), ctx, wwn, collectorSmartData) +} + +// UpdateDeviceStatus mocks base method. +func (m *MockDeviceRepo) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateDeviceStatus", ctx, wwn, status) + ret0, _ := ret[0].(models.Device) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateDeviceStatus indicates an expected call of UpdateDeviceStatus. +func (mr *MockDeviceRepoMockRecorder) UpdateDeviceStatus(ctx, wwn, status interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceStatus", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceStatus), ctx, wwn, status) +} diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go index 96015bb..b1abf6c 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go @@ -3,13 +3,14 @@ package database import ( "context" "fmt" + "strings" + "time" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" influxdb2 "github.com/influxdata/influxdb-client-go/v2" "github.com/influxdata/influxdb-client-go/v2/api" log "github.com/sirupsen/logrus" - "strings" - "time" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -30,14 +31,17 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin } // GetSmartAttributeHistory MUST return in sorted order, where newest entries are at the beginning of the list, and oldest are at the end. -func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) { +// When selectEntries is > 0, only the most recent selectEntries database entries are returned, starting from the selectEntriesOffset entry. +// For example, with selectEntries = 5, selectEntries = 0, the most recent 5 are returned. With selectEntries = 3, selectEntries = 2, entries +// 2 to 4 are returned (2 being the third newest, since it is zero-indexed) +func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) { // Get SMartResults from InfluxDB //TODO: change the filter startrange to a real number. // Get parser flux query result //appConfig.GetString("web.influxdb.bucket") - queryStr := sr.aggregateSmartAttributesQuery(wwn, durationKey) + queryStr := sr.aggregateSmartAttributesQuery(wwn, durationKey, selectEntries, selectEntriesOffset, attributes) log.Infoln(queryStr) smartResults := []measurements.Smart{} @@ -65,9 +69,6 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn return nil, err } - //we have to sort the smartResults again, because the `union` command will return multiple 'tables' and only sort the records in each table. - sortSmartMeasurementsDesc(smartResults) - return smartResults, nil //if err := device.SquashHistory(); err != nil { @@ -99,7 +100,7 @@ func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, return influxWriteApi.WritePoint(ctx, p) } -func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string) string { +func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string { /* @@ -108,28 +109,34 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration |> range(start: -1w, stop: now()) |> filter(fn: (r) => r["_measurement"] == "smart" ) |> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" ) + |> tail(n: 10, offset: 0) |> schema.fieldsAsCols() monthData = from(bucket: "metrics_weekly") |> range(start: -1mo, stop: -1w) |> filter(fn: (r) => r["_measurement"] == "smart" ) |> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" ) + |> tail(n: 10, offset: 0) |> schema.fieldsAsCols() yearData = from(bucket: "metrics_monthly") |> range(start: -1y, stop: -1mo) |> filter(fn: (r) => r["_measurement"] == "smart" ) |> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" ) + |> tail(n: 10, offset: 0) |> schema.fieldsAsCols() foreverData = from(bucket: "metrics_yearly") |> range(start: -10y, stop: -1y) |> filter(fn: (r) => r["_measurement"] == "smart" ) |> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" ) + |> tail(n: 10, offset: 0) |> schema.fieldsAsCols() union(tables: [weekData, monthData, yearData, foreverData]) - |> sort(columns: ["_time"], desc: false) + |> group() + |> sort(columns: ["_time"], desc: true) + |> tail(n: 6, offset: 4) |> yield(name: "last") */ @@ -140,34 +147,57 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration nestedDurationKeys := sr.lookupNestedDurationKeys(durationKey) + if len(nestedDurationKeys) == 1 { + //there's only one bucket being queried, no need to union, just aggregate the dataset and return + partialQueryStr = append(partialQueryStr, []string{ + sr.generateSmartAttributesSubquery(wwn, nestedDurationKeys[0], selectEntries, selectEntriesOffset, attributes), + fmt.Sprintf(`%sData`, nestedDurationKeys[0]), + `|> sort(columns: ["_time"], desc: true)`, + `|> yield()`, + }...) + return strings.Join(partialQueryStr, "\n") + } + + subQueries := []string{} subQueryNames := []string{} for _, nestedDurationKey := range nestedDurationKeys { - bucketName := sr.lookupBucketName(nestedDurationKey) - durationRange := sr.lookupDuration(nestedDurationKey) - subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey)) - partialQueryStr = append(partialQueryStr, []string{ - fmt.Sprintf(`%sData = from(bucket: "%s")`, nestedDurationKey, bucketName), - fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]), - `|> filter(fn: (r) => r["_measurement"] == "smart" )`, - fmt.Sprintf(`|> filter(fn: (r) => r["device_wwn"] == "%s" )`, wwn), - "|> schema.fieldsAsCols()", - }...) + if selectEntries > 0 { + // We only need the last `n + offset` # of entries from each table to guarantee we can + // get the last `n` # of entries starting from `offset` of the union + subQueries = append(subQueries, sr.generateSmartAttributesSubquery(wwn, nestedDurationKey, selectEntries+selectEntriesOffset, 0, attributes)) + } else { + subQueries = append(subQueries, sr.generateSmartAttributesSubquery(wwn, nestedDurationKey, 0, 0, attributes)) + } + } + partialQueryStr = append(partialQueryStr, subQueries...) + partialQueryStr = append(partialQueryStr, []string{ + fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")), + `|> group()`, + `|> sort(columns: ["_time"], desc: true)`, + }...) + if selectEntries > 0 { + partialQueryStr = append(partialQueryStr, fmt.Sprintf(`|> tail(n: %d, offset: %d)`, selectEntries, selectEntriesOffset)) } + partialQueryStr = append(partialQueryStr, `|> yield(name: "last")`) - if len(subQueryNames) == 1 { - //there's only one bucket being queried, no need to union, just aggregate the dataset and return - partialQueryStr = append(partialQueryStr, []string{ - subQueryNames[0], - `|> yield()`, - }...) - } else { - partialQueryStr = append(partialQueryStr, []string{ - fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")), - `|> sort(columns: ["_time"], desc: false)`, - `|> yield(name: "last")`, - }...) + return strings.Join(partialQueryStr, "\n") +} + +func (sr *scrutinyRepository) generateSmartAttributesSubquery(wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string { + bucketName := sr.lookupBucketName(durationKey) + durationRange := sr.lookupDuration(durationKey) + + partialQueryStr := []string{ + fmt.Sprintf(`%sData = from(bucket: "%s")`, durationKey, bucketName), + fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]), + `|> filter(fn: (r) => r["_measurement"] == "smart" )`, + fmt.Sprintf(`|> filter(fn: (r) => r["device_wwn"] == "%s" )`, wwn), + } + if selectEntries > 0 { + partialQueryStr = append(partialQueryStr, fmt.Sprintf(`|> tail(n: %d, offset: %d)`, selectEntries, selectEntriesOffset)) } + partialQueryStr = append(partialQueryStr, "|> schema.fieldsAsCols()") return strings.Join(partialQueryStr, "\n") } diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 542677f..e1b948b 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "strconv" + "time" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000" @@ -17,8 +20,6 @@ import ( "github.com/influxdata/influxdb-client-go/v2/api/http" log "github.com/sirupsen/logrus" "gorm.io/gorm" - "strconv" - "time" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -369,6 +370,21 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { return tx.Create(&defaultSettings).Error }, }, + { + ID: "m20231123123300", // add repeat_notifications setting. + Migrate: func(tx *gorm.DB) error { + //add repeat_notifications setting default. + var defaultSettings = []m20220716214900.Setting{ + { + SettingKeyName: "metrics.repeat_notifications", + SettingKeyDescription: "Whether to repeat all notifications or just when values change (true | false)", + SettingDataType: "bool", + SettingValueBool: true, + }, + } + return tx.Create(&defaultSettings).Error + }, + }, }) if err := m.Migrate(); err != nil { diff --git a/webapp/backend/pkg/models/measurements/smart_ata_attribute.go b/webapp/backend/pkg/models/measurements/smart_ata_attribute.go index 8b9a3f4..c1ace9c 100644 --- a/webapp/backend/pkg/models/measurements/smart_ata_attribute.go +++ b/webapp/backend/pkg/models/measurements/smart_ata_attribute.go @@ -2,10 +2,11 @@ package measurements import ( "fmt" - "github.com/analogj/scrutiny/webapp/backend/pkg" - "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" "strconv" "strings" + + "github.com/analogj/scrutiny/webapp/backend/pkg" + "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" ) type SmartAtaAttribute struct { @@ -24,6 +25,10 @@ type SmartAtaAttribute struct { FailureRate float64 `json:"failure_rate,omitempty"` } +func (sa *SmartAtaAttribute) GetTransformedValue() int64 { + return sa.TransformedValue +} + func (sa *SmartAtaAttribute) GetStatus() pkg.AttributeStatus { return sa.Status } diff --git a/webapp/backend/pkg/models/measurements/smart_attribute.go b/webapp/backend/pkg/models/measurements/smart_attribute.go index a8de369..d885acf 100644 --- a/webapp/backend/pkg/models/measurements/smart_attribute.go +++ b/webapp/backend/pkg/models/measurements/smart_attribute.go @@ -6,4 +6,5 @@ type SmartAttribute interface { Flatten() (fields map[string]interface{}) Inflate(key string, val interface{}) GetStatus() pkg.AttributeStatus + GetTransformedValue() int64 } diff --git a/webapp/backend/pkg/models/measurements/smart_nvme_attribute.go b/webapp/backend/pkg/models/measurements/smart_nvme_attribute.go index a2c58a1..4e251d0 100644 --- a/webapp/backend/pkg/models/measurements/smart_nvme_attribute.go +++ b/webapp/backend/pkg/models/measurements/smart_nvme_attribute.go @@ -2,9 +2,10 @@ package measurements import ( "fmt" + "strings" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" - "strings" ) type SmartNvmeAttribute struct { @@ -18,6 +19,10 @@ type SmartNvmeAttribute struct { FailureRate float64 `json:"failure_rate,omitempty"` } +func (sa *SmartNvmeAttribute) GetTransformedValue() int64 { + return sa.TransformedValue +} + func (sa *SmartNvmeAttribute) GetStatus() pkg.AttributeStatus { return sa.Status } diff --git a/webapp/backend/pkg/models/measurements/smart_scsci_attribute.go b/webapp/backend/pkg/models/measurements/smart_scsci_attribute.go index 4fd8a36..347a3f6 100644 --- a/webapp/backend/pkg/models/measurements/smart_scsci_attribute.go +++ b/webapp/backend/pkg/models/measurements/smart_scsci_attribute.go @@ -2,9 +2,10 @@ package measurements import ( "fmt" + "strings" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" - "strings" ) type SmartScsiAttribute struct { @@ -18,6 +19,10 @@ type SmartScsiAttribute struct { FailureRate float64 `json:"failure_rate,omitempty"` } +func (sa *SmartScsiAttribute) GetTransformedValue() int64 { + return sa.TransformedValue +} + func (sa *SmartScsiAttribute) GetStatus() pkg.AttributeStatus { return sa.Status } diff --git a/webapp/backend/pkg/models/settings.go b/webapp/backend/pkg/models/settings.go index f5564ef..e564301 100644 --- a/webapp/backend/pkg/models/settings.go +++ b/webapp/backend/pkg/models/settings.go @@ -17,8 +17,9 @@ type Settings struct { LineStroke string `json:"line_stroke" mapstructure:"line_stroke"` Metrics struct { - NotifyLevel int `json:"notify_level" mapstructure:"notify_level"` - StatusFilterAttributes int `json:"status_filter_attributes" mapstructure:"status_filter_attributes"` - StatusThreshold int `json:"status_threshold" mapstructure:"status_threshold"` + NotifyLevel int `json:"notify_level" mapstructure:"notify_level"` + StatusFilterAttributes int `json:"status_filter_attributes" mapstructure:"status_filter_attributes"` + StatusThreshold int `json:"status_threshold" mapstructure:"status_threshold"` + RepeatNotifications bool `json:"repeat_notifications" mapstructure:"repeat_notifications"` } `json:"metrics" mapstructure:"metrics"` } diff --git a/webapp/backend/pkg/models/testdata/helper.go b/webapp/backend/pkg/models/testdata/helper.go index 771557c..5e2c623 100644 --- a/webapp/backend/pkg/models/testdata/helper.go +++ b/webapp/backend/pkg/models/testdata/helper.go @@ -4,13 +4,14 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "io" "io/ioutil" "log" "net/http" "os" "time" + + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" ) func main() { @@ -32,7 +33,7 @@ func main() { log.Fatalf("ERROR %v", err) } defer file.Close() - _, err = SendPostRequest("http://localhost:9090/api/devices/register", file) + _, err = SendPostRequest("http://localhost:8080/api/devices/register", file) if err != nil { log.Fatalf("ERROR %v", err) } @@ -46,7 +47,7 @@ func main() { log.Fatalf("ERROR %v", err) } - _, err = SendPostRequest(fmt.Sprintf("http://localhost:9090/api/device/%s/smart", diskId), smartDataReader) + _, err = SendPostRequest(fmt.Sprintf("http://localhost:8080/api/device/%s/smart", diskId), smartDataReader) if err != nil { log.Fatalf("ERROR %v", err) } diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index 4a1862e..bf927ee 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -15,11 +15,13 @@ import ( "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/config" + "github.com/analogj/scrutiny/webapp/backend/pkg/database" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" "github.com/containrrr/shoutrrr" shoutrrrTypes "github.com/containrrr/shoutrrr/pkg/types" + "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) @@ -30,7 +32,7 @@ const NotifyFailureTypeSmartFailure = "SmartFailure" const NotifyFailureTypeScrutinyFailure = "ScrutinyFailure" // ShouldNotify check if the error Message should be filtered (level mismatch or filtered_attributes) -func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThreshold pkg.MetricsStatusThreshold, statusFilterAttributes pkg.MetricsStatusFilterAttributes) bool { +func ShouldNotify(logger logrus.FieldLogger, device models.Device, smartAttrs measurements.Smart, statusThreshold pkg.MetricsStatusThreshold, statusFilterAttributes pkg.MetricsStatusFilterAttributes, repeatNotifications bool, c *gin.Context, deviceRepo database.DeviceRepo) bool { // 1. check if the device is healthy if device.DeviceStatus == pkg.DeviceStatusPassed { return false @@ -54,52 +56,69 @@ func ShouldNotify(device models.Device, smartAttrs measurements.Smart, statusThr requiredAttrStatus = pkg.AttributeStatusFailedScrutiny } - // 2. check if the attributes that are failing should be filtered (non-critical) - // 3. for any unfiltered attribute, store the failure reason (Smart or Scrutiny) - if statusFilterAttributes == pkg.MetricsStatusFilterAttributesCritical { - hasFailingCriticalAttr := false - var statusFailingCriticalAttr pkg.AttributeStatus - - for attrId, attrData := range smartAttrs.Attributes { - //find failing attribute - if attrData.GetStatus() == pkg.AttributeStatusPassed { - continue //skip all passing attributes - } + // This is the only case where individual attributes need not be considered + if statusFilterAttributes == pkg.MetricsStatusFilterAttributesAll && repeatNotifications { + return pkg.DeviceStatusHas(device.DeviceStatus, requiredDeviceStatus) + } - // merge the status's of all critical attributes - statusFailingCriticalAttr = pkg.AttributeStatusSet(statusFailingCriticalAttr, attrData.GetStatus()) + var failingAttributes []string + // Loop through the attributes to find the failing ones + for attrId, attrData := range smartAttrs.Attributes { + var status pkg.AttributeStatus = attrData.GetStatus() + // Skip over passing attributes + if status == pkg.AttributeStatusPassed { + continue + } - //found a failing attribute, see if its critical - if device.IsScsi() && thresholds.ScsiMetadata[attrId].Critical { - hasFailingCriticalAttr = true - } else if device.IsNvme() && thresholds.NmveMetadata[attrId].Critical { - hasFailingCriticalAttr = true + // If the user only wants to consider critical attributes, we have to check + // if the not-passing attribute is critical or not + if statusFilterAttributes == pkg.MetricsStatusFilterAttributesCritical { + critical := false + if device.IsScsi() { + critical = thresholds.ScsiMetadata[attrId].Critical + } else if device.IsNvme() { + critical = thresholds.NmveMetadata[attrId].Critical } else { //this is ATA attrIdInt, err := strconv.Atoi(attrId) if err != nil { continue } - if thresholds.AtaMetadata[attrIdInt].Critical { - hasFailingCriticalAttr = true - } + critical = thresholds.AtaMetadata[attrIdInt].Critical + } + // Skip non-critical, non-passing attributes when this setting is on + if !critical { + continue } - } - if !hasFailingCriticalAttr { - //no critical attributes are failing, and notifyFilterAttributes == "critical" - return false - } else { - // check if any of the critical attributes have a status that we're looking for - return pkg.AttributeStatusHas(statusFailingCriticalAttr, requiredAttrStatus) - } + // Record any attribute that doesn't get skipped by the above two checks + failingAttributes = append(failingAttributes, attrId) + } - } else { - // 2. SKIP - we are processing every attribute. - // 3. check if the device failure level matches the wanted failure level. - return pkg.DeviceStatusHas(device.DeviceStatus, requiredDeviceStatus) + // If the user doesn't want repeated notifications when the failing value doesn't change, we need to get the last value from the db + var lastPoints []measurements.Smart + var err error + if !repeatNotifications { + lastPoints, err = deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), database.DURATION_KEY_FOREVER, 1, 1, failingAttributes) + if err == nil || len(lastPoints) < 1 { + logger.Warningln("Could not get the most recent data points from the database. This is expected to happen only if this is the very first submission of data for the device.") + } + } + for _, attrId := range failingAttributes { + attrStatus := smartAttrs.Attributes[attrId].GetStatus() + if pkg.AttributeStatusHas(attrStatus, requiredAttrStatus) { + if repeatNotifications { + return true + } + // This is checked again here to avoid repeating the entire for loop in the check above. + // Probably unnoticeably worse performance, but cleaner code. + if err != nil || len(lastPoints) < 1 || lastPoints[0].Attributes[attrId].GetTransformedValue() != smartAttrs.Attributes[attrId].GetTransformedValue() { + return true + } + } } + return false } // TODO: include user label for device. @@ -222,7 +241,7 @@ func (n *Notify) Send() error { notifyScripts := []string{} notifyShoutrrr := []string{} - for ndx, _ := range configUrls { + for ndx := range configUrls { if strings.HasPrefix(configUrls[ndx], "https://") || strings.HasPrefix(configUrls[ndx], "http://") { notifyWebhooks = append(notifyWebhooks, configUrls[ndx]) } else if strings.HasPrefix(configUrls[ndx], "script://") { diff --git a/webapp/backend/pkg/notify/notify_test.go b/webapp/backend/pkg/notify/notify_test.go index c76a924..1c9dc11 100644 --- a/webapp/backend/pkg/notify/notify_test.go +++ b/webapp/backend/pkg/notify/notify_test.go @@ -1,13 +1,20 @@ package notify import ( + "errors" "fmt" + "testing" + "time" + "github.com/analogj/scrutiny/webapp/backend/pkg" + "github.com/analogj/scrutiny/webapp/backend/pkg/database" + mock_database "github.com/analogj/scrutiny/webapp/backend/pkg/database/mock" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" + "github.com/gin-gonic/gin" + "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" - "testing" - "time" ) func TestShouldNotify_MustSkipPassingDevices(t *testing.T) { @@ -20,22 +27,27 @@ func TestShouldNotify_MustSkipPassingDevices(t *testing.T) { statusThreshold := pkg.MetricsStatusThresholdBoth notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) //assert - require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) + require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase)) } func TestShouldNotify_MetricsStatusThresholdBoth_FailingSmartDevice(t *testing.T) { t.Parallel() - //setup + //setupD device := models.Device{ DeviceStatus: pkg.DeviceStatusFailedSmart, } smartAttrs := measurements.Smart{} statusThreshold := pkg.MetricsStatusThresholdBoth notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll - + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) //assert - require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) + require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase)) } func TestShouldNotify_MetricsStatusThresholdSmart_FailingSmartDevice(t *testing.T) { @@ -47,9 +59,11 @@ func TestShouldNotify_MetricsStatusThresholdSmart_FailingSmartDevice(t *testing. smartAttrs := measurements.Smart{} statusThreshold := pkg.MetricsStatusThresholdSmart notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll - + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) //assert - require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) + require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase)) } func TestShouldNotify_MetricsStatusThresholdScrutiny_FailingSmartDevice(t *testing.T) { @@ -61,9 +75,11 @@ func TestShouldNotify_MetricsStatusThresholdScrutiny_FailingSmartDevice(t *testi smartAttrs := measurements.Smart{} statusThreshold := pkg.MetricsStatusThresholdScrutiny notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll - + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) //assert - require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) + require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase)) } func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithCriticalAttrs(t *testing.T) { @@ -79,9 +95,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithCriticalAttrs(t }} statusThreshold := pkg.MetricsStatusThresholdBoth notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) //assert - require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) + require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase)) } func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithMultipleCriticalAttrs(t *testing.T) { @@ -100,9 +119,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithMultipleCritical }} statusThreshold := pkg.MetricsStatusThresholdBoth notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) //assert - require.True(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) + require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase)) } func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoCriticalAttrs(t *testing.T) { @@ -118,9 +140,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoCriticalAttrs( }} statusThreshold := pkg.MetricsStatusThresholdBoth notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) //assert - require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) + require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase)) } func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoFailingCriticalAttrs(t *testing.T) { @@ -136,9 +161,12 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_WithNoFailingCritica }} statusThreshold := pkg.MetricsStatusThresholdBoth notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) //assert - require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) + require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase)) } func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresholdSmart_WithCriticalAttrsFailingScrutiny(t *testing.T) { @@ -157,9 +185,77 @@ func TestShouldNotify_MetricsStatusFilterAttributesCritical_MetricsStatusThresho }} statusThreshold := pkg.MetricsStatusThresholdSmart notifyFilterAttributes := pkg.MetricsStatusFilterAttributesCritical + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) + + //assert + require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, true, &gin.Context{}, fakeDatabase)) +} +func TestShouldNotify_NoRepeat_DatabaseFailure(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedScrutiny, + } + smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{ + "5": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusFailedScrutiny, + }, + }} + statusThreshold := pkg.MetricsStatusThresholdBoth + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) + fakeDatabase.EXPECT().GetSmartAttributeHistory(&gin.Context{}, "", database.DURATION_KEY_FOREVER, 1, 1, []string{"5"}).Return([]measurements.Smart{}, errors.New("")).Times(1) + + //assert + require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, false, &gin.Context{}, fakeDatabase)) +} + +func TestShouldNotify_NoRepeat_NoDatabaseData(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedScrutiny, + } + smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{ + "5": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusFailedScrutiny, + }, + }} + statusThreshold := pkg.MetricsStatusThresholdBoth + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) + fakeDatabase.EXPECT().GetSmartAttributeHistory(&gin.Context{}, "", database.DURATION_KEY_FOREVER, 1, 1, []string{"5"}).Return([]measurements.Smart{}, nil).Times(1) + + //assert + require.True(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, false, &gin.Context{}, fakeDatabase)) +} +func TestShouldNotify_NoRepeat(t *testing.T) { + t.Parallel() + //setup + device := models.Device{ + DeviceStatus: pkg.DeviceStatusFailedScrutiny, + } + smartAttrs := measurements.Smart{Attributes: map[string]measurements.SmartAttribute{ + "5": &measurements.SmartAtaAttribute{ + Status: pkg.AttributeStatusFailedScrutiny, + TransformedValue: 0, + }, + }} + statusThreshold := pkg.MetricsStatusThresholdBoth + notifyFilterAttributes := pkg.MetricsStatusFilterAttributesAll + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeDatabase := mock_database.NewMockDeviceRepo(mockCtrl) + fakeDatabase.EXPECT().GetSmartAttributeHistory(&gin.Context{}, "", database.DURATION_KEY_FOREVER, 1, 1, []string{"5"}).Return([]measurements.Smart{smartAttrs}, nil).Times(1) //assert - require.False(t, ShouldNotify(device, smartAttrs, statusThreshold, notifyFilterAttributes)) + require.False(t, ShouldNotify(logrus.StandardLogger(), device, smartAttrs, statusThreshold, notifyFilterAttributes, false, &gin.Context{}, fakeDatabase)) } func TestNewPayload(t *testing.T) { diff --git a/webapp/backend/pkg/web/handler/get_device_details.go b/webapp/backend/pkg/web/handler/get_device_details.go index b4e24ee..49e48a1 100644 --- a/webapp/backend/pkg/web/handler/get_device_details.go +++ b/webapp/backend/pkg/web/handler/get_device_details.go @@ -1,11 +1,12 @@ package handler import ( + "net/http" + "github.com/analogj/scrutiny/webapp/backend/pkg/database" "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" - "net/http" ) func GetDeviceDetails(c *gin.Context) { @@ -24,7 +25,7 @@ func GetDeviceDetails(c *gin.Context) { durationKey = "forever" } - smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), durationKey, nil) + smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), durationKey, 0, 0, nil) if err != nil { logger.Errorln("An error occurred while retrieving device smart results", err) c.JSON(http.StatusInternalServerError, gin.H{"success": false}) diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index f58d6ed..0814433 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -2,6 +2,8 @@ package handler import ( "fmt" + "net/http" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/database" @@ -9,7 +11,6 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/notify" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" - "net/http" ) func UploadDeviceMetrics(c *gin.Context) { @@ -69,10 +70,14 @@ func UploadDeviceMetrics(c *gin.Context) { //check for error if notify.ShouldNotify( + logger, updatedDevice, smartData, pkg.MetricsStatusThreshold(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_threshold", config.DB_USER_SETTINGS_SUBKEY))), pkg.MetricsStatusFilterAttributes(appConfig.GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY))), + appConfig.GetBool(fmt.Sprintf("%s.metrics.repeat_notifications", config.DB_USER_SETTINGS_SUBKEY)), + c, + deviceRepo, ) { //send notifications diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index ac775a0..bd2407c 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -4,6 +4,16 @@ import ( "bytes" "encoding/json" "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path" + "strings" + "testing" + "time" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/config" mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock" @@ -14,15 +24,6 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "path" - "strings" - "testing" - "time" ) /* @@ -189,6 +190,7 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() { fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() + fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { @@ -247,6 +249,7 @@ func (suite *ServerTestSuite) TestPopulateMultiple() { fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() + fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { @@ -529,6 +532,7 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() { fakeConfig.EXPECT().GetString("web.influxdb.token").Return("my-super-secret-auth-token").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() + fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{}) diff --git a/webapp/frontend/src/app/core/config/app.config.ts b/webapp/frontend/src/app/core/config/app.config.ts index c48d25a..1b6dfe3 100644 --- a/webapp/frontend/src/app/core/config/app.config.ts +++ b/webapp/frontend/src/app/core/config/app.config.ts @@ -55,6 +55,7 @@ export interface AppConfig { notify_level?: MetricsNotifyLevel status_filter_attributes?: MetricsStatusFilterAttributes status_threshold?: MetricsStatusThreshold + repeat_notifications?: boolean } } @@ -82,7 +83,8 @@ export const appConfig: AppConfig = { metrics: { notify_level: MetricsNotifyLevel.Fail, status_filter_attributes: MetricsStatusFilterAttributes.All, - status_threshold: MetricsStatusThreshold.Both + status_threshold: MetricsStatusThreshold.Both, + repeat_notifications: true } }; diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html index 69ebd76..0eb9a03 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html @@ -84,6 +84,16 @@ + +
+ + Repeat Notifications + + Always + Only when the value has changed + + +
diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts index ea2acce..94f262f 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts @@ -28,6 +28,7 @@ export class DashboardSettingsComponent implements OnInit { theme: string; statusThreshold: number; statusFilterAttributes: number; + repeatNotifications: boolean; // Private private _unsubscribeAll: Subject; @@ -55,6 +56,7 @@ export class DashboardSettingsComponent implements OnInit { this.statusFilterAttributes = config.metrics.status_filter_attributes; this.statusThreshold = config.metrics.status_threshold; + this.repeatNotifications = config.metrics.repeat_notifications; }); @@ -70,7 +72,8 @@ export class DashboardSettingsComponent implements OnInit { theme: this.theme as Theme, metrics: { status_filter_attributes: this.statusFilterAttributes as MetricsStatusFilterAttributes, - status_threshold: this.statusThreshold as MetricsStatusThreshold + status_threshold: this.statusThreshold as MetricsStatusThreshold, + repeat_notifications: this.repeatNotifications } } this._configService.config = newSettings