pull/228/head
parent
f569ab6474
commit
7a7771981a
@ -0,0 +1,74 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"github.com/analogj/scrutiny/webapp/backend/pkg"
|
||||||
|
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
|
||||||
|
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
|
)
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// Device
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
//insert device into DB (and update specified columns if device is already registered)
|
||||||
|
// update device fields that may change: (DeviceType, HostID)
|
||||||
|
func (sr *scrutinyRepository) RegisterDevice(ctx context.Context, dev models.Device) error {
|
||||||
|
if err := sr.gormClient.WithContext(ctx).Clauses(clause.OnConflict{
|
||||||
|
Columns: []clause.Column{{Name: "wwn"}},
|
||||||
|
DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}),
|
||||||
|
}).Create(&dev).Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// get a list of all devices (only device metadata, no SMART data)
|
||||||
|
func (sr *scrutinyRepository) GetDevices(ctx context.Context) ([]models.Device, error) {
|
||||||
|
//Get a list of all the active devices.
|
||||||
|
devices := []models.Device{}
|
||||||
|
if err := sr.gormClient.WithContext(ctx).Find(&devices).Error; err != nil {
|
||||||
|
return nil, fmt.Errorf("Could not get device summary from DB: %v", err)
|
||||||
|
}
|
||||||
|
return devices, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// update device (only metadata) from collector
|
||||||
|
func (sr *scrutinyRepository) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) {
|
||||||
|
var device models.Device
|
||||||
|
if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
|
||||||
|
return device, fmt.Errorf("Could not get device from DB: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
//TODO catch GormClient err
|
||||||
|
err := device.UpdateFromCollectorSmartInfo(collectorSmartData)
|
||||||
|
if err != nil {
|
||||||
|
return device, err
|
||||||
|
}
|
||||||
|
return device, sr.gormClient.Model(&device).Updates(device).Error
|
||||||
|
}
|
||||||
|
|
||||||
|
//Update Device Status
|
||||||
|
func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) {
|
||||||
|
var device models.Device
|
||||||
|
if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
|
||||||
|
return device, fmt.Errorf("Could not get device from DB: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
device.DeviceStatus = pkg.Set(device.DeviceStatus, status)
|
||||||
|
return device, sr.gormClient.Model(&device).Updates(device).Error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) {
|
||||||
|
var device models.Device
|
||||||
|
|
||||||
|
fmt.Println("GetDeviceDetails from GORM")
|
||||||
|
|
||||||
|
if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
|
||||||
|
return models.Device{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return device, nil
|
||||||
|
}
|
@ -0,0 +1,122 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"github.com/influxdata/influxdb-client-go/v2/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// Tasks
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) error {
|
||||||
|
weeklyTaskName := "tsk-weekly-aggr"
|
||||||
|
if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: weeklyTaskName}); findErr == nil && len(found) == 0 {
|
||||||
|
//weekly on Sunday at 1:00am
|
||||||
|
_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, weeklyTaskName, sr.DownsampleScript("weekly"), "0 1 * * 0", orgID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
monthlyTaskName := "tsk-monthly-aggr"
|
||||||
|
if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: monthlyTaskName}); findErr == nil && len(found) == 0 {
|
||||||
|
//monthly on first day of the month at 1:30am
|
||||||
|
_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, monthlyTaskName, sr.DownsampleScript("monthly"), "30 1 1 * *", orgID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
yearlyTaskName := "tsk-yearly-aggr"
|
||||||
|
if found, findErr := sr.influxTaskApi.FindTasks(ctx, &api.TaskFilter{Name: yearlyTaskName}); findErr == nil && len(found) == 0 {
|
||||||
|
//yearly on the first day of the year at 2:00am
|
||||||
|
_, err := sr.influxTaskApi.CreateTaskWithCron(ctx, yearlyTaskName, sr.DownsampleScript("yearly"), "0 2 1 1 *", orgID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string {
|
||||||
|
var sourceBucket string // the source of the data
|
||||||
|
var destBucket string // the destination for the aggregated data
|
||||||
|
var rangeStart string
|
||||||
|
var rangeEnd string
|
||||||
|
var aggWindow string
|
||||||
|
switch aggregationType {
|
||||||
|
case "weekly":
|
||||||
|
sourceBucket = sr.appConfig.GetString("web.influxdb.bucket")
|
||||||
|
destBucket = fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
|
||||||
|
rangeStart = "-2w"
|
||||||
|
rangeEnd = "-1w"
|
||||||
|
aggWindow = "1w"
|
||||||
|
case "monthly":
|
||||||
|
sourceBucket = fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))
|
||||||
|
destBucket = fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
|
||||||
|
rangeStart = "-2mo"
|
||||||
|
rangeEnd = "-1mo"
|
||||||
|
aggWindow = "1mo"
|
||||||
|
case "yearly":
|
||||||
|
sourceBucket = fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))
|
||||||
|
destBucket = fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))
|
||||||
|
rangeStart = "-2y"
|
||||||
|
rangeEnd = "-1y"
|
||||||
|
aggWindow = "1y"
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: using "last" function for aggregation. This should eventually be replaced with a more accurate represenation
|
||||||
|
/*
|
||||||
|
import "types"
|
||||||
|
smart_data = from(bucket: sourceBucket)
|
||||||
|
|> range(start: rangeStart, stop: rangeEnd)
|
||||||
|
|> filter(fn: (r) => r["_measurement"] == "smart" )
|
||||||
|
|> group(columns: ["device_wwn", "_field"])
|
||||||
|
|
||||||
|
non_numeric_smart_data = smart_data
|
||||||
|
|> filter(fn: (r) => types.isType(v: r._value, type: "string") or types.isType(v: r._value, type: "bool"))
|
||||||
|
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|
||||||
|
|
||||||
|
numeric_smart_data = smart_data
|
||||||
|
|> filter(fn: (r) => types.isType(v: r._value, type: "int") or types.isType(v: r._value, type: "float"))
|
||||||
|
|> aggregateWindow(every: aggWindow, fn: mean, createEmpty: false)
|
||||||
|
|
||||||
|
union(tables: [non_numeric_smart_data, numeric_smart_data])
|
||||||
|
|> to(bucket: destBucket, org: destOrg)
|
||||||
|
|
||||||
|
*/
|
||||||
|
|
||||||
|
return fmt.Sprintf(`
|
||||||
|
sourceBucket = "%s"
|
||||||
|
rangeStart = %s
|
||||||
|
rangeEnd = %s
|
||||||
|
aggWindow = %s
|
||||||
|
destBucket = "%s"
|
||||||
|
destOrg = "%s"
|
||||||
|
|
||||||
|
from(bucket: sourceBucket)
|
||||||
|
|> range(start: rangeStart, stop: rangeEnd)
|
||||||
|
|> filter(fn: (r) => r["_measurement"] == "smart" )
|
||||||
|
|> group(columns: ["device_wwn", "_field"])
|
||||||
|
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|
||||||
|
|> to(bucket: destBucket, org: destOrg)
|
||||||
|
|
||||||
|
temp_data = from(bucket: sourceBucket)
|
||||||
|
|> range(start: rangeStart, stop: rangeEnd)
|
||||||
|
|> filter(fn: (r) => r["_measurement"] == "temp")
|
||||||
|
|> group(columns: ["device_wwn"])
|
||||||
|
|> toInt()
|
||||||
|
|
||||||
|
temp_data
|
||||||
|
|> aggregateWindow(fn: mean, every: aggWindow)
|
||||||
|
|> to(bucket: destBucket, org: destOrg)
|
||||||
|
`,
|
||||||
|
sourceBucket,
|
||||||
|
rangeStart,
|
||||||
|
rangeEnd,
|
||||||
|
aggWindow,
|
||||||
|
destBucket,
|
||||||
|
sr.appConfig.GetString("web.influxdb.org"),
|
||||||
|
)
|
||||||
|
}
|
@ -0,0 +1,95 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
|
||||||
|
"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
|
||||||
|
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// Temperature Data
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error {
|
||||||
|
if len(collectorSmartData.AtaSctTemperatureHistory.Table) > 0 {
|
||||||
|
|
||||||
|
for ndx, temp := range collectorSmartData.AtaSctTemperatureHistory.Table {
|
||||||
|
|
||||||
|
minutesOffset := collectorSmartData.AtaSctTemperatureHistory.LoggingIntervalMinutes * int64(ndx) * 60
|
||||||
|
smartTemp := measurements.SmartTemperature{
|
||||||
|
Date: time.Unix(collectorSmartData.LocalTime.TimeT-minutesOffset, 0),
|
||||||
|
Temp: temp,
|
||||||
|
}
|
||||||
|
|
||||||
|
tags, fields := smartTemp.Flatten()
|
||||||
|
tags["device_wwn"] = wwn
|
||||||
|
p := influxdb2.NewPoint("temp",
|
||||||
|
tags,
|
||||||
|
fields,
|
||||||
|
smartTemp.Date)
|
||||||
|
err := sr.influxWriteApi.WritePoint(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// also add the current temperature.
|
||||||
|
} else {
|
||||||
|
|
||||||
|
smartTemp := measurements.SmartTemperature{
|
||||||
|
Date: time.Unix(collectorSmartData.LocalTime.TimeT, 0),
|
||||||
|
Temp: collectorSmartData.Temperature.Current,
|
||||||
|
}
|
||||||
|
|
||||||
|
tags, fields := smartTemp.Flatten()
|
||||||
|
tags["device_wwn"] = wwn
|
||||||
|
p := influxdb2.NewPoint("temp",
|
||||||
|
tags,
|
||||||
|
fields,
|
||||||
|
smartTemp.Date)
|
||||||
|
return sr.influxWriteApi.WritePoint(ctx, p)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) {
|
||||||
|
//we can get temp history for "week", "month", DURATION_KEY_YEAR, "forever"
|
||||||
|
|
||||||
|
deviceTempHistory := map[string][]measurements.SmartTemperature{}
|
||||||
|
|
||||||
|
//TODO: change the query range to a variable.
|
||||||
|
queryStr := sr.aggregateTempQuery(durationKey)
|
||||||
|
|
||||||
|
result, err := sr.influxQueryApi.Query(ctx, queryStr)
|
||||||
|
if err == nil {
|
||||||
|
// Use Next() to iterate over query result lines
|
||||||
|
for result.Next() {
|
||||||
|
|
||||||
|
if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
|
||||||
|
|
||||||
|
//check if deviceWWN has been seen and initialized already
|
||||||
|
if _, ok := deviceTempHistory[deviceWWN.(string)]; !ok {
|
||||||
|
deviceTempHistory[deviceWWN.(string)] = []measurements.SmartTemperature{}
|
||||||
|
}
|
||||||
|
|
||||||
|
currentTempHistory := deviceTempHistory[deviceWWN.(string)]
|
||||||
|
smartTemp := measurements.SmartTemperature{}
|
||||||
|
|
||||||
|
for key, val := range result.Record().Values() {
|
||||||
|
smartTemp.Inflate(key, val)
|
||||||
|
}
|
||||||
|
smartTemp.Date = result.Record().Values()["_time"].(time.Time)
|
||||||
|
currentTempHistory = append(currentTempHistory, smartTemp)
|
||||||
|
deviceTempHistory[deviceWWN.(string)] = currentTempHistory
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result.Err() != nil {
|
||||||
|
fmt.Printf("Query error: %s\n", result.Err().Error())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return deviceTempHistory, nil
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in new issue