From b44ef5cb9cf6cbcff70fea3fab5b6dc16adac4dc Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 7 Oct 2020 21:54:29 -0600 Subject: [PATCH 1/3] adding support for a collecto config file. /scrutiny/config/collector.yaml Adding ability to specify host identifier (label), that is updated on every collector run. Can be specified by `host-id` CLI or `COLLECTOR_HOST_ID` env var. Created a config class, interface and associated tests. Created a "TransformDetectedDrives" function, that will allow users to insert drives not detected by Smarctl --scan, ignore drives that they dont want, and override smartctl device type. Added Upsert functionality when registering devices. Replaced "github.com/jinzhu/gorm" with "gorm.io/gorm" (ORM location moved, was using incorrect lib url) Removed machineid library. --- README.md | 2 +- .../collector-metrics/collector-metrics.go | 40 +++- collector/pkg/collector/metrics.go | 6 +- collector/pkg/config/config.go | 100 ++++++++ collector/pkg/config/config_test.go | 64 +++++ collector/pkg/config/factory.go | 9 + collector/pkg/config/interface.go | 26 +++ collector/pkg/config/mock/mock_config.go | 218 ++++++++++++++++++ .../pkg/config/testdata/ignore_device.yaml | 4 + .../pkg/config/testdata/raid_device.yaml | 19 ++ .../pkg/config/testdata/simple_device.yaml | 27 +++ collector/pkg/detect/detect.go | 79 +++++-- collector/pkg/detect/detect_test.go | 138 +++++++++++ collector/pkg/detect/devices_darwin.go | 4 +- collector/pkg/detect/devices_freebsd.go | 4 +- collector/pkg/detect/devices_linux.go | 4 +- collector/pkg/detect/devices_windows.go | 4 +- collector/pkg/models/device.go | 3 +- collector/pkg/models/scan.go | 13 +- collector/pkg/models/scan_override.go | 7 + example.collector.yaml | 65 ++++++ example.scrutiny.yaml | 2 +- go.mod | 6 +- go.sum | 11 + webapp/backend/cmd/scrutiny/scrutiny.go | 4 +- webapp/backend/pkg/models/db/device.go | 5 +- webapp/backend/pkg/models/db/smart.go | 2 +- .../pkg/models/db/smart_ata_attribute.go | 2 +- .../pkg/models/db/smart_scsci_attribute.go | 2 +- .../pkg/web/handler/get_device_details.go | 2 +- .../pkg/web/handler/get_devices_summary.go | 2 +- .../pkg/web/handler/register_devices.go | 17 +- webapp/backend/pkg/web/middleware/sqlite3.go | 14 +- 33 files changed, 851 insertions(+), 54 deletions(-) create mode 100644 collector/pkg/config/config.go create mode 100644 collector/pkg/config/config_test.go create mode 100644 collector/pkg/config/factory.go create mode 100644 collector/pkg/config/interface.go create mode 100644 collector/pkg/config/mock/mock_config.go create mode 100644 collector/pkg/config/testdata/ignore_device.yaml create mode 100644 collector/pkg/config/testdata/raid_device.yaml create mode 100644 collector/pkg/config/testdata/simple_device.yaml create mode 100644 collector/pkg/detect/detect_test.go create mode 100644 collector/pkg/models/scan_override.go create mode 100644 example.collector.yaml diff --git a/README.md b/README.md index 57220fa..21c7062 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ docker exec scrutiny /scrutiny/bin/scrutiny-collector-metrics run ``` # Configuration -We support a global YAML configuration file that must be located at /scrutiny/config/scrutiny.yaml +We support a global YAML configuration file that must be located at `/scrutiny/config/scrutiny.yaml` Check the [example.scrutiny.yml](example.scrutiny.yaml) file for a fully commented version. diff --git a/collector/cmd/collector-metrics/collector-metrics.go b/collector/cmd/collector-metrics/collector-metrics.go index bfb0521..12275e4 100644 --- a/collector/cmd/collector-metrics/collector-metrics.go +++ b/collector/cmd/collector-metrics/collector-metrics.go @@ -3,6 +3,8 @@ package main import ( "fmt" "github.com/analogj/scrutiny/collector/pkg/collector" + "github.com/analogj/scrutiny/collector/pkg/config" + "github.com/analogj/scrutiny/collector/pkg/errors" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/sirupsen/logrus" "io" @@ -20,6 +22,20 @@ var goarch string func main() { + config, err := config.Create() + if err != nil { + fmt.Printf("FATAL: %+v\n", err) + os.Exit(1) + } + + //we're going to load the config file manually, since we need to validate it. + err = config.ReadConfig("/scrutiny/config/collector.yaml") // Find and read the config file + if _, ok := err.(errors.ConfigFileMissingError); ok { // Handle errors reading the config file + //ignore "could not find config file" + } else if err != nil { + os.Exit(1) + } + cli.CommandHelpTemplate = `NAME: {{.HelpName}} - {{.Usage}} USAGE: @@ -75,6 +91,17 @@ OPTIONS: Name: "run", Usage: "Run the scrutiny smartctl metrics collector", Action: func(c *cli.Context) error { + if c.IsSet("config") { + err = config.ReadConfig(c.String("config")) // Find and read the config file + if err != nil { // Handle errors reading the config file + //ignore "could not find config file" + fmt.Printf("Could not find config file at specified path: %s", c.String("config")) + return err + } + } + if c.IsSet("host-id") { + config.Set("host.id", c.String("host-id")) // set/override the host-id using CLI. + } collectorLogger := logrus.WithFields(logrus.Fields{ "type": "metrics", @@ -97,6 +124,7 @@ OPTIONS: } metricCollector, err := collector.CreateMetricsCollector( + config, collectorLogger, c.String("api-endpoint"), ) @@ -109,6 +137,10 @@ OPTIONS: }, Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Usage: "Specify the path to the devices file", + }, &cli.StringFlag{ Name: "api-endpoint", Usage: "The api server endpoint", @@ -128,12 +160,18 @@ OPTIONS: Usage: "Enable debug logging", EnvVars: []string{"COLLECTOR_DEBUG", "DEBUG"}, }, + + &cli.BoolFlag{ + Name: "host-id", + Usage: "Host identifier/label, used for grouping devices", + EnvVars: []string{"COLLECTOR_HOST_ID"}, + }, }, }, }, } - err := app.Run(os.Args) + err = app.Run(os.Args) if err != nil { log.Fatal(color.HiRedString("ERROR: %v", err)) } diff --git a/collector/pkg/collector/metrics.go b/collector/pkg/collector/metrics.go index 394e0e3..a83dd35 100644 --- a/collector/pkg/collector/metrics.go +++ b/collector/pkg/collector/metrics.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/analogj/scrutiny/collector/pkg/common" + "github.com/analogj/scrutiny/collector/pkg/config" "github.com/analogj/scrutiny/collector/pkg/detect" "github.com/analogj/scrutiny/collector/pkg/errors" "github.com/analogj/scrutiny/collector/pkg/models" @@ -16,17 +17,19 @@ import ( ) type MetricsCollector struct { + config config.Interface BaseCollector apiEndpoint *url.URL } -func CreateMetricsCollector(logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) { +func CreateMetricsCollector(appConfig config.Interface, logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) { apiEndpointUrl, err := url.Parse(apiEndpoint) if err != nil { return MetricsCollector{}, err } sc := MetricsCollector{ + config: appConfig, apiEndpoint: apiEndpointUrl, BaseCollector: BaseCollector{ logger: logger, @@ -49,6 +52,7 @@ func (mc *MetricsCollector) Run() error { deviceDetector := detect.Detect{ Logger: mc.logger, + Config: mc.config, } detectedStorageDevices, err := deviceDetector.Start() if err != nil { diff --git a/collector/pkg/config/config.go b/collector/pkg/config/config.go new file mode 100644 index 0000000..592f549 --- /dev/null +++ b/collector/pkg/config/config.go @@ -0,0 +1,100 @@ +package config + +import ( + "github.com/analogj/go-util/utils" + "github.com/analogj/scrutiny/collector/pkg/errors" + "github.com/analogj/scrutiny/collector/pkg/models" + "github.com/mitchellh/mapstructure" + "github.com/spf13/viper" + "log" + "os" +) + +// When initializing this class the following methods must be called: +// Config.New +// Config.Init +// This is done automatically when created via the Factory. +type configuration struct { + *viper.Viper +} + +//Viper uses the following precedence order. Each item takes precedence over the item below it: +// explicit call to Set +// flag +// env +// config +// key/value store +// default + +func (c *configuration) Init() error { + c.Viper = viper.New() + //set defaults + c.SetDefault("host.id", "") + + c.SetDefault("devices", []string{}) + + //c.SetDefault("collect.short.command", "-a -o on -S on") + + //if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig + c.SetConfigType("yaml") + //c.SetConfigName("drawbridge") + //c.AddConfigPath("$HOME/") + + //CLI options will be added via the `Set()` function + return nil +} + +func (c *configuration) ReadConfig(configFilePath string) error { + configFilePath, err := utils.ExpandPath(configFilePath) + if err != nil { + return err + } + + if !utils.FileExists(configFilePath) { + log.Printf("No configuration file found at %v. Using Defaults.", configFilePath) + return errors.ConfigFileMissingError("The configuration file could not be found.") + } + + //validate config file contents + //err = c.ValidateConfigFile(configFilePath) + //if err != nil { + // log.Printf("Config file at `%v` is invalid: %s", configFilePath, err) + // return err + //} + + log.Printf("Loading configuration file: %s", configFilePath) + + config_data, err := os.Open(configFilePath) + if err != nil { + log.Printf("Error reading configuration file: %s", err) + return err + } + + err = c.MergeConfig(config_data) + if err != nil { + return err + } + + return c.ValidateConfig() +} + +// This function ensures that the merged config works correctly. +func (c *configuration) ValidateConfig() error { + + //TODO: + // check that device prefix matches OS + // check that schema of config file is valid + + return nil +} + +func (c *configuration) GetScanOverrides() []models.ScanOverride { + // we have to support 2 types of device types. + // - simple device type (device_type: 'sat') + // and list of device types (type: \n- 3ware,0 \n- 3ware,1 \n- 3ware,2) + // GetString will return "" if this is a list of device types. + + overrides := []models.ScanOverride{} + c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true }) + return overrides +} diff --git a/collector/pkg/config/config_test.go b/collector/pkg/config/config_test.go new file mode 100644 index 0000000..1e8fd6d --- /dev/null +++ b/collector/pkg/config/config_test.go @@ -0,0 +1,64 @@ +package config_test + +import ( + "github.com/analogj/scrutiny/collector/pkg/config" + "github.com/analogj/scrutiny/collector/pkg/models" + "github.com/stretchr/testify/require" + "path" + "testing" +) + +func TestConfiguration_GetScanOverrides_Simple(t *testing.T) { + t.Parallel() + + //setup + testConfig, _ := config.Create() + + //test + err := testConfig.ReadConfig(path.Join("testdata", "simple_device.yaml")) + require.NoError(t, err, "should correctly load simple device config") + scanOverrides := testConfig.GetScanOverrides() + + //assert + require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat"}, Ignore: false}}, scanOverrides) +} + +func TestConfiguration_GetScanOverrides_Ignore(t *testing.T) { + t.Parallel() + + //setup + testConfig, _ := config.Create() + + //test + err := testConfig.ReadConfig(path.Join("testdata", "ignore_device.yaml")) + require.NoError(t, err, "should correctly load ignore device config") + scanOverrides := testConfig.GetScanOverrides() + + //assert + require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}, scanOverrides) +} + +func TestConfiguration_GetScanOverrides_Raid(t *testing.T) { + t.Parallel() + + //setup + testConfig, _ := config.Create() + + //test + err := testConfig.ReadConfig(path.Join("testdata", "raid_device.yaml")) + require.NoError(t, err, "should correctly load ignore device config") + scanOverrides := testConfig.GetScanOverrides() + + //assert + require.Equal(t, []models.ScanOverride{ + { + Device: "/dev/bus/0", + DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"}, + Ignore: false, + }, + { + Device: "/dev/twa0", + DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"}, + Ignore: false, + }}, scanOverrides) +} diff --git a/collector/pkg/config/factory.go b/collector/pkg/config/factory.go new file mode 100644 index 0000000..965ea2f --- /dev/null +++ b/collector/pkg/config/factory.go @@ -0,0 +1,9 @@ +package config + +func Create() (Interface, error) { + config := new(configuration) + if err := config.Init(); err != nil { + return nil, err + } + return config, nil +} diff --git a/collector/pkg/config/interface.go b/collector/pkg/config/interface.go new file mode 100644 index 0000000..53dd2b8 --- /dev/null +++ b/collector/pkg/config/interface.go @@ -0,0 +1,26 @@ +package config + +import ( + "github.com/analogj/scrutiny/collector/pkg/models" + "github.com/spf13/viper" +) + +// Create mock using: +// mockgen -source=collector/pkg/config/interface.go -destination=collector/pkg/config/mock/mock_config.go +type Interface interface { + Init() error + ReadConfig(configFilePath string) error + Set(key string, value interface{}) + SetDefault(key string, value interface{}) + + AllSettings() map[string]interface{} + IsSet(key string) bool + Get(key string) interface{} + GetBool(key string) bool + GetInt(key string) int + GetString(key string) string + GetStringSlice(key string) []string + UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error + + GetScanOverrides() []models.ScanOverride +} diff --git a/collector/pkg/config/mock/mock_config.go b/collector/pkg/config/mock/mock_config.go new file mode 100644 index 0000000..4c1d1d1 --- /dev/null +++ b/collector/pkg/config/mock/mock_config.go @@ -0,0 +1,218 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: collector/pkg/config/interface.go + +// Package mock_config is a generated GoMock package. +package mock_config + +import ( + models "github.com/analogj/scrutiny/collector/pkg/models" + gomock "github.com/golang/mock/gomock" + viper "github.com/spf13/viper" + reflect "reflect" +) + +// MockInterface is a mock of Interface interface +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// Init mocks base method +func (m *MockInterface) Init() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init") + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init +func (mr *MockInterfaceMockRecorder) Init() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init)) +} + +// ReadConfig mocks base method +func (m *MockInterface) ReadConfig(configFilePath string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadConfig", configFilePath) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReadConfig indicates an expected call of ReadConfig +func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath) +} + +// Set mocks base method +func (m *MockInterface) Set(key string, value interface{}) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Set", key, value) +} + +// Set indicates an expected call of Set +func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value) +} + +// SetDefault mocks base method +func (m *MockInterface) SetDefault(key string, value interface{}) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetDefault", key, value) +} + +// SetDefault indicates an expected call of SetDefault +func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value) +} + +// AllSettings mocks base method +func (m *MockInterface) AllSettings() map[string]interface{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllSettings") + ret0, _ := ret[0].(map[string]interface{}) + return ret0 +} + +// AllSettings indicates an expected call of AllSettings +func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings)) +} + +// IsSet mocks base method +func (m *MockInterface) IsSet(key string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsSet", key) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsSet indicates an expected call of IsSet +func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key) +} + +// Get mocks base method +func (m *MockInterface) Get(key string) interface{} { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", key) + ret0, _ := ret[0].(interface{}) + return ret0 +} + +// Get indicates an expected call of Get +func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key) +} + +// GetBool mocks base method +func (m *MockInterface) GetBool(key string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBool", key) + ret0, _ := ret[0].(bool) + return ret0 +} + +// GetBool indicates an expected call of GetBool +func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key) +} + +// GetInt mocks base method +func (m *MockInterface) GetInt(key string) int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInt", key) + ret0, _ := ret[0].(int) + return ret0 +} + +// GetInt indicates an expected call of GetInt +func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key) +} + +// GetString mocks base method +func (m *MockInterface) GetString(key string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetString", key) + ret0, _ := ret[0].(string) + return ret0 +} + +// GetString indicates an expected call of GetString +func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key) +} + +// GetStringSlice mocks base method +func (m *MockInterface) GetStringSlice(key string) []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStringSlice", key) + ret0, _ := ret[0].([]string) + return ret0 +} + +// GetStringSlice indicates an expected call of GetStringSlice +func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key) +} + +// UnmarshalKey mocks base method +func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{key, rawVal} + for _, a := range decoderOpts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UnmarshalKey", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UnmarshalKey indicates an expected call of UnmarshalKey +func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{key, rawVal}, decoderOpts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...) +} + +// GetScanOverrides mocks base method +func (m *MockInterface) GetScanOverrides() []models.ScanOverride { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetScanOverrides") + ret0, _ := ret[0].([]models.ScanOverride) + return ret0 +} + +// GetScanOverrides indicates an expected call of GetScanOverrides +func (mr *MockInterfaceMockRecorder) GetScanOverrides() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScanOverrides", reflect.TypeOf((*MockInterface)(nil).GetScanOverrides)) +} diff --git a/collector/pkg/config/testdata/ignore_device.yaml b/collector/pkg/config/testdata/ignore_device.yaml new file mode 100644 index 0000000..8520e91 --- /dev/null +++ b/collector/pkg/config/testdata/ignore_device.yaml @@ -0,0 +1,4 @@ +version: 1 +devices: + - device: /dev/sda + ignore: true diff --git a/collector/pkg/config/testdata/raid_device.yaml b/collector/pkg/config/testdata/raid_device.yaml new file mode 100644 index 0000000..a53861f --- /dev/null +++ b/collector/pkg/config/testdata/raid_device.yaml @@ -0,0 +1,19 @@ +version: 1 +devices: + - device: /dev/bus/0 + type: + - megaraid,14 + - megaraid,15 + - megaraid,18 + - megaraid,19 + - megaraid,20 + - megaraid,21 + + - device: /dev/twa0 + type: + - 3ware,0 + - 3ware,1 + - 3ware,2 + - 3ware,3 + - 3ware,4 + - 3ware,5 diff --git a/collector/pkg/config/testdata/simple_device.yaml b/collector/pkg/config/testdata/simple_device.yaml new file mode 100644 index 0000000..ed6e000 --- /dev/null +++ b/collector/pkg/config/testdata/simple_device.yaml @@ -0,0 +1,27 @@ +version: 1 +devices: + - device: /dev/sda + type: 'sat' +# +# # example to show how to ignore a specific disk/device. +# - device: /dev/sda +# ignore: true +# +# # examples showing how to force smartctl to detect disks inside a raid array/virtual disk +# - device: /dev/bus/0 +# type: +# - megaraid,14 +# - megaraid,15 +# - megaraid,18 +# - megaraid,19 +# - megaraid,20 +# - megaraid,21 +# +# - device: /dev/twa0 +# type: +# - 3ware,0 +# - 3ware,1 +# - 3ware,2 +# - 3ware,3 +# - 3ware,4 +# - 3ware,5 diff --git a/collector/pkg/detect/detect.go b/collector/pkg/detect/detect.go index 345eb8f..984bee3 100644 --- a/collector/pkg/detect/detect.go +++ b/collector/pkg/detect/detect.go @@ -4,9 +4,9 @@ import ( "encoding/json" "fmt" "github.com/analogj/scrutiny/collector/pkg/common" + "github.com/analogj/scrutiny/collector/pkg/config" "github.com/analogj/scrutiny/collector/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" - "github.com/denisbrodbeck/machineid" "github.com/sirupsen/logrus" "os" "strings" @@ -14,6 +14,7 @@ import ( type Detect struct { Logger *logrus.Entry + Config config.Interface } //private/common functions @@ -24,7 +25,7 @@ type Detect struct { // // To handle these issues, we have OS specific wrapper functions that update/modify these detected devices. // models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file). -func (d *Detect) smartctlScan() ([]models.Device, error) { +func (d *Detect) SmartctlScan() ([]models.Device, error) { //we use smartctl to detect all the drives available. detectedDeviceConnJson, err := common.ExecCmd(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ()) if err != nil { @@ -39,14 +40,7 @@ func (d *Detect) smartctlScan() ([]models.Device, error) { return nil, err } - detectedDevices := []models.Device{} - - for _, detectedDevice := range detectedDeviceConns.Devices { - detectedDevices = append(detectedDevices, models.Device{ - DeviceType: detectedDevice.Type, - DeviceName: strings.TrimPrefix(detectedDevice.Name, DevicePrefix()), - }) - } + detectedDevices := d.TransformDetectedDevices(detectedDeviceConns) return detectedDevices, nil } @@ -55,7 +49,7 @@ func (d *Detect) smartctlScan() ([]models.Device, error) { // It has a couple of issues however: // - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves // - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN. -func (d *Detect) smartCtlInfo(device *models.Device) error { +func (d *Detect) SmartCtlInfo(device *models.Device) error { args := []string{"--info", "-j"} //only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost. @@ -77,8 +71,8 @@ func (d *Detect) smartCtlInfo(device *models.Device) error { return err } - //DeviceType and DeviceName are already populated. - //WWN + //WWN: this is a serial number/world-wide number that will not change. + //DeviceType and DeviceName are already populated, however may change between collector runs (eg. config/host restart) //InterfaceType: device.ModelName = availableDeviceInfo.ModelName device.InterfaceSpeed = availableDeviceInfo.InterfaceSpeed.Current.String @@ -110,7 +104,60 @@ func (d *Detect) smartCtlInfo(device *models.Device) error { return nil } -//uses https://github.com/denisbrodbeck/machineid to get a OS specific unique machine ID. -func (d *Detect) getMachineId() (string, error) { - return machineid.ProtectedID("scrutiny") +// function will remove devices that are marked for "ignore" in config file +// will also add devices that are specified in config file, but "missing" from smartctl --scan +// this function will also update the deviceType to the option specified in config. +func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []models.Device { + groupedDevices := map[string][]models.Device{} + + for _, scannedDevice := range detectedDeviceConns.Devices { + + deviceFile := strings.ToLower(scannedDevice.Name) + + detectedDevice := models.Device{ + HostId: d.Config.GetString("host.id"), + DeviceType: scannedDevice.Type, + DeviceName: strings.TrimPrefix(deviceFile, DevicePrefix()), + } + + //find (or create) a slice to contain the devices in this group + if groupedDevices[deviceFile] == nil { + groupedDevices[deviceFile] = []models.Device{} + } + + // add this scanned device to the group + groupedDevices[deviceFile] = append(groupedDevices[deviceFile], detectedDevice) + } + + //now tha we've "grouped" all the devices, lets override any groups specified in the config file. + + for _, overrideDevice := range d.Config.GetScanOverrides() { + overrideDeviceFile := strings.ToLower(overrideDevice.Device) + + if overrideDevice.Ignore { + // this device file should be deleted if it exists + delete(groupedDevices, overrideDeviceFile) + } else { + //create a new device group, and replace the one generated by smartctl --scan + overrideDeviceGroup := []models.Device{} + + for _, overrideDeviceType := range overrideDevice.DeviceType { + overrideDeviceGroup = append(overrideDeviceGroup, models.Device{ + HostId: d.Config.GetString("host.id"), + DeviceType: overrideDeviceType, + DeviceName: strings.TrimPrefix(overrideDeviceFile, DevicePrefix()), + }) + } + + groupedDevices[overrideDeviceFile] = overrideDeviceGroup + } + } + + //flatten map + detectedDevices := []models.Device{} + for _, group := range groupedDevices { + detectedDevices = append(detectedDevices, group...) + } + + return detectedDevices } diff --git a/collector/pkg/detect/detect_test.go b/collector/pkg/detect/detect_test.go new file mode 100644 index 0000000..6c034ec --- /dev/null +++ b/collector/pkg/detect/detect_test.go @@ -0,0 +1,138 @@ +package detect_test + +import ( + mock_config "github.com/analogj/scrutiny/collector/pkg/config/mock" + "github.com/analogj/scrutiny/collector/pkg/detect" + "github.com/analogj/scrutiny/collector/pkg/models" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "testing" +) + +func TestDetect_TransformDetectedDevices_Empty(t *testing.T) { + //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") + fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{}) + detectedDevices := models.Scan{ + Devices: []models.ScanDevice{ + { + Name: "/dev/sda", + InfoName: "/dev/sda", + Protocol: "scsi", + Type: "scsi", + }, + }, + } + + d := detect.Detect{ + Config: fakeConfig, + } + + //test + transformedDevices := d.TransformDetectedDevices(detectedDevices) + + //assert + require.Equal(t, "sda", transformedDevices[0].DeviceName) + require.Equal(t, "scsi", transformedDevices[0].DeviceType) +} + +func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) { + //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") + fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}) + detectedDevices := models.Scan{ + Devices: []models.ScanDevice{ + { + Name: "/dev/sda", + InfoName: "/dev/sda", + Protocol: "scsi", + Type: "scsi", + }, + }, + } + + d := detect.Detect{ + Config: fakeConfig, + } + + //test + transformedDevices := d.TransformDetectedDevices(detectedDevices) + + //assert + require.Equal(t, []models.Device{}, transformedDevices) +} + +func TestDetect_TransformDetectedDevices_Raid(t *testing.T) { + //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") + fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{ + { + Device: "/dev/bus/0", + DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"}, + Ignore: false, + }, + { + Device: "/dev/twa0", + DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"}, + Ignore: false, + }}) + detectedDevices := models.Scan{ + Devices: []models.ScanDevice{ + { + Name: "/dev/bus/0", + InfoName: "/dev/bus/0", + Protocol: "scsi", + Type: "scsi", + }, + }, + } + + d := detect.Detect{ + Config: fakeConfig, + } + + //test + transformedDevices := d.TransformDetectedDevices(detectedDevices) + + //assert + require.Equal(t, 12, len(transformedDevices)) +} + +func TestDetect_TransformDetectedDevices_Simple(t *testing.T) { + //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") + fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}}) + detectedDevices := models.Scan{ + Devices: []models.ScanDevice{ + { + Name: "/dev/sda", + InfoName: "/dev/sda", + Protocol: "ata", + Type: "ata", + }, + }, + } + + d := detect.Detect{ + Config: fakeConfig, + } + + //test + transformedDevices := d.TransformDetectedDevices(detectedDevices) + + //assert + require.Equal(t, 1, len(transformedDevices)) + require.Equal(t, "sat+megaraid", transformedDevices[0].DeviceType) +} diff --git a/collector/pkg/detect/devices_darwin.go b/collector/pkg/detect/devices_darwin.go index d6b4f49..73c0657 100644 --- a/collector/pkg/detect/devices_darwin.go +++ b/collector/pkg/detect/devices_darwin.go @@ -12,7 +12,7 @@ func DevicePrefix() string { func (d *Detect) Start() ([]models.Device, error) { // call the base/common functionality to get a list of devicess - detectedDevices, err := d.smartctlScan() + detectedDevices, err := d.SmartctlScan() if err != nil { return nil, err } @@ -25,7 +25,7 @@ func (d *Detect) Start() ([]models.Device, error) { //inflate device info for detected devices. for ndx, _ := range detectedDevices { - d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors. + d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors. } return detectedDevices, nil diff --git a/collector/pkg/detect/devices_freebsd.go b/collector/pkg/detect/devices_freebsd.go index 912070a..619e731 100644 --- a/collector/pkg/detect/devices_freebsd.go +++ b/collector/pkg/detect/devices_freebsd.go @@ -12,14 +12,14 @@ func DevicePrefix() string { func (d *Detect) Start() ([]models.Device, error) { // call the base/common functionality to get a list of devices - detectedDevices, err := d.smartctlScan() + detectedDevices, err := d.SmartctlScan() if err != nil { return nil, err } //inflate device info for detected devices. for ndx, _ := range detectedDevices { - d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors. + d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors. } return detectedDevices, nil diff --git a/collector/pkg/detect/devices_linux.go b/collector/pkg/detect/devices_linux.go index 912070a..619e731 100644 --- a/collector/pkg/detect/devices_linux.go +++ b/collector/pkg/detect/devices_linux.go @@ -12,14 +12,14 @@ func DevicePrefix() string { func (d *Detect) Start() ([]models.Device, error) { // call the base/common functionality to get a list of devices - detectedDevices, err := d.smartctlScan() + detectedDevices, err := d.SmartctlScan() if err != nil { return nil, err } //inflate device info for detected devices. for ndx, _ := range detectedDevices { - d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors. + d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors. } return detectedDevices, nil diff --git a/collector/pkg/detect/devices_windows.go b/collector/pkg/detect/devices_windows.go index 8ecb781..d9ddbaf 100644 --- a/collector/pkg/detect/devices_windows.go +++ b/collector/pkg/detect/devices_windows.go @@ -11,14 +11,14 @@ func DevicePrefix() string { func (d *Detect) Start() ([]models.Device, error) { // call the base/common functionality to get a list of devices - detectedDevices, err := d.smartctlScan() + detectedDevices, err := d.SmartctlScan() if err != nil { return nil, err } //inflate device info for detected devices. for ndx, _ := range detectedDevices { - d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors. + d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors. } return detectedDevices, nil diff --git a/collector/pkg/models/device.go b/collector/pkg/models/device.go index ecfcb06..dd77794 100644 --- a/collector/pkg/models/device.go +++ b/collector/pkg/models/device.go @@ -1,7 +1,8 @@ package models type Device struct { - WWN string `json:"wwn"` + WWN string `json:"wwn"` + HostId string `json:"host_id"` DeviceName string `json:"device_name"` Manufacturer string `json:"manufacturer"` diff --git a/collector/pkg/models/scan.go b/collector/pkg/models/scan.go index d85ff57..92752de 100644 --- a/collector/pkg/models/scan.go +++ b/collector/pkg/models/scan.go @@ -10,10 +10,11 @@ type Scan struct { Argv []string `json:"argv"` ExitStatus int `json:"exit_status"` } `json:"smartctl"` - Devices []struct { - Name string `json:"name"` - InfoName string `json:"info_name"` - Type string `json:"type"` - Protocol string `json:"protocol"` - } `json:"devices"` + Devices []ScanDevice `json:"devices"` +} +type ScanDevice struct { + Name string `json:"name"` + InfoName string `json:"info_name"` + Type string `json:"type"` + Protocol string `json:"protocol"` } diff --git a/collector/pkg/models/scan_override.go b/collector/pkg/models/scan_override.go new file mode 100644 index 0000000..682687d --- /dev/null +++ b/collector/pkg/models/scan_override.go @@ -0,0 +1,7 @@ +package models + +type ScanOverride struct { + Device string `mapstructure:"device"` + DeviceType []string `mapstructure:"type"` + Ignore bool `mapstructure:"ignore"` +} diff --git a/example.collector.yaml b/example.collector.yaml new file mode 100644 index 0000000..fa72814 --- /dev/null +++ b/example.collector.yaml @@ -0,0 +1,65 @@ +# Commented Scrutiny Configuration File +# +# The default location for this file is /scrutiny/config/collector.yaml. +# In some cases to improve clarity default values are specified, +# uncommented. Other example values are commented out. +# +# When this file is parsed by Scrutiny, all configuration file keys are +# lowercased automatically. As such, Configuration keys are case-insensitive, +# and should be lowercase in this file to be consistent with usage. + + +###################################################################### +# Version +# +# version specifies the version of this configuration file schema, not +# the scrutiny binary. There is only 1 version available at the moment +version: 1 + +# The host id is a label used for identifying groups of disks running on the same host +# Primiarly used for hub/spoke deployments (can be left empty if using all-in-one image). +host: + id: "" + + +# This block allows you to override/customize the settings for devices detected by +# Scrutiny via `smartctl --scan` +# See the "--device=TYPE" section of https://linux.die.net/man/8/smartctl +# type can be a 'string' or a 'list' +devices: +# # example for forcing device type detection for a single disk +# - device: /dev/sda +# type: 'sat' +# +# # example to show how to ignore a specific disk/device. +# - device: /dev/sda +# ignore: true +# +# # examples showing how to force smartctl to detect disks inside a raid array/virtual disk +# - device: /dev/bus/0 +# type: +# - megaraid,14 +# - megaraid,15 +# - megaraid,18 +# - megaraid,19 +# - megaraid,20 +# - megaraid,21 +# +# - device: /dev/twa0 +# type: +# - 3ware,0 +# - 3ware,1 +# - 3ware,2 +# - 3ware,3 +# - 3ware,4 +# - 3ware,5 + + + +######################################################################################################################## +# FEATURES COMING SOON +# +# The following commented out sections are a preview of additional configuration options that will be available soon. +# +######################################################################################################################## + diff --git a/example.scrutiny.yaml b/example.scrutiny.yaml index 2846f46..9744f19 100644 --- a/example.scrutiny.yaml +++ b/example.scrutiny.yaml @@ -1,6 +1,6 @@ # Commented Scrutiny Configuration File # -# The default location for this file is ~/scrutiny.yaml. +# The default location for this file is /scrutiny/config/scrutiny.yaml. # In some cases to improve clarity default values are specified, # uncommented. Other example values are commented out. # diff --git a/go.mod b/go.mod index c1027e5..c4127eb 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,15 @@ require ( github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae // indirect github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14 github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a - github.com/denisbrodbeck/machineid v1.0.1 + github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/fatih/color v1.9.0 github.com/gin-gonic/gin v1.6.3 github.com/golang/mock v1.4.3 github.com/jaypipes/ghw v0.6.1 github.com/jinzhu/gorm v1.9.14 github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect + github.com/mattn/go-sqlite3 v1.14.4 + github.com/mitchellh/mapstructure v1.2.2 github.com/sirupsen/logrus v1.2.0 github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.5.1 @@ -20,4 +22,6 @@ require ( golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 gopkg.in/yaml.v2 v2.3.0 // indirect + gorm.io/driver/sqlite v1.1.3 + gorm.io/gorm v1.20.2 ) diff --git a/go.sum b/go.sum index df4c220..5a73a29 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBef github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= @@ -186,6 +188,10 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA= +github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= +github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI= +github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -441,6 +447,11 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc= +gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c= +gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.2 h1:bZzSEnq7NDGsrd+n3evOOedDrY5oLM5QPlCjZJUK2ro= +gorm.io/gorm v1.20.2/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gosrc.io/xmpp v0.1.1 h1:iMtE9W3fx254+4E6rI34AOPJDqWvpfQR6EYaVMzhJ4s= gosrc.io/xmpp v0.1.1/go.mod h1:4JgaXzw4MnEv2sGltONtK3GMhj+h9gpQ7cO8nwbFJLU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/webapp/backend/cmd/scrutiny/scrutiny.go b/webapp/backend/cmd/scrutiny/scrutiny.go index 4bcf290..b02475e 100644 --- a/webapp/backend/cmd/scrutiny/scrutiny.go +++ b/webapp/backend/cmd/scrutiny/scrutiny.go @@ -60,7 +60,7 @@ OPTIONS: }, Before: func(c *cli.Context) error { - drawbridge := "github.com/AnalogJ/scrutiny" + scrutiny := "github.com/AnalogJ/scrutiny" var versionInfo string if len(goos) > 0 && len(goarch) > 0 { @@ -69,7 +69,7 @@ OPTIONS: versionInfo = fmt.Sprintf("dev-%s", version.VERSION) } - subtitle := drawbridge + utils.LeftPad2Len(versionInfo, " ", 65-len(drawbridge)) + subtitle := scrutiny + utils.LeftPad2Len(versionInfo, " ", 65-len(scrutiny)) color.New(color.FgGreen).Fprintf(c.App.Writer, fmt.Sprintf(utils.StripIndent( ` diff --git a/webapp/backend/pkg/models/db/device.go b/webapp/backend/pkg/models/db/device.go index 96e81d4..2786103 100644 --- a/webapp/backend/pkg/models/db/device.go +++ b/webapp/backend/pkg/models/db/device.go @@ -21,7 +21,8 @@ type Device struct { UpdatedAt time.Time DeletedAt *time.Time - WWN string `json:"wwn" gorm:"primary_key"` + WWN string `json:"wwn" gorm:"primary_key"` + HostId string `json:"host_id"` DeviceName string `json:"device_name"` Manufacturer string `json:"manufacturer"` @@ -151,6 +152,8 @@ func (dv *Device) ApplyMetadataRules() error { return nil } +// This function is called every time the collector sends SMART data to the API. +// It can be used to update device data that can change over time. func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error { dv.Firmware = info.FirmwareVersion return nil diff --git a/webapp/backend/pkg/models/db/smart.go b/webapp/backend/pkg/models/db/smart.go index 7e11a45..510dcb2 100644 --- a/webapp/backend/pkg/models/db/smart.go +++ b/webapp/backend/pkg/models/db/smart.go @@ -3,7 +3,7 @@ package db import ( "github.com/analogj/scrutiny/webapp/backend/pkg/metadata" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" - "github.com/jinzhu/gorm" + "gorm.io/gorm" "time" ) diff --git a/webapp/backend/pkg/models/db/smart_ata_attribute.go b/webapp/backend/pkg/models/db/smart_ata_attribute.go index 2f3a6ff..510672d 100644 --- a/webapp/backend/pkg/models/db/smart_ata_attribute.go +++ b/webapp/backend/pkg/models/db/smart_ata_attribute.go @@ -2,7 +2,7 @@ package db import ( "github.com/analogj/scrutiny/webapp/backend/pkg/metadata" - "github.com/jinzhu/gorm" + "gorm.io/gorm" "strings" ) diff --git a/webapp/backend/pkg/models/db/smart_scsci_attribute.go b/webapp/backend/pkg/models/db/smart_scsci_attribute.go index b988b0c..c6b1325 100644 --- a/webapp/backend/pkg/models/db/smart_scsci_attribute.go +++ b/webapp/backend/pkg/models/db/smart_scsci_attribute.go @@ -2,7 +2,7 @@ package db import ( "github.com/analogj/scrutiny/webapp/backend/pkg/metadata" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type SmartScsiAttribute struct { diff --git a/webapp/backend/pkg/web/handler/get_device_details.go b/webapp/backend/pkg/web/handler/get_device_details.go index fd72264..4bee8fc 100644 --- a/webapp/backend/pkg/web/handler/get_device_details.go +++ b/webapp/backend/pkg/web/handler/get_device_details.go @@ -4,8 +4,8 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/metadata" dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" "github.com/gin-gonic/gin" - "github.com/jinzhu/gorm" "github.com/sirupsen/logrus" + "gorm.io/gorm" "net/http" ) diff --git a/webapp/backend/pkg/web/handler/get_devices_summary.go b/webapp/backend/pkg/web/handler/get_devices_summary.go index 01e94ff..b0c9c1c 100644 --- a/webapp/backend/pkg/web/handler/get_devices_summary.go +++ b/webapp/backend/pkg/web/handler/get_devices_summary.go @@ -3,8 +3,8 @@ package handler import ( dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" "github.com/gin-gonic/gin" - "github.com/jinzhu/gorm" "github.com/sirupsen/logrus" + "gorm.io/gorm" "net/http" ) diff --git a/webapp/backend/pkg/web/handler/register_devices.go b/webapp/backend/pkg/web/handler/register_devices.go index 6bdfd5b..56d29f6 100644 --- a/webapp/backend/pkg/web/handler/register_devices.go +++ b/webapp/backend/pkg/web/handler/register_devices.go @@ -3,12 +3,15 @@ package handler import ( dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" "github.com/gin-gonic/gin" - "github.com/jinzhu/gorm" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "github.com/sirupsen/logrus" "net/http" ) -// filter devices that are detected by various collectors. +// register devices that are detected by various collectors. +// This function is run everytime a collector is about to start a run. It can be used to update device data. func RegisterDevices(c *gin.Context) { db := c.MustGet("DB").(*gorm.DB) logger := c.MustGet("LOGGER").(logrus.FieldLogger) @@ -21,11 +24,15 @@ func RegisterDevices(c *gin.Context) { return } - //TODO: filter devices here (remove excludes, force includes) errs := []error{} for _, dev := range collectorDeviceWrapper.Data { - //insert devices into DB if not already there. - if err := db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev).Error; err != nil { + //insert devices into DB (and update specified columns if device is already registered) + // update device fields that may change: (DeviceType, HostID) + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "wwn"}}, + DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name"}), + }).Create(&dev).Error; err != nil { + errs = append(errs, err) } } diff --git a/webapp/backend/pkg/web/middleware/sqlite3.go b/webapp/backend/pkg/web/middleware/sqlite3.go index 3146359..3eeee13 100644 --- a/webapp/backend/pkg/web/middleware/sqlite3.go +++ b/webapp/backend/pkg/web/middleware/sqlite3.go @@ -5,20 +5,24 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" "github.com/gin-gonic/gin" - "github.com/jinzhu/gorm" - _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/sirupsen/logrus" + "gorm.io/driver/sqlite" + "gorm.io/gorm" ) -func DatabaseMiddleware(appConfig config.Interface, logger logrus.FieldLogger) gin.HandlerFunc { +func DatabaseMiddleware(appConfig config.Interface, globalLogger logrus.FieldLogger) gin.HandlerFunc { + //var database *gorm.DB fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location")) - database, err := gorm.Open("sqlite3", appConfig.GetString("web.database.location")) + database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{ + //TODO: figure out how to log database queries again. + //Logger: logger + }) if err != nil { panic("Failed to connect to database!") } - database.SetLogger(&GormLogger{Logger: logger}) + //database.SetLogger() database.AutoMigrate(&db.Device{}) database.AutoMigrate(&db.SelfTest{}) database.AutoMigrate(&db.Smart{}) From 1cd5ebaa436967710005a5ee781067eb7fac82e8 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 7 Oct 2020 21:59:48 -0600 Subject: [PATCH 2/3] missing device type update. --- webapp/backend/pkg/web/handler/register_devices.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/backend/pkg/web/handler/register_devices.go b/webapp/backend/pkg/web/handler/register_devices.go index 56d29f6..746278a 100644 --- a/webapp/backend/pkg/web/handler/register_devices.go +++ b/webapp/backend/pkg/web/handler/register_devices.go @@ -30,7 +30,7 @@ func RegisterDevices(c *gin.Context) { // update device fields that may change: (DeviceType, HostID) if err := db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "wwn"}}, - DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name"}), + DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}), }).Create(&dev).Error; err != nil { errs = append(errs, err) From 9fac3c6308dc6cb6cd5bbc43a68cd93e8fb20b87 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 7 Oct 2020 22:07:51 -0600 Subject: [PATCH 3/3] fix gorm dependency. --- go.mod | 6 ++-- go.sum | 35 +++++++------------ .../pkg/models/db/smart_nvme_attribute.go | 2 +- .../pkg/web/handler/upload_device_metrics.go | 2 +- 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index c4127eb..44aca66 100644 --- a/go.mod +++ b/go.mod @@ -3,23 +3,21 @@ module github.com/analogj/scrutiny go 1.13 require ( - github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae // indirect github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14 github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a - github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/fatih/color v1.9.0 github.com/gin-gonic/gin v1.6.3 github.com/golang/mock v1.4.3 github.com/jaypipes/ghw v0.6.1 - github.com/jinzhu/gorm v1.9.14 github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect - github.com/mattn/go-sqlite3 v1.14.4 + github.com/mattn/go-sqlite3 v1.14.4 // indirect github.com/mitchellh/mapstructure v1.2.2 github.com/sirupsen/logrus v1.2.0 github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.5.1 github.com/urfave/cli/v2 v2.2.0 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect + golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 gopkg.in/yaml.v2 v2.3.0 // indirect gorm.io/driver/sqlite v1.1.3 diff --git a/go.sum b/go.sum index 5a73a29..1d0185d 100644 --- a/go.sum +++ b/go.sum @@ -11,19 +11,16 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae h1:iYSadgTTTmFTvZwdDImnytps8Hq9zlpWeNfYpe1RTPs= -github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae/go.mod h1:0jFBtvNk8rNzZjL8j1b852fcka5/VJfJvRiU2w6OIkI= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14 h1:wsrSjiqQtseStRIoLLxS4C5IEtXkazZVEPDHq8jW7r8= github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14/go.mod h1:lJQVqFKMV5/oDGYR2bra2OljcF3CvolAoyDRyOA4k4E= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -50,17 +47,11 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= -github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -77,6 +68,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -84,11 +76,9 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -104,6 +94,7 @@ github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -112,6 +103,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -142,17 +134,15 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jaypipes/ghw v0.6.1 h1:Ewt3mdpiyhWotGyzg1ursV/6SnToGcG4215X6rR2af8= github.com/jaypipes/ghw v0.6.1/go.mod h1:QOXppNRCLGYR1H+hu09FxZPqjNt09bqUZUnOL3Rcero= github.com/jaypipes/pcidb v0.5.0 h1:4W5gZ+G7QxydevI8/MmmKdnIPJpURqJ2JNXTzfLxF5c= github.com/jaypipes/pcidb v0.5.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jinzhu/gorm v1.9.14 h1:Kg3ShyTPcM6nzVo148fRrcMO6MNKuqtOUwnzqMgVniM= -github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -160,6 +150,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -167,14 +158,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 h1:3tLzEnUizyN9YLWFTT9loC30lSBvh2y70LTDcZOTs1s= github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0/go.mod h1:8/LTPeDLaklcUjgSQBHbhBF1ibKAFxzS5o+H7USfMSA= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -186,8 +178,6 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= -github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA= github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI= @@ -217,6 +207,7 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= @@ -248,7 +239,9 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -303,10 +296,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -327,7 +318,6 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -344,7 +334,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -429,6 +419,7 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= diff --git a/webapp/backend/pkg/models/db/smart_nvme_attribute.go b/webapp/backend/pkg/models/db/smart_nvme_attribute.go index 145a0a9..b681181 100644 --- a/webapp/backend/pkg/models/db/smart_nvme_attribute.go +++ b/webapp/backend/pkg/models/db/smart_nvme_attribute.go @@ -2,7 +2,7 @@ package db import ( "github.com/analogj/scrutiny/webapp/backend/pkg/metadata" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type SmartNvmeAttribute struct { diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index 2024b07..949116a 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -6,8 +6,8 @@ import ( dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" "github.com/analogj/scrutiny/webapp/backend/pkg/notify" "github.com/gin-gonic/gin" - "github.com/jinzhu/gorm" "github.com/sirupsen/logrus" + "gorm.io/gorm" "net/http" )