From a53397210ce47b24f637630fa2fd0cff6f27b0b0 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Sat, 28 May 2022 15:32:44 -0700 Subject: [PATCH] adding mechanism to override the smartctl commands used by scrutiny for device scanning, device identification and smart data retrieval. adding tests for command overrides. rename GetScanOverrides() to GetDeviceOverrides() fixes #255 --- collector/pkg/collector/metrics.go | 7 +- collector/pkg/config/config.go | 92 +++++++- collector/pkg/config/config_test.go | 56 ++++- collector/pkg/config/interface.go | 4 +- collector/pkg/config/mock/mock_config.go | 215 ++++++++++-------- .../invalid_commands_includes_device.yaml | 4 + .../invalid_commands_missing_json.yaml | 4 + .../config/testdata/override_commands.yaml | 4 + .../testdata/override_device_commands.yaml | 5 + collector/pkg/detect/detect.go | 13 +- collector/pkg/detect/detect_test.go | 23 +- collector/pkg/models/scan_override.go | 4 + example.collector.yaml | 13 ++ 13 files changed, 326 insertions(+), 118 deletions(-) create mode 100644 collector/pkg/config/testdata/invalid_commands_includes_device.yaml create mode 100644 collector/pkg/config/testdata/invalid_commands_missing_json.yaml create mode 100644 collector/pkg/config/testdata/override_commands.yaml create mode 100644 collector/pkg/config/testdata/override_device_commands.yaml diff --git a/collector/pkg/collector/metrics.go b/collector/pkg/collector/metrics.go index 31b6af7..ed1b777 100644 --- a/collector/pkg/collector/metrics.go +++ b/collector/pkg/collector/metrics.go @@ -116,12 +116,13 @@ func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceT } mc.logger.Infof("Collecting smartctl results for %s\n", deviceName) - args := []string{"-x", "-j"} + fullDeviceName := fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName) + args := strings.Split(mc.config.GetCommandMetricsSmartArgs(fullDeviceName), " ") //only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost. if len(deviceType) > 0 && deviceType != "scsi" && deviceType != "ata" { - args = append(args, "-d", deviceType) + args = append(args, "--device", deviceType) } - args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName)) + args = append(args, fullDeviceName) result, err := mc.shell.Command(mc.logger, "smartctl", args, "", os.Environ()) resultBytes := []byte(result) diff --git a/collector/pkg/config/config.go b/collector/pkg/config/config.go index 5995659..025082e 100644 --- a/collector/pkg/config/config.go +++ b/collector/pkg/config/config.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/collector/pkg/errors" "github.com/analogj/scrutiny/collector/pkg/models" @@ -8,6 +9,8 @@ import ( "github.com/spf13/viper" "log" "os" + "sort" + "strings" ) // When initializing this class the following methods must be called: @@ -16,6 +19,8 @@ import ( // This is done automatically when created via the Factory. type configuration struct { *viper.Viper + + deviceOverrides []models.ScanOverride } //Viper uses the following precedence order. Each item takes precedence over the item below it: @@ -38,6 +43,10 @@ func (c *configuration) Init() error { c.SetDefault("api.endpoint", "http://localhost:8080") + c.SetDefault("commands.metrics_scan_args", "--scan --json") + c.SetDefault("commands.metrics_info_args", "--info --json") + c.SetDefault("commands.metrics_smart_args", "--xall --json") + //c.SetDefault("collect.short.command", "-a -o on -S on") //if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig @@ -90,16 +99,89 @@ func (c *configuration) ValidateConfig() error { // check that device prefix matches OS // check that schema of config file is valid - return nil + // check that the collector commands are valid + commandArgStrings := map[string]string{ + "commands.metrics_scan_args": c.GetString("commands.metrics_scan_args"), + "commands.metrics_info_args": c.GetString("commands.metrics_info_args"), + "commands.metrics_smart_args": c.GetString("commands.metrics_smart_args"), + } + + errorStrings := []string{} + for configKey, commandArgString := range commandArgStrings { + args := strings.Split(commandArgString, " ") + //ensure that the args string contains `--json` or `-j` flag + containsJsonFlag := false + containsDeviceFlag := false + for _, flag := range args { + if strings.HasPrefix(flag, "--json") || strings.HasPrefix(flag, "-j") { + containsJsonFlag = true + } + if strings.HasPrefix(flag, "--device") || strings.HasPrefix(flag, "-d") { + containsDeviceFlag = true + } + } + + if !containsJsonFlag { + errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' is missing '--json' flag", configKey)) + } + + if containsDeviceFlag { + errorStrings = append(errorStrings, fmt.Sprintf("configuration key '%s' must not contain '--device' or '-d' flag", configKey)) + } + } + //sort(errorStrings) + sort.Strings(errorStrings) + + if len(errorStrings) == 0 { + return nil + } else { + return errors.ConfigValidationError(strings.Join(errorStrings, ", ")) + } } -func (c *configuration) GetScanOverrides() []models.ScanOverride { +func (c *configuration) GetDeviceOverrides() []models.ScanOverride { // we have to support 2 types of device types. // - simple device type (device_type: 'sat') // and list of device types (type: \n- 3ware,0 \n- 3ware,1 \n- 3ware,2) // GetString will return "" if this is a list of device types. - overrides := []models.ScanOverride{} - c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true }) - return overrides + if c.deviceOverrides == nil { + overrides := []models.ScanOverride{} + c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true }) + c.deviceOverrides = overrides + } + + return c.deviceOverrides +} + +func (c *configuration) GetCommandMetricsInfoArgs(deviceName string) string { + overrides := c.GetDeviceOverrides() + + for _, deviceOverrides := range overrides { + if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) { + //found matching device + if len(deviceOverrides.Commands.MetricsInfoArgs) > 0 { + return deviceOverrides.Commands.MetricsInfoArgs + } else { + return c.GetString("commands.metrics_info_args") + } + } + } + return c.GetString("commands.metrics_info_args") +} + +func (c *configuration) GetCommandMetricsSmartArgs(deviceName string) string { + overrides := c.GetDeviceOverrides() + + for _, deviceOverrides := range overrides { + if strings.ToLower(deviceName) == strings.ToLower(deviceOverrides.Device) { + //found matching device + if len(deviceOverrides.Commands.MetricsSmartArgs) > 0 { + return deviceOverrides.Commands.MetricsSmartArgs + } else { + return c.GetString("commands.metrics_smart_args") + } + } + } + return c.GetString("commands.metrics_smart_args") } diff --git a/collector/pkg/config/config_test.go b/collector/pkg/config/config_test.go index 8511c14..d5af79d 100644 --- a/collector/pkg/config/config_test.go +++ b/collector/pkg/config/config_test.go @@ -30,7 +30,7 @@ func TestConfiguration_GetScanOverrides_Simple(t *testing.T) { //test err := testConfig.ReadConfig(path.Join("testdata", "simple_device.yaml")) require.NoError(t, err, "should correctly load simple device config") - scanOverrides := testConfig.GetScanOverrides() + scanOverrides := testConfig.GetDeviceOverrides() //assert require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat"}, Ignore: false}}, scanOverrides) @@ -45,7 +45,7 @@ func TestConfiguration_GetScanOverrides_Ignore(t *testing.T) { //test err := testConfig.ReadConfig(path.Join("testdata", "ignore_device.yaml")) require.NoError(t, err, "should correctly load ignore device config") - scanOverrides := testConfig.GetScanOverrides() + scanOverrides := testConfig.GetDeviceOverrides() //assert require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}, scanOverrides) @@ -60,7 +60,7 @@ func TestConfiguration_GetScanOverrides_Raid(t *testing.T) { //test err := testConfig.ReadConfig(path.Join("testdata", "raid_device.yaml")) require.NoError(t, err, "should correctly load ignore device config") - scanOverrides := testConfig.GetScanOverrides() + scanOverrides := testConfig.GetDeviceOverrides() //assert require.Equal(t, []models.ScanOverride{ @@ -75,3 +75,53 @@ func TestConfiguration_GetScanOverrides_Raid(t *testing.T) { Ignore: false, }}, scanOverrides) } + +func TestConfiguration_InvalidCommands_MissingJson(t *testing.T) { + t.Parallel() + + //setup + testConfig, _ := config.Create() + + //test + err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_missing_json.yaml")) + require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_scan_args' is missing '--json' flag"`, "should throw an error because json flag is missing") +} + +func TestConfiguration_InvalidCommands_IncludesDevice(t *testing.T) { + t.Parallel() + + //setup + testConfig, _ := config.Create() + + //test + err := testConfig.ReadConfig(path.Join("testdata", "invalid_commands_includes_device.yaml")) + require.EqualError(t, err, `ConfigValidationError: "configuration key 'commands.metrics_info_args' must not contain '--device' or '-d' flag, configuration key 'commands.metrics_smart_args' must not contain '--device' or '-d' flag"`, "should throw an error because device flags detected") +} + +func TestConfiguration_OverrideCommands(t *testing.T) { + t.Parallel() + + //setup + testConfig, _ := config.Create() + + //test + err := testConfig.ReadConfig(path.Join("testdata", "override_commands.yaml")) + require.NoError(t, err, "should not throw an error") + require.Equal(t, "--xall --json -T permissive", testConfig.GetString("commands.metrics_smart_args")) +} + +func TestConfiguration_OverrideDeviceCommands_MetricsInfoArgs(t *testing.T) { + t.Parallel() + + //setup + testConfig, _ := config.Create() + + //test + err := testConfig.ReadConfig(path.Join("testdata", "override_device_commands.yaml")) + require.NoError(t, err, "should correctly override device command") + + //assert + require.Equal(t, "--info --json -T permissive", testConfig.GetCommandMetricsInfoArgs("/dev/sda")) + require.Equal(t, "--info --json", testConfig.GetCommandMetricsInfoArgs("/dev/sdb")) + //require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Commands: {MetricsInfoArgs: "--info --json -T "}}}, scanOverrides) +} diff --git a/collector/pkg/config/interface.go b/collector/pkg/config/interface.go index 53dd2b8..e810cec 100644 --- a/collector/pkg/config/interface.go +++ b/collector/pkg/config/interface.go @@ -22,5 +22,7 @@ type Interface interface { GetStringSlice(key string) []string UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error - GetScanOverrides() []models.ScanOverride + GetDeviceOverrides() []models.ScanOverride + GetCommandMetricsInfoArgs(deviceName string) string + GetCommandMetricsSmartArgs(deviceName string) string } diff --git a/collector/pkg/config/mock/mock_config.go b/collector/pkg/config/mock/mock_config.go index 4c1d1d1..1b135b6 100644 --- a/collector/pkg/config/mock/mock_config.go +++ b/collector/pkg/config/mock/mock_config.go @@ -5,144 +5,121 @@ package mock_config import ( + reflect "reflect" + models "github.com/analogj/scrutiny/collector/pkg/models" gomock "github.com/golang/mock/gomock" viper "github.com/spf13/viper" - reflect "reflect" ) -// MockInterface is a mock of Interface interface +// MockInterface is a mock of Interface interface. type MockInterface struct { ctrl *gomock.Controller recorder *MockInterfaceMockRecorder } -// MockInterfaceMockRecorder is the mock recorder for MockInterface +// MockInterfaceMockRecorder is the mock recorder for MockInterface. type MockInterfaceMockRecorder struct { mock *MockInterface } -// NewMockInterface creates a new mock instance +// NewMockInterface creates a new mock instance. func NewMockInterface(ctrl *gomock.Controller) *MockInterface { mock := &MockInterface{ctrl: ctrl} mock.recorder = &MockInterfaceMockRecorder{mock} return mock } -// EXPECT returns an object that allows the caller to indicate expected use +// EXPECT returns an object that allows the caller to indicate expected use. func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { return m.recorder } -// Init mocks base method -func (m *MockInterface) Init() error { +// AllSettings mocks base method. +func (m *MockInterface) AllSettings() map[string]interface{} { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Init") - ret0, _ := ret[0].(error) + ret := m.ctrl.Call(m, "AllSettings") + ret0, _ := ret[0].(map[string]interface{}) return ret0 } -// Init indicates an expected call of Init -func (mr *MockInterfaceMockRecorder) Init() *gomock.Call { +// AllSettings indicates an expected call of AllSettings. +func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings)) } -// ReadConfig mocks base method -func (m *MockInterface) ReadConfig(configFilePath string) error { +// Get mocks base method. +func (m *MockInterface) Get(key string) interface{} { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReadConfig", configFilePath) - ret0, _ := ret[0].(error) + ret := m.ctrl.Call(m, "Get", key) + ret0, _ := ret[0].(interface{}) return ret0 } -// ReadConfig indicates an expected call of ReadConfig -func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath) -} - -// Set mocks base method -func (m *MockInterface) Set(key string, value interface{}) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Set", key, value) -} - -// Set indicates an expected call of Set -func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value) -} - -// SetDefault mocks base method -func (m *MockInterface) SetDefault(key string, value interface{}) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "SetDefault", key, value) -} - -// SetDefault indicates an expected call of SetDefault -func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call { +// Get indicates an expected call of Get. +func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key) } -// AllSettings mocks base method -func (m *MockInterface) AllSettings() map[string]interface{} { +// GetBool mocks base method. +func (m *MockInterface) GetBool(key string) bool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllSettings") - ret0, _ := ret[0].(map[string]interface{}) + ret := m.ctrl.Call(m, "GetBool", key) + ret0, _ := ret[0].(bool) return ret0 } -// AllSettings indicates an expected call of AllSettings -func (mr *MockInterfaceMockRecorder) AllSettings() *gomock.Call { +// GetBool indicates an expected call of GetBool. +func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllSettings", reflect.TypeOf((*MockInterface)(nil).AllSettings)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key) } -// IsSet mocks base method -func (m *MockInterface) IsSet(key string) bool { +// GetCommandMetricsInfoArgs mocks base method. +func (m *MockInterface) GetCommandMetricsInfoArgs(deviceName string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsSet", key) - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "GetCommandMetricsInfoArgs", deviceName) + ret0, _ := ret[0].(string) return ret0 } -// IsSet indicates an expected call of IsSet -func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call { +// GetCommandMetricsInfoArgs indicates an expected call of GetCommandMetricsInfoArgs. +func (mr *MockInterfaceMockRecorder) GetCommandMetricsInfoArgs(deviceName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsInfoArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsInfoArgs), deviceName) } -// Get mocks base method -func (m *MockInterface) Get(key string) interface{} { +// GetCommandMetricsSmartArgs mocks base method. +func (m *MockInterface) GetCommandMetricsSmartArgs(deviceName string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", key) - ret0, _ := ret[0].(interface{}) + ret := m.ctrl.Call(m, "GetCommandMetricsSmartArgs", deviceName) + ret0, _ := ret[0].(string) return ret0 } -// Get indicates an expected call of Get -func (mr *MockInterfaceMockRecorder) Get(key interface{}) *gomock.Call { +// GetCommandMetricsSmartArgs indicates an expected call of GetCommandMetricsSmartArgs. +func (mr *MockInterfaceMockRecorder) GetCommandMetricsSmartArgs(deviceName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), key) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCommandMetricsSmartArgs", reflect.TypeOf((*MockInterface)(nil).GetCommandMetricsSmartArgs), deviceName) } -// GetBool mocks base method -func (m *MockInterface) GetBool(key string) bool { +// GetDeviceOverrides mocks base method. +func (m *MockInterface) GetDeviceOverrides() []models.ScanOverride { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBool", key) - ret0, _ := ret[0].(bool) + ret := m.ctrl.Call(m, "GetDeviceOverrides") + ret0, _ := ret[0].([]models.ScanOverride) return ret0 } -// GetBool indicates an expected call of GetBool -func (mr *MockInterfaceMockRecorder) GetBool(key interface{}) *gomock.Call { +// GetDeviceOverrides indicates an expected call of GetDeviceOverrides. +func (mr *MockInterfaceMockRecorder) GetDeviceOverrides() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBool", reflect.TypeOf((*MockInterface)(nil).GetBool), key) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceOverrides", reflect.TypeOf((*MockInterface)(nil).GetDeviceOverrides)) } -// GetInt mocks base method +// GetInt mocks base method. func (m *MockInterface) GetInt(key string) int { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetInt", key) @@ -150,13 +127,13 @@ func (m *MockInterface) GetInt(key string) int { return ret0 } -// GetInt indicates an expected call of GetInt +// GetInt indicates an expected call of GetInt. func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key) } -// GetString mocks base method +// GetString mocks base method. func (m *MockInterface) GetString(key string) string { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetString", key) @@ -164,13 +141,13 @@ func (m *MockInterface) GetString(key string) string { return ret0 } -// GetString indicates an expected call of GetString +// GetString indicates an expected call of GetString. func (mr *MockInterfaceMockRecorder) GetString(key interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetString", reflect.TypeOf((*MockInterface)(nil).GetString), key) } -// GetStringSlice mocks base method +// GetStringSlice mocks base method. func (m *MockInterface) GetStringSlice(key string) []string { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetStringSlice", key) @@ -178,13 +155,79 @@ func (m *MockInterface) GetStringSlice(key string) []string { return ret0 } -// GetStringSlice indicates an expected call of GetStringSlice +// GetStringSlice indicates an expected call of GetStringSlice. func (mr *MockInterfaceMockRecorder) GetStringSlice(key interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStringSlice", reflect.TypeOf((*MockInterface)(nil).GetStringSlice), key) } -// UnmarshalKey mocks base method +// Init mocks base method. +func (m *MockInterface) Init() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init") + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockInterfaceMockRecorder) Init() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockInterface)(nil).Init)) +} + +// IsSet mocks base method. +func (m *MockInterface) IsSet(key string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsSet", key) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsSet indicates an expected call of IsSet. +func (mr *MockInterfaceMockRecorder) IsSet(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSet", reflect.TypeOf((*MockInterface)(nil).IsSet), key) +} + +// ReadConfig mocks base method. +func (m *MockInterface) ReadConfig(configFilePath string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadConfig", configFilePath) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReadConfig indicates an expected call of ReadConfig. +func (mr *MockInterfaceMockRecorder) ReadConfig(configFilePath interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadConfig", reflect.TypeOf((*MockInterface)(nil).ReadConfig), configFilePath) +} + +// Set mocks base method. +func (m *MockInterface) Set(key string, value interface{}) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Set", key, value) +} + +// Set indicates an expected call of Set. +func (mr *MockInterfaceMockRecorder) Set(key, value interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockInterface)(nil).Set), key, value) +} + +// SetDefault mocks base method. +func (m *MockInterface) SetDefault(key string, value interface{}) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetDefault", key, value) +} + +// SetDefault indicates an expected call of SetDefault. +func (mr *MockInterfaceMockRecorder) SetDefault(key, value interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefault", reflect.TypeOf((*MockInterface)(nil).SetDefault), key, value) +} + +// UnmarshalKey mocks base method. func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error { m.ctrl.T.Helper() varargs := []interface{}{key, rawVal} @@ -196,23 +239,9 @@ func (m *MockInterface) UnmarshalKey(key string, rawVal interface{}, decoderOpts return ret0 } -// UnmarshalKey indicates an expected call of UnmarshalKey +// UnmarshalKey indicates an expected call of UnmarshalKey. func (mr *MockInterfaceMockRecorder) UnmarshalKey(key, rawVal interface{}, decoderOpts ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() varargs := append([]interface{}{key, rawVal}, decoderOpts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnmarshalKey", reflect.TypeOf((*MockInterface)(nil).UnmarshalKey), varargs...) } - -// GetScanOverrides mocks base method -func (m *MockInterface) GetScanOverrides() []models.ScanOverride { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetScanOverrides") - ret0, _ := ret[0].([]models.ScanOverride) - return ret0 -} - -// GetScanOverrides indicates an expected call of GetScanOverrides -func (mr *MockInterfaceMockRecorder) GetScanOverrides() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScanOverrides", reflect.TypeOf((*MockInterface)(nil).GetScanOverrides)) -} diff --git a/collector/pkg/config/testdata/invalid_commands_includes_device.yaml b/collector/pkg/config/testdata/invalid_commands_includes_device.yaml new file mode 100644 index 0000000..1e6386f --- /dev/null +++ b/collector/pkg/config/testdata/invalid_commands_includes_device.yaml @@ -0,0 +1,4 @@ +commands: + metrics_scan_args: '--scan --json' # used to detect devices + metrics_info_args: '--info --json --device=sat' # used to determine device unique ID & register device with Scrutiny + metrics_smart_args: '--xall --json -d sat' # used to retrieve smart data for each device. diff --git a/collector/pkg/config/testdata/invalid_commands_missing_json.yaml b/collector/pkg/config/testdata/invalid_commands_missing_json.yaml new file mode 100644 index 0000000..3ecc667 --- /dev/null +++ b/collector/pkg/config/testdata/invalid_commands_missing_json.yaml @@ -0,0 +1,4 @@ +commands: + metrics_scan_args: '--scan' # used to detect devices + metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny + metrics_smart_args: '--xall --json' # used to retrieve smart data for each device. diff --git a/collector/pkg/config/testdata/override_commands.yaml b/collector/pkg/config/testdata/override_commands.yaml new file mode 100644 index 0000000..77a3b88 --- /dev/null +++ b/collector/pkg/config/testdata/override_commands.yaml @@ -0,0 +1,4 @@ +commands: + metrics_scan_args: '--scan --json' # used to detect devices + metrics_info_args: '--info -j' # used to determine device unique ID & register device with Scrutiny + metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device. diff --git a/collector/pkg/config/testdata/override_device_commands.yaml b/collector/pkg/config/testdata/override_device_commands.yaml new file mode 100644 index 0000000..fc4fd3d --- /dev/null +++ b/collector/pkg/config/testdata/override_device_commands.yaml @@ -0,0 +1,5 @@ +version: 1 +devices: + - device: /dev/sda + commands: + metrics_info_args: "--info --json -T permissive" \ No newline at end of file diff --git a/collector/pkg/detect/detect.go b/collector/pkg/detect/detect.go index 1475453..5f2fa55 100644 --- a/collector/pkg/detect/detect.go +++ b/collector/pkg/detect/detect.go @@ -28,7 +28,8 @@ type Detect struct { // models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file). func (d *Detect) SmartctlScan() ([]models.Device, error) { //we use smartctl to detect all the drives available. - detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ()) + args := strings.Split(d.Config.GetString("commands.metrics_scan_args"), " ") + detectedDeviceConnJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ()) if err != nil { d.Logger.Errorf("Error scanning for devices: %v", err) return nil, err @@ -51,13 +52,13 @@ func (d *Detect) SmartctlScan() ([]models.Device, error) { // - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves // - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN. func (d *Detect) SmartCtlInfo(device *models.Device) error { - - args := []string{"--info", "-j"} + fullDeviceName := fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName) + args := strings.Split(d.Config.GetCommandMetricsInfoArgs(fullDeviceName), " ") //only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost. if len(device.DeviceType) > 0 && device.DeviceType != "scsi" && device.DeviceType != "ata" { - args = append(args, "-d", device.DeviceType) + args = append(args, "--device", device.DeviceType) } - args = append(args, fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName)) + args = append(args, fullDeviceName) availableDeviceInfoJson, err := d.Shell.Command(d.Logger, "smartctl", args, "", os.Environ()) if err != nil { @@ -138,7 +139,7 @@ func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []mod //now tha we've "grouped" all the devices, lets override any groups specified in the config file. - for _, overrideDevice := range d.Config.GetScanOverrides() { + for _, overrideDevice := range d.Config.GetDeviceOverrides() { overrideDeviceFile := strings.ToLower(overrideDevice.Device) if overrideDevice.Ignore { diff --git a/collector/pkg/detect/detect_test.go b/collector/pkg/detect/detect_test.go index 1c00ee7..70cb2f6 100644 --- a/collector/pkg/detect/detect_test.go +++ b/collector/pkg/detect/detect_test.go @@ -18,7 +18,8 @@ func TestDetect_SmartctlScan(t *testing.T) { defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") - fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{}) + fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{}) + fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json") fakeShell := mock_shell.NewMockInterface(mockCtrl) testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_simple.json") @@ -45,7 +46,8 @@ func TestDetect_SmartctlScan_Megaraid(t *testing.T) { defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") - fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{}) + fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{}) + fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json") fakeShell := mock_shell.NewMockInterface(mockCtrl) testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_megaraid.json") @@ -75,7 +77,8 @@ func TestDetect_SmartctlScan_Nvme(t *testing.T) { defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") - fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{}) + fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{}) + fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json") fakeShell := mock_shell.NewMockInterface(mockCtrl) testScanResults, err := ioutil.ReadFile("testdata/smartctl_scan_nvme.json") @@ -104,7 +107,9 @@ func TestDetect_TransformDetectedDevices_Empty(t *testing.T) { defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") - fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{}) + fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{}) + fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json") + detectedDevices := models.Scan{ Devices: []models.ScanDevice{ { @@ -134,7 +139,9 @@ func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) { defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") - fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}) + fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}) + fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json") + detectedDevices := models.Scan{ Devices: []models.ScanDevice{ { @@ -163,7 +170,8 @@ func TestDetect_TransformDetectedDevices_Raid(t *testing.T) { defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") - fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{ + fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json") + fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{ { Device: "/dev/bus/0", DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"}, @@ -202,7 +210,8 @@ func TestDetect_TransformDetectedDevices_Simple(t *testing.T) { defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("") - fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}}) + fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json") + fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}}) detectedDevices := models.Scan{ Devices: []models.ScanDevice{ { diff --git a/collector/pkg/models/scan_override.go b/collector/pkg/models/scan_override.go index 682687d..406582d 100644 --- a/collector/pkg/models/scan_override.go +++ b/collector/pkg/models/scan_override.go @@ -4,4 +4,8 @@ type ScanOverride struct { Device string `mapstructure:"device"` DeviceType []string `mapstructure:"type"` Ignore bool `mapstructure:"ignore"` + Commands struct { + MetricsInfoArgs string `mapstructure:"metrics_info_args"` + MetricsSmartArgs string `mapstructure:"metrics_smart_args"` + } `mapstructure:"commands"` } diff --git a/example.collector.yaml b/example.collector.yaml index 854de94..60ab408 100644 --- a/example.collector.yaml +++ b/example.collector.yaml @@ -53,6 +53,13 @@ devices: # - 3ware,3 # - 3ware,4 # - 3ware,5 +# +# # example to show how to override the smartctl command args (per device), see below for how to override these globally. +# - device: /dev/sda +# commands: +# metrics_info_args: '--info --json -T permissive' # used to determine device unique ID & register device with Scrutiny +# metrics_smart_args: '--xall --json -T permissive' # used to retrieve smart data for each device. + #log: # file: '' #absolute or relative paths allowed, eg. web.log @@ -64,6 +71,12 @@ devices: # if you need to use a custom base path (for a reverse proxy), you can add a suffix to the endpoint. # See docs/TROUBLESHOOTING_REVERSE_PROXY.md for more info, +# example to show how to override the smartctl command args globally +#commands: +# metrics_scan_args: '--scan --json' # used to detect devices +# metrics_info_args: '--info --json' # used to determine device unique ID & register device with Scrutiny +# metrics_smart_args: '--xall --json' # used to retrieve smart data for each device. + ######################################################################################################################## # FEATURES COMING SOON