diff --git a/collector/cmd/collector-metrics/collector-metrics.go b/collector/cmd/collector-metrics/collector-metrics.go index c7f76ac..dcaed1f 100644 --- a/collector/cmd/collector-metrics/collector-metrics.go +++ b/collector/cmd/collector-metrics/collector-metrics.go @@ -5,6 +5,7 @@ import ( "github.com/analogj/scrutiny/collector/pkg/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/sirupsen/logrus" + "io" "log" "os" "time" @@ -85,6 +86,16 @@ OPTIONS: logrus.SetLevel(logrus.InfoLevel) } + if c.IsSet("log-file") { + logFile, err := os.OpenFile(c.String("log-file"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + logrus.Errorf("Failed to open log file %s for output: %s", c.String("log-file"), err) + return err + } + defer logFile.Close() + logrus.SetOutput(io.MultiWriter(os.Stderr, logFile)) + } + metricCollector, err := collector.CreateMetricsCollector( collectorLogger, c.String("api-endpoint"), @@ -105,6 +116,12 @@ OPTIONS: EnvVars: []string{"SCRUTINY_API_ENDPOINT"}, }, + &cli.StringFlag{ + Name: "log-file", + Usage: "Path to file for logging. Leave empty to use STDOUT", + Value: "", + }, + &cli.BoolFlag{ Name: "debug", Usage: "Enable debug logging", diff --git a/collector/cmd/collector-selftest/collector-selftest.go b/collector/cmd/collector-selftest/collector-selftest.go index 3800a5f..aa01826 100644 --- a/collector/cmd/collector-selftest/collector-selftest.go +++ b/collector/cmd/collector-selftest/collector-selftest.go @@ -5,6 +5,7 @@ import ( "github.com/analogj/scrutiny/collector/pkg/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/sirupsen/logrus" + "io" "log" "os" "time" @@ -85,6 +86,16 @@ OPTIONS: logrus.SetLevel(logrus.InfoLevel) } + if c.IsSet("log-file") { + logFile, err := os.OpenFile(c.String("log-file"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + logrus.Errorf("Failed to open log file %s for output: %s", c.String("log-file"), err) + return err + } + defer logFile.Close() + logrus.SetOutput(io.MultiWriter(os.Stderr, logFile)) + } + stCollector, err := collector.CreateSelfTestCollector( collectorLogger, c.String("api-endpoint"), @@ -105,6 +116,12 @@ OPTIONS: EnvVars: []string{"SCRUTINY_API_ENDPOINT"}, }, + &cli.StringFlag{ + Name: "log-file", + Usage: "Path to file for logging. Leave empty to use STDOUT", + Value: "", + }, + &cli.BoolFlag{ Name: "debug", Usage: "Enable debug logging", diff --git a/collector/pkg/collector/base.go b/collector/pkg/collector/base.go index c35cd53..ff1801f 100644 --- a/collector/pkg/collector/base.go +++ b/collector/pkg/collector/base.go @@ -3,15 +3,8 @@ package collector import ( "bytes" "encoding/json" - "errors" - "github.com/analogj/scrutiny/collector/pkg/models" - "github.com/jaypipes/ghw" "github.com/sirupsen/logrus" - "io" "net/http" - "os" - "os/exec" - "path" "time" ) @@ -21,77 +14,6 @@ type BaseCollector struct { logger *logrus.Entry } -func (c *BaseCollector) DetectStorageDevices() ([]models.Device, error) { - - //availableDisksJson, err := c.ExecCmd("smartctl", []string{"-j", "--scan"}, "", os.Environ()) - //if err != nil { - // c.logger.Errorf("Error getting block storage info: %v", err) - // return nil, err - //} - // - //var smartctlScan models.Scan - //err = json.Unmarshal([]byte(availableDisksJson), &smartctlScan) - //if err != nil { - // return nil, err - //} - - block, err := ghw.Block() - if err != nil { - c.logger.Errorf("Error getting block storage info: %v", err) - return nil, err - } - - approvedDisks := []models.Device{} - for _, disk := range block.Disks { - - // ignore optical drives and floppy disks - if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD { - c.logger.Debugf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String()) - continue - } - - // ignore removable disks - if disk.IsRemovable { - c.logger.Debugf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable) - continue - } - - // ignore virtual disks & mobile phone storage devices - if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC { - c.logger.Debugf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String()) - continue - } - - // Skip unknown storage controllers, not usually S.M.A.R.T compatible. - if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN { - c.logger.Debugf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String()) - continue - } - - diskModel := models.Device{ - WWN: disk.WWN, - Manufacturer: disk.Vendor, - ModelName: disk.Model, - InterfaceType: disk.StorageController.String(), - //InterfaceSpeed: string - SerialNumber: disk.SerialNumber, - Capacity: int64(disk.SizeBytes), - //Firmware string - //RotationSpeed int - - DeviceName: disk.Name, - } - if len(diskModel.WWN) == 0 { - //(macOS and some other os's) do not provide a WWN, so we're going to fallback to - //diskname as identifier if WWN is not present - diskModel.WWN = disk.Name - } - - approvedDisks = append(approvedDisks, diskModel) - } - return approvedDisks, nil -} - func (c *BaseCollector) getJson(url string, target interface{}) error { r, err := httpClient.Get(url) @@ -118,29 +40,6 @@ func (c *BaseCollector) postJson(url string, body interface{}, target interface{ return json.NewDecoder(r.Body).Decode(target) } -func (c *BaseCollector) ExecCmd(cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) { - - cmd := exec.Command(cmdName, cmdArgs...) - var stdBuffer bytes.Buffer - mw := io.MultiWriter(os.Stdout, &stdBuffer) - - cmd.Stdout = mw - cmd.Stderr = mw - - if environ != nil { - cmd.Env = environ - } - if workingDir != "" && path.IsAbs(workingDir) { - cmd.Dir = workingDir - } else if workingDir != "" { - return "", errors.New("Working Directory must be an absolute path") - } - - err := cmd.Run() - return stdBuffer.String(), err - -} - func (c *BaseCollector) LogSmartctlExitCode(exitCode int) { if exitCode&0x01 != 0 { c.logger.Errorln("smartctl could not parse commandline") diff --git a/collector/pkg/collector/metrics.go b/collector/pkg/collector/metrics.go index 90b13b4..6061cd5 100644 --- a/collector/pkg/collector/metrics.go +++ b/collector/pkg/collector/metrics.go @@ -2,11 +2,15 @@ package collector import ( "bytes" + "encoding/json" "fmt" + "github.com/analogj/scrutiny/collector/pkg/common" + "github.com/analogj/scrutiny/collector/pkg/detect" "github.com/analogj/scrutiny/collector/pkg/errors" "github.com/analogj/scrutiny/collector/pkg/models" "github.com/sirupsen/logrus" "net/url" + "os" "os/exec" "strings" "sync" @@ -43,13 +47,18 @@ func (mc *MetricsCollector) Run() error { apiEndpoint.Path = "/api/devices/register" deviceRespWrapper := new(models.DeviceWrapper) - detectedStorageDevices, err := mc.DetectStorageDevices() + + deviceDetector := detect.Detect{ + Logger: mc.logger, + } + detectedStorageDevices, err := deviceDetector.Start() if err != nil { return err } mc.logger.Infoln("Sending detected devices to API, for filtering & validation") - mc.logger.Debugf("Detected devices: %v", detectedStorageDevices) + jsonObj, _ := json.Marshal(detectedStorageDevices) + mc.logger.Debugf("Detected devices: %v", string(jsonObj)) err = mc.postJson(apiEndpoint.String(), models.DeviceWrapper{ Data: detectedStorageDevices, }, &deviceRespWrapper) @@ -67,7 +76,7 @@ func (mc *MetricsCollector) Run() error { for _, device := range deviceRespWrapper.Data { // execute collection in parallel go-routines wg.Add(1) - go mc.Collect(&wg, device.WWN, device.DeviceName) + go mc.Collect(&wg, device.WWN, device.DeviceName, device.DeviceType) } mc.logger.Infoln("Main: Waiting for workers to finish") @@ -89,11 +98,18 @@ func (mc *MetricsCollector) Validate() error { return nil } -func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string) { +func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) { defer wg.Done() mc.logger.Infof("Collecting smartctl results for %s\n", deviceName) - result, err := mc.ExecCmd("smartctl", []string{"-a", "-j", fmt.Sprintf("/dev/%s", deviceName)}, "", nil) + args := []string{"-a", "-j"} + //only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost. + if len(deviceType) > 0 && deviceType != "scsi" && deviceType != "ata" { + args = append(args, "-d", deviceType) + } + args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName)) + + result, err := common.ExecCmd(mc.logger, "smartctl", args, "", os.Environ()) resultBytes := []byte(result) if err != nil { if exitError, ok := err.(*exec.ExitError); ok { @@ -121,6 +137,7 @@ func (mc *MetricsCollector) Publish(deviceWWN string, payload []byte) error { resp, err := httpClient.Post(apiEndpoint.String(), "application/json", bytes.NewBuffer(payload)) if err != nil { + mc.logger.Errorf("An error occurred while publishing SMART data for device (%s): %v", deviceWWN, err) return err } defer resp.Body.Close() diff --git a/collector/pkg/common/exec.go b/collector/pkg/common/exec.go new file mode 100644 index 0000000..b17ea22 --- /dev/null +++ b/collector/pkg/common/exec.go @@ -0,0 +1,35 @@ +package common + +import ( + "bytes" + "errors" + "github.com/sirupsen/logrus" + "io" + "os/exec" + "path" + "strings" +) + +func ExecCmd(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) { + logger.Infof("Executing command: %s %s", cmdName, strings.Join(cmdArgs, " ")) + + cmd := exec.Command(cmdName, cmdArgs...) + var stdBuffer bytes.Buffer + mw := io.MultiWriter(logger.Logger.Out, &stdBuffer) + + cmd.Stdout = mw + cmd.Stderr = mw + + if environ != nil { + cmd.Env = environ + } + if workingDir != "" && path.IsAbs(workingDir) { + cmd.Dir = workingDir + } else if workingDir != "" { + return "", errors.New("Working Directory must be an absolute path") + } + + err := cmd.Run() + return stdBuffer.String(), err + +} diff --git a/collector/pkg/collector/base_test.go b/collector/pkg/common/exec_test.go similarity index 67% rename from collector/pkg/collector/base_test.go rename to collector/pkg/common/exec_test.go index def6dff..846417c 100644 --- a/collector/pkg/collector/base_test.go +++ b/collector/pkg/common/exec_test.go @@ -1,7 +1,8 @@ -package collector_test +package common_test import ( - "github.com/analogj/scrutiny/collector/pkg/collector" + "github.com/analogj/scrutiny/collector/pkg/common" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "os/exec" "testing" @@ -11,10 +12,9 @@ func TestExecCmd(t *testing.T) { t.Parallel() //setup - bc := collector.BaseCollector{} //test - result, err := bc.ExecCmd("echo", []string{"hello world"}, "", nil) + result, err := common.ExecCmd(logrus.WithField("exec", "test"), "echo", []string{"hello world"}, "", nil) //assert require.NoError(t, err) @@ -25,10 +25,9 @@ func TestExecCmd_Date(t *testing.T) { t.Parallel() //setup - bc := collector.BaseCollector{} //test - _, err := bc.ExecCmd("date", []string{}, "", nil) + _, err := common.ExecCmd(logrus.WithField("exec", "test"), "date", []string{}, "", nil) //assert require.NoError(t, err) @@ -56,10 +55,9 @@ func TestExecCmd_InvalidCommand(t *testing.T) { t.Parallel() //setup - bc := collector.BaseCollector{} //test - _, err := bc.ExecCmd("invalid_binary", []string{}, "", nil) + _, err := common.ExecCmd(logrus.WithField("exec", "test"), "invalid_binary", []string{}, "", nil) //assert _, castOk := err.(*exec.ExitError) diff --git a/collector/pkg/detect/detect.go b/collector/pkg/detect/detect.go new file mode 100644 index 0000000..7eb830a --- /dev/null +++ b/collector/pkg/detect/detect.go @@ -0,0 +1,116 @@ +package detect + +import ( + "encoding/json" + "fmt" + "github.com/analogj/scrutiny/collector/pkg/common" + "github.com/analogj/scrutiny/collector/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + "github.com/denisbrodbeck/machineid" + "github.com/sirupsen/logrus" + "os" + "strings" +) + +type Detect struct { + Logger *logrus.Entry +} + +//private/common functions + +// This function calls smartctl --scan which can be used to detect storage devices. +// It has a couple of issues however: +// - --scan does not return any results on mac +// +// To handle these issues, we have OS specific wrapper functions that update/modify these detected devices. +// models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file). +func (d *Detect) smartctlScan() ([]models.Device, error) { + //we use smartctl to detect all the drives available. + detectedDeviceConnJson, err := common.ExecCmd(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ()) + if err != nil { + d.Logger.Errorf("Error scanning for devices: %v", err) + return nil, err + } + + var detectedDeviceConns models.Scan + err = json.Unmarshal([]byte(detectedDeviceConnJson), &detectedDeviceConns) + if err != nil { + d.Logger.Errorf("Error decoding detected devices: %v", err) + return nil, err + } + + detectedDevices := []models.Device{} + + for _, detectedDevice := range detectedDeviceConns.Devices { + detectedDevices = append(detectedDevices, models.Device{ + DeviceType: detectedDevice.Type, + DeviceName: strings.TrimPrefix(detectedDevice.Name, DevicePrefix()), + }) + } + + return detectedDevices, nil +} + +//updates a device model with information from smartctl --scan +// It has a couple of issues however: +// - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves +// - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN. +func (d *Detect) smartCtlInfo(device *models.Device) error { + + args := []string{"--info", "-j"} + //only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost. + if len(device.DeviceType) > 0 && device.DeviceType != "scsi" && device.DeviceType != "ata" { + args = append(args, "-d", device.DeviceType) + } + args = append(args, fmt.Sprintf("%s%s", DevicePrefix(), device.DeviceName)) + + availableDeviceInfoJson, err := common.ExecCmd(d.Logger, "smartctl", args, "", os.Environ()) + if err != nil { + d.Logger.Errorf("Could not retrieve device information for %s: %v", device.DeviceName, err) + return err + } + + var availableDeviceInfo collector.SmartInfo + err = json.Unmarshal([]byte(availableDeviceInfoJson), &availableDeviceInfo) + if err != nil { + d.Logger.Errorf("Could not decode device information for %s: %v", device.DeviceName, err) + return err + } + + //DeviceType and DeviceName are already populated. + //WWN + //InterfaceType: + device.ModelName = availableDeviceInfo.ModelName + device.InterfaceSpeed = availableDeviceInfo.InterfaceSpeed.Current.String + device.SerialNumber = availableDeviceInfo.SerialNumber + device.Firmware = availableDeviceInfo.FirmwareVersion + device.RotationSpeed = availableDeviceInfo.RotationRate + device.Capacity = availableDeviceInfo.UserCapacity.Bytes + device.FormFactor = availableDeviceInfo.FormFactor.Name + device.DeviceProtocol = availableDeviceInfo.Device.Protocol + if len(availableDeviceInfo.Vendor) > 0 { + device.Manufacturer = availableDeviceInfo.Vendor + } + + //populate WWN is possible if present + if availableDeviceInfo.Wwn.Naa != 0 { //valid values are 1-6 (5 is what we handle correctly) + d.Logger.Info("Generating WWN") + wwn := Wwn{ + Naa: availableDeviceInfo.Wwn.Naa, + Oui: availableDeviceInfo.Wwn.Oui, + Id: availableDeviceInfo.Wwn.ID, + } + device.WWN = wwn.ToString() + d.Logger.Debugf("NAA: %d OUI: %d Id: %d => WWN: %s", wwn.Naa, wwn.Oui, wwn.Id, device.WWN) + } else { + d.Logger.Info("Using WWN Fallback") + d.wwnFallback(device) + } + + return nil +} + +//uses https://github.com/denisbrodbeck/machineid to get a OS specific unique machine ID. +func (d *Detect) getMachineId() (string, error) { + return machineid.ProtectedID("scrutiny") +} diff --git a/collector/pkg/detect/devices_darwin.go b/collector/pkg/detect/devices_darwin.go new file mode 100644 index 0000000..b2a2ffd --- /dev/null +++ b/collector/pkg/detect/devices_darwin.go @@ -0,0 +1,108 @@ +package detect + +import ( + "github.com/analogj/scrutiny/collector/pkg/models" + "github.com/jaypipes/ghw" + "strings" +) + +func DevicePrefix() string { + return "/dev/" +} + +func (d *Detect) Start() ([]models.Device, error) { + // call the base/common functionality to get a list of devicess + detectedDevices, err := d.smartctlScan() + if err != nil { + return nil, err + } + + //smartctl --scan doesn't seem to detect mac nvme drives, lets see if we can detect them manually. + missingDevices, err := d.findMissingDevices(detectedDevices) //we dont care about the error here, just continue retrieving device info. + if err == nil { + detectedDevices = append(detectedDevices, missingDevices...) + } + + //inflate device info for detected devices. + for ndx, _ := range detectedDevices { + d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors. + } + + return detectedDevices, nil +} + +func (d *Detect) findMissingDevices(detectedDevices []models.Device) ([]models.Device, error) { + + missingDevices := []models.Device{} + + block, err := ghw.Block() + if err != nil { + d.Logger.Errorf("Error getting block storage info: %v", err) + return nil, err + } + + for _, disk := range block.Disks { + + // ignore optical drives and floppy disks + if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD { + d.Logger.Debugf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String()) + continue + } + + // ignore removable disks + if disk.IsRemovable { + d.Logger.Debugf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable) + continue + } + + // ignore virtual disks & mobile phone storage devices + if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC { + d.Logger.Debugf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String()) + continue + } + + // Skip unknown storage controllers, not usually S.M.A.R.T compatible. + if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN { + d.Logger.Debugf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String()) + continue + } + + //check if device is already detected. + alreadyDetected := false + diskName := strings.TrimPrefix(disk.Name, DevicePrefix()) + for _, detectedDevice := range detectedDevices { + + if detectedDevice.DeviceName == diskName { + alreadyDetected = true + break + } + } + if !alreadyDetected { + missingDevices = append(missingDevices, models.Device{ + DeviceName: diskName, + DeviceType: "", + }) + } + } + return missingDevices, nil +} + +//WWN values NVMe and SCSI +func (d *Detect) wwnFallback(detectedDevice *models.Device) { + block, err := ghw.Block() + if err == nil { + for _, disk := range block.Disks { + if disk.Name == detectedDevice.DeviceName { + d.Logger.Debugf("Found matching block device. WWN: %s", disk.WWN) + detectedDevice.WWN = disk.WWN + break + } + } + } + + //no WWN found, or could not open Block devices. Either way, fallback to serial number + if len(detectedDevice.WWN) == 0 { + d.Logger.Debugf("WWN is empty, falling back to serial number: %s", detectedDevice.SerialNumber) + detectedDevice.WWN = detectedDevice.SerialNumber + } +} diff --git a/collector/pkg/detect/devices_linux.go b/collector/pkg/detect/devices_linux.go new file mode 100644 index 0000000..3b12406 --- /dev/null +++ b/collector/pkg/detect/devices_linux.go @@ -0,0 +1,45 @@ +package detect + +import ( + "github.com/analogj/scrutiny/collector/pkg/models" + "github.com/jaypipes/ghw" +) + +func DevicePrefix() string { + return "/dev/" +} + +func (d *Detect) Start() ([]models.Device, error) { + // call the base/common functionality to get a list of devices + detectedDevices, err := d.smartctlScan() + if err != nil { + return nil, err + } + + //inflate device info for detected devices. + for ndx, _ := range detectedDevices { + d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors. + } + + return detectedDevices, nil +} + +//WWN values NVMe and SCSI +func (d *Detect) wwnFallback(detectedDevice *models.Device) { + block, err := ghw.Block() + if err == nil { + for _, disk := range block.Disks { + if disk.Name == detectedDevice.DeviceName { + d.Logger.Debugf("Found matching block device. WWN: %s", disk.WWN) + detectedDevice.WWN = disk.WWN + break + } + } + } + + //no WWN found, or could not open Block devices. Either way, fallback to serial number + if len(detectedDevice.WWN) == 0 { + d.Logger.Debugf("WWN is empty, falling back to serial number: %s", detectedDevice.SerialNumber) + detectedDevice.WWN = detectedDevice.SerialNumber + } +} diff --git a/collector/pkg/detect/devices_windows.go b/collector/pkg/detect/devices_windows.go new file mode 100644 index 0000000..3346842 --- /dev/null +++ b/collector/pkg/detect/devices_windows.go @@ -0,0 +1,29 @@ +package detect + +func DevicePrefix() string { + return "" +} + +func (d *Detect) Start() ([]models.Device, error) { + // call the base/common functionality to get a list of devices + detectedDevices, err := d.smartctlScan() + if err != nil { + return nil, err + } + + //inflate device info for detected devices. + for ndx, _ := range detectedDevices { + d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors. + } + + return detectedDevices, nil +} + +//WWN values NVMe and SCSI +func (d *Detect) wwnFallback(detectedDevice *models.Device) { + + //fallback to serial number + if len(detectedDevice.WWN) == 0 { + detectedDevice.WWN = detectedDevice.SerialNumber + } +} diff --git a/collector/pkg/detect/wwn.go b/collector/pkg/detect/wwn.go new file mode 100644 index 0000000..d9163ca --- /dev/null +++ b/collector/pkg/detect/wwn.go @@ -0,0 +1,58 @@ +package detect + +import ( + "fmt" +) + +type Wwn struct { + Naa uint64 `json:"naa"` + Oui uint64 `json:"oui"` + Id uint64 `json:"id"` + VendorCode string `json:"vendor_code"` +} + +// this is an incredibly basic converter, that only works for "Registered" IEEE format - NAA5 +// https://standards.ieee.org/content/dam/ieee-standards/standards/web/documents/tutorials/fibre.pdf +// references: +// - https://metacpan.org/pod/Device::WWN +// - https://en.wikipedia.org/wiki/World_Wide_Name +// - https://storagemeat.blogspot.com/2012/08/decoding-wwids-or-how-to-tell-whats-what.html +// - https://bryanchain.com/2016/01/20/breaking-down-an-naa-id-world-wide-name/ + +/* ++----------+---+---+---+---+---+---+---+---+ +| Byte/Bit | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | ++----------+---+---+---+---+---+---+---+---+ +| 0 | NAA (5h) | (MSB) | ++----------+---------------+ + +| 1 | | ++----------+ IEEE OUI | +| 2 | | ++----------+ +---------------+ +| 3 | (LSB) | (MSB) | ++----------+---------------+ + +| 4 | | +| | | ++----------+ | +| 5 | Vendor ID | ++----------+ | +| 6 | | ++----------+ | +| 7 | (LSB) | ++----------+-------------------------------+ + + +*/ + +func (wwn *Wwn) ToString() string { + + var wwnBuffer uint64 + + wwnBuffer = wwn.Id //start with vendor ID + wwnBuffer += (wwn.Oui << 36) //add left-shifted OUI + wwnBuffer += (wwn.Naa << 60) //NAA is a number from 1-6, so decimal == hex. + + //TODO: may need to support additional versions in the future. + + return fmt.Sprintf("%#x", wwnBuffer) +} diff --git a/collector/pkg/detect/wwn_test.go b/collector/pkg/detect/wwn_test.go new file mode 100644 index 0000000..d059035 --- /dev/null +++ b/collector/pkg/detect/wwn_test.go @@ -0,0 +1,35 @@ +package detect_test + +import ( + "fmt" + "github.com/analogj/scrutiny/collector/pkg/detect" + "github.com/stretchr/testify/require" + "testing" +) + +func TestWwn_FromStringTable(t *testing.T) { + + //setup + var tests = []struct { + wwnStr string + wwn detect.Wwn + }{ + + {"0x5002538e40a22954", detect.Wwn{Naa: 5, Oui: 9528, Id: 61213911380}}, //sda + {"0x5000cca264eb01d7", detect.Wwn{Naa: 5, Oui: 3274, Id: 10283057623}}, //sdb + {"0x5000cca264ec3183", detect.Wwn{Naa: 5, Oui: 3274, Id: 10283135363}}, //sdc + {"0x5000cca252c859cc", detect.Wwn{Naa: 5, Oui: 3274, Id: 9978796492}}, //sdd + {"0x50014ee20b2a72a9", detect.Wwn{Naa: 5, Oui: 5358, Id: 8777265833}}, //sde + {"0x5000cca264ebc248", detect.Wwn{Naa: 5, Oui: 3274, Id: 10283106888}}, //sdf + {"0x5000c500673e6b5f", detect.Wwn{Naa: 5, Oui: 3152, Id: 1732143967}}, //sdg + } + //test + for _, tt := range tests { + testname := fmt.Sprintf("%s", tt.wwnStr) + t.Run(testname, func(t *testing.T) { + str := tt.wwn.ToString() + require.Equal(t, tt.wwnStr, str) + }) + } + +} diff --git a/go.mod b/go.mod index e9211df..4b9bbb7 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,10 @@ module github.com/analogj/scrutiny go 1.13 require ( - github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae + github.com/AnalogJ/go-util v0.0.0-20200905200945-3b93d31215ae // indirect github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14 github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a + github.com/denisbrodbeck/machineid v1.0.1 github.com/fatih/color v1.9.0 github.com/gin-gonic/gin v1.6.3 github.com/golang/mock v1.4.3 diff --git a/go.sum b/go.sum index 47ec7ef..e32dd06 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= +github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= diff --git a/webapp/backend/pkg/models/collector/smart.go b/webapp/backend/pkg/models/collector/smart.go index c17b0a4..04d5e4c 100644 --- a/webapp/backend/pkg/models/collector/smart.go +++ b/webapp/backend/pkg/models/collector/smart.go @@ -23,9 +23,9 @@ type SmartInfo struct { ModelName string `json:"model_name"` SerialNumber string `json:"serial_number"` Wwn struct { - Naa int `json:"naa"` - Oui int `json:"oui"` - ID int64 `json:"id"` + Naa uint64 `json:"naa"` + Oui uint64 `json:"oui"` + ID uint64 `json:"id"` } `json:"wwn"` FirmwareVersion string `json:"firmware_version"` UserCapacity struct { diff --git a/webapp/backend/pkg/models/db/device.go b/webapp/backend/pkg/models/db/device.go index 8721cea..96e81d4 100644 --- a/webapp/backend/pkg/models/db/device.go +++ b/webapp/backend/pkg/models/db/device.go @@ -152,16 +152,6 @@ func (dv *Device) ApplyMetadataRules() error { } func (dv *Device) UpdateFromCollectorSmartInfo(info collector.SmartInfo) error { - dv.InterfaceSpeed = info.InterfaceSpeed.Current.String dv.Firmware = info.FirmwareVersion - dv.RotationSpeed = info.RotationRate - dv.Capacity = info.UserCapacity.Bytes - dv.FormFactor = info.FormFactor.Name - dv.DeviceProtocol = info.Device.Protocol - dv.DeviceType = info.Device.Type - if len(info.Vendor) > 0 { - dv.Manufacturer = info.Vendor - } - return nil } diff --git a/webapp/backend/pkg/models/testdata/smart-nvme2.json b/webapp/backend/pkg/models/testdata/smart-nvme2.json new file mode 100644 index 0000000..4f47c1a --- /dev/null +++ b/webapp/backend/pkg/models/testdata/smart-nvme2.json @@ -0,0 +1,105 @@ +{ + "json_format_version": [ + 1, + 0 + ], + "smartctl": { + "version": [ + 7, + 0 + ], + "svn_revision": "4883", + "platform_info": "x86_64-linux-4.19.107-Unraid", + "build_info": "(local build)", + "argv": [ + "smartctl", + "-a", + "-j", + "-d", + "nvme", + "/dev/nvme0" + ], + "exit_status": 0 + }, + "device": { + "name": "/dev/nvme0", + "info_name": "/dev/nvme0", + "type": "nvme", + "protocol": "NVMe" + }, + "model_name": "Force MP510", + "serial_number": "yes", + "firmware_version": "ECFM12.3", + "nvme_pci_vendor": { + "id": 6535, + "subsystem_id": 6535 + }, + "nvme_ieee_oui_identifier": 6584743, + "nvme_total_capacity": 480103981056, + "nvme_unallocated_capacity": 0, + "nvme_controller_id": 1, + "nvme_number_of_namespaces": 1, + "nvme_namespaces": [ + { + "id": 1, + "size": { + "blocks": 937703088, + "bytes": 480103981056 + }, + "capacity": { + "blocks": 937703088, + "bytes": 480103981056 + }, + "utilization": { + "blocks": 937703088, + "bytes": 480103981056 + }, + "formatted_lba_size": 512, + "eui64": { + "oui": 6584743, + "ext_id": 171819811633 + } + } + ], + "user_capacity": { + "blocks": 937703088, + "bytes": 480103981056 + }, + "logical_block_size": 512, + "local_time": { + "time_t": 1600619090, + "asctime": "Sun Sep 20 16:24:50 2020 Europe" + }, + "smart_status": { + "passed": true, + "nvme": { + "value": 0 + } + }, + "nvme_smart_health_information_log": { + "critical_warning": 0, + "temperature": 38, + "available_spare": 100, + "available_spare_threshold": 5, + "percentage_used": 1, + "data_units_read": 6932144, + "data_units_written": 16093122, + "host_reads": 29878811, + "host_writes": 17533252, + "controller_busy_time": 305, + "power_cycles": 4, + "power_on_hours": 6487, + "unsafe_shutdowns": 4, + "media_errors": 0, + "num_err_log_entries": 8382, + "warning_temp_time": 0, + "critical_comp_time": 0 + }, + "temperature": { + "current": 38 + }, + "power_cycle_count": 4, + "power_on_time": { + "hours": 6487 + } +} diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index 785c9ed..c1949c5 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -1,24 +1,30 @@ package web_test import ( + "encoding/json" mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock" + dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" "github.com/analogj/scrutiny/webapp/backend/pkg/web" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "io/ioutil" "net/http" "net/http/httptest" "os" + "path" "strings" "testing" ) func TestHealthRoute(t *testing.T) { //setup + parentPath, _ := ioutil.TempDir("", "") + defer os.RemoveAll(parentPath) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().GetString("web.database.location").Return("testdata/scrutiny_test.db") - fakeConfig.EXPECT().GetString("web.src.frontend.path").Return("testdata") + fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")) + fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath) ae := web.AppEngine{ Config: fakeConfig, @@ -38,11 +44,13 @@ func TestHealthRoute(t *testing.T) { func TestRegisterDevicesRoute(t *testing.T) { //setup + parentPath, _ := ioutil.TempDir("", "") + defer os.RemoveAll(parentPath) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().GetString("web.database.location").Return("testdata/scrutiny_test.db") - fakeConfig.EXPECT().GetString("web.src.frontend.path").Return("testdata") + fakeConfig.EXPECT().GetString("web.database.location").Return(path.Join(parentPath, "scrutiny_test.db")) + fakeConfig.EXPECT().GetString("web.src.frontend.path").Return(parentPath) ae := web.AppEngine{ Config: fakeConfig, } @@ -61,11 +69,13 @@ func TestRegisterDevicesRoute(t *testing.T) { func TestUploadDeviceMetricsRoute(t *testing.T) { //setup + parentPath, _ := ioutil.TempDir("", "") + defer os.RemoveAll(parentPath) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db") - fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return("testdata") + fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) + fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) ae := web.AppEngine{ Config: fakeConfig, } @@ -92,11 +102,14 @@ func TestUploadDeviceMetricsRoute(t *testing.T) { func TestPopulateMultiple(t *testing.T) { //setup + parentPath, _ := ioutil.TempDir("", "") + defer os.RemoveAll(parentPath) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db") - fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return("testdata") + //fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db") + fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) + fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) ae := web.AppEngine{ Config: fakeConfig, } @@ -151,11 +164,13 @@ func TestPopulateMultiple(t *testing.T) { func TestSendTestNotificationRoute(t *testing.T) { //setup + parentPath, _ := ioutil.TempDir("", "") + defer os.RemoveAll(parentPath) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() fakeConfig := mock_config.NewMockInterface(mockCtrl) - fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return("testdata/scrutiny_test.db") - fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return("testdata") + fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) + fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://scrutiny.requestcatcher.com/test"}) ae := web.AppEngine{ Config: fakeConfig, @@ -170,3 +185,45 @@ func TestSendTestNotificationRoute(t *testing.T) { //assert require.Equal(t, 200, wr.Code) } + +func TestGetDevicesSummaryRoute_Nvme(t *testing.T) { + //setup + parentPath, _ := ioutil.TempDir("", "") + defer os.RemoveAll(parentPath) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetString("web.database.location").AnyTimes().Return(path.Join(parentPath, "scrutiny_test.db")) + fakeConfig.EXPECT().GetString("web.src.frontend.path").AnyTimes().Return(parentPath) + ae := web.AppEngine{ + Config: fakeConfig, + } + router := ae.Setup() + devicesfile, err := os.Open("testdata/register-devices-req-2.json") + require.NoError(t, err) + + metricsfile, err := os.Open("../models/testdata/smart-nvme2.json") + require.NoError(t, err) + + //test + wr := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/api/devices/register", devicesfile) + router.ServeHTTP(wr, req) + require.Equal(t, 200, wr.Code) + + mr := httptest.NewRecorder() + req, _ = http.NewRequest("POST", "/api/device/a4c8e8ed-11a0-4c97-9bba-306440f1b944/smart", metricsfile) + router.ServeHTTP(mr, req) + require.Equal(t, 200, mr.Code) + + sr := httptest.NewRecorder() + req, _ = http.NewRequest("GET", "/api/summary", nil) + router.ServeHTTP(sr, req) + require.Equal(t, 200, sr.Code) + var device dbModels.DeviceWrapper + json.Unmarshal(sr.Body.Bytes(), &device) + + //assert + require.Equal(t, "a4c8e8ed-11a0-4c97-9bba-306440f1b944", device.Data[0].WWN) + require.Equal(t, "passed", device.Data[0].SmartResults[0].SmartStatus) +} diff --git a/webapp/backend/pkg/web/testdata/register-devices-req-2.json b/webapp/backend/pkg/web/testdata/register-devices-req-2.json new file mode 100644 index 0000000..eb300b9 --- /dev/null +++ b/webapp/backend/pkg/web/testdata/register-devices-req-2.json @@ -0,0 +1,20 @@ +{ + "data": [ + { + "wwn": "a4c8e8ed-11a0-4c97-9bba-306440f1b944", + "device_name": "nvme0", + "manufacturer": "", + "model_name": "Force MP510", + "interface_type": "", + "interface_speed": "", + "serial_number": "a4c8e8ed-11a0-4c97-9bba-306440f1b944", + "firmware": "ECFM12.3", + "rotational_speed": 0, + "capacity": 480103981056, + "form_factor": "", + "smart_support": false, + "device_protocol": "NVMe", + "device_type": "nvme" + } + ] +} diff --git a/webapp/backend/pkg/web/testdata/scrutiny_test.db b/webapp/backend/pkg/web/testdata/scrutiny_test.db deleted file mode 100644 index 15f10fd..0000000 Binary files a/webapp/backend/pkg/web/testdata/scrutiny_test.db and /dev/null differ diff --git a/webapp/frontend/src/app/modules/dashboard/dashboard.component.html b/webapp/frontend/src/app/modules/dashboard/dashboard.component.html index af1bc68..68d42d7 100644 --- a/webapp/frontend/src/app/modules/dashboard/dashboard.component.html +++ b/webapp/frontend/src/app/modules/dashboard/dashboard.component.html @@ -66,7 +66,7 @@
/dev/{{disk.device_name}} - {{disk.model_name}} + class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceTitle(disk)}}
Last Updated on {{disk.smart_results[0]?.date | date:'MMMM dd, yyyy' }} diff --git a/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts b/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts index a6e0b82..b244655 100644 --- a/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts +++ b/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts @@ -164,6 +164,15 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy }); } + deviceTitle(disk){ + var title = [`/dev/${disk.device_name}`] + if (disk.device_type && disk.device_type != 'scsi' && disk.device_type != 'ata'){ + title.push(disk.device_type) + } + title.push(disk.model_name) + return title.join(' - ') + } + /** * Track by function for ngFor loops * diff --git a/webapp/frontend/src/app/modules/detail/detail.component.html b/webapp/frontend/src/app/modules/detail/detail.component.html index 3f64cdb..7706f90 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.html +++ b/webapp/frontend/src/app/modules/detail/detail.component.html @@ -56,6 +56,10 @@
/dev/{{data.data.device_name}}
+
+
{{data.data.device_type}}
+
Device Type
+
{{data.data.manufacturer}}
Model Family