From 40f5cbef6cd0050826b4d2f53f0d6714a0992cc1 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 19 Aug 2020 22:34:27 -0700 Subject: [PATCH] moved hard drive device detection to collector (So we can run multiple in parallel). --- collector/pkg/collector/base.go | 88 ++++++++++++++++++++++++++ collector/pkg/collector/metrics.go | 15 ++--- collector/pkg/models/device.go | 14 ++-- webapp/backend/pkg/models/db/device.go | 2 +- webapp/backend/pkg/web/disk.go | 77 ---------------------- webapp/backend/pkg/web/server.go | 26 +++++--- 6 files changed, 121 insertions(+), 101 deletions(-) delete mode 100644 webapp/backend/pkg/web/disk.go diff --git a/collector/pkg/collector/base.go b/collector/pkg/collector/base.go index e5e0352..0e66839 100644 --- a/collector/pkg/collector/base.go +++ b/collector/pkg/collector/base.go @@ -4,14 +4,87 @@ import ( "bytes" "encoding/json" "errors" + "fmt" + "github.com/analogj/scrutiny/collector/pkg/models" + "github.com/jaypipes/ghw" "io" + "net/http" "os" "os/exec" "path" + "time" ) +var httpClient = &http.Client{Timeout: 10 * time.Second} + type BaseCollector struct{} +func (c *BaseCollector) detectStorageDevices() ([]models.Device, error) { + + block, err := ghw.Block() + if err != nil { + fmt.Printf("Error getting block storage info: %v", err) + return nil, err + } + + approvedDisks := []models.Device{} + for _, disk := range block.Disks { + + // ignore optical drives and floppy disks + if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD { + fmt.Printf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String()) + continue + } + + // ignore removable disks + if disk.IsRemovable { + fmt.Printf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable) + continue + } + + // ignore virtual disks & mobile phone storage devices + if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC { + fmt.Printf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String()) + continue + } + + // ignore NVMe devices (not currently supported) TBA + if disk.StorageController == ghw.STORAGE_CONTROLLER_NVME { + fmt.Printf(" => Ignore: NVMe storage controller - (found %s)\n", disk.StorageController.String()) + continue + } + + // Skip unknown storage controllers, not usually S.M.A.R.T compatible. + if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN { + fmt.Printf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String()) + continue + } + + diskModel := models.Device{ + WWN: disk.WWN, + Manufacturer: disk.Vendor, + ModelName: disk.Model, + InterfaceType: disk.StorageController.String(), + //InterfaceSpeed: string + SerialNumber: disk.SerialNumber, + Capacity: int64(disk.SizeBytes), + //Firmware string + //RotationSpeed int + + DeviceName: disk.Name, + } + if len(diskModel.WWN) == 0 { + //(macOS and some other os's) do not provide a WWN, so we're going to fallback to + //diskname as identifier if WWN is not present + diskModel.WWN = disk.Name + } + + approvedDisks = append(approvedDisks, diskModel) + } + + return approvedDisks, nil +} + func (c *BaseCollector) getJson(url string, target interface{}) error { r, err := httpClient.Get(url) @@ -23,6 +96,21 @@ func (c *BaseCollector) getJson(url string, target interface{}) error { return json.NewDecoder(r.Body).Decode(target) } +func (c *BaseCollector) postJson(url string, body interface{}, target interface{}) error { + requestBody, err := json.Marshal(body) + if err != nil { + return err + } + + r, err := httpClient.Post(url, "application/json", bytes.NewBuffer(requestBody)) + if err != nil { + return err + } + defer r.Body.Close() + + return json.NewDecoder(r.Body).Decode(target) +} + func (c *BaseCollector) execCmd(cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) { cmd := exec.Command(cmdName, cmdArgs...) diff --git a/collector/pkg/collector/metrics.go b/collector/pkg/collector/metrics.go index efa4b53..a901dc7 100644 --- a/collector/pkg/collector/metrics.go +++ b/collector/pkg/collector/metrics.go @@ -6,16 +6,12 @@ import ( "github.com/analogj/scrutiny/collector/pkg/errors" "github.com/analogj/scrutiny/collector/pkg/models" "github.com/sirupsen/logrus" - "net/http" "net/url" "os/exec" "strings" "sync" - "time" ) -var httpClient = &http.Client{Timeout: 10 * time.Second} - type MetricsCollector struct { BaseCollector @@ -44,12 +40,15 @@ func (mc *MetricsCollector) Run() error { } apiEndpoint, _ := url.Parse(mc.apiEndpoint.String()) - apiEndpoint.Path = "/api/devices" + apiEndpoint.Path = "/api/devices/register" - deviceRespWrapper := new(models.DeviceRespWrapper) + deviceRespWrapper := new(models.DeviceWrapper) + detectedStorageDevices, err := mc.detectStorageDevices() - fmt.Println("Getting devices") - err = mc.getJson(apiEndpoint.String(), &deviceRespWrapper) + fmt.Println("Sending detected devices to API, for filtering & validation") + err = mc.postJson(apiEndpoint.String(), models.DeviceWrapper{ + Data: detectedStorageDevices, + }, &deviceRespWrapper) if err != nil { return err } diff --git a/collector/pkg/models/device.go b/collector/pkg/models/device.go index fcac53d..fcac712 100644 --- a/collector/pkg/models/device.go +++ b/collector/pkg/models/device.go @@ -1,21 +1,23 @@ package models type Device struct { - WWN string `json:"wwn" gorm:"primary_key"` + WWN string `json:"wwn"` DeviceName string `json:"device_name"` Manufacturer string `json:"manufacturer"` ModelName string `json:"model_name"` InterfaceType string `json:"interface_type"` InterfaceSpeed string `json:"interface_speed"` - SerialNumber string `json:"serial_name"` - Capacity int64 `json:"capacity"` + SerialNumber string `json:"serial_number"` Firmware string `json:"firmware"` RotationSpeed int `json:"rotational_speed"` + Capacity int64 `json:"capacity"` + FormFactor string `json:"form_factor"` + SmartSupport bool `json:"smart_support"` } -type DeviceRespWrapper struct { - Success bool `json:"success"` - Errors []error `json:"errors"` +type DeviceWrapper struct { + Success bool `json:"success,omitempty"` + Errors []error `json:"errors,omitempty"` Data []Device `json:"data"` } diff --git a/webapp/backend/pkg/models/db/device.go b/webapp/backend/pkg/models/db/device.go index f2c3789..42f3a69 100644 --- a/webapp/backend/pkg/models/db/device.go +++ b/webapp/backend/pkg/models/db/device.go @@ -7,7 +7,7 @@ import ( "time" ) -type DeviceRespWrapper struct { +type DeviceWrapper struct { Success bool `json:"success"` Errors []error `json:"errors"` Data []Device `json:"data"` diff --git a/webapp/backend/pkg/web/disk.go b/webapp/backend/pkg/web/disk.go deleted file mode 100644 index f9e0ab3..0000000 --- a/webapp/backend/pkg/web/disk.go +++ /dev/null @@ -1,77 +0,0 @@ -package web - -import ( - "fmt" - "github.com/analogj/scrutiny/webapp/backend/pkg/models/db" - "github.com/jaypipes/ghw" -) - -func RetrieveStorageDevices() ([]db.Device, error) { - - block, err := ghw.Block() - if err != nil { - fmt.Printf("Error getting block storage info: %v", err) - return nil, err - } - - approvedDisks := []db.Device{} - for _, disk := range block.Disks { - //TODO: always allow if in approved list - fmt.Printf(" %v\n", disk) - - // ignore optical drives and floppy disks - if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD { - fmt.Printf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String()) - continue - } - - // ignore removable disks - if disk.IsRemovable { - fmt.Printf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable) - continue - } - - // ignore virtual disks & mobile phone storage devices - if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC { - fmt.Printf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String()) - continue - } - - // ignore NVMe devices (not currently supported) TBA - if disk.StorageController == ghw.STORAGE_CONTROLLER_NVME { - fmt.Printf(" => Ignore: NVMe storage controller - (found %s)\n", disk.StorageController.String()) - continue - } - - // Skip unknown storage controllers, not usually S.M.A.R.T compatible. - if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN { - fmt.Printf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String()) - continue - } - - //TODO: remove if in excluded list - - diskModel := db.Device{ - WWN: disk.WWN, - Manufacturer: disk.Vendor, - ModelName: disk.Model, - InterfaceType: disk.StorageController.String(), - //InterfaceSpeed: string - SerialNumber: disk.SerialNumber, - Capacity: int64(disk.SizeBytes), - //Firmware string - //RotationSpeed int - - DeviceName: disk.Name, - } - if len(diskModel.WWN) == 0 { - //(macOS and some other os's) do not provide a WWN, so we're going to fallback to - //diskname as identifier if WWN is not present - diskModel.WWN = disk.Name - } - - approvedDisks = append(approvedDisks, diskModel) - } - - return approvedDisks, nil -} diff --git a/webapp/backend/pkg/web/server.go b/webapp/backend/pkg/web/server.go index 0ee6059..95fc319 100644 --- a/webapp/backend/pkg/web/server.go +++ b/webapp/backend/pkg/web/server.go @@ -31,7 +31,7 @@ func (ae *AppEngine) Start() error { }) //TODO: notifications - api.GET("/devices", GetDevicesHandler) + api.POST("/devices/register", RegisterDevices) api.GET("/summary", GetDevicesSummary) api.POST("/device/:wwn/smart", UploadDeviceSmartData) api.POST("/device/:wwn/selftest", UploadDeviceSelfTestData) @@ -55,12 +55,20 @@ func (ae *AppEngine) Start() error { return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port"))) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080") } -// Get all active disks for processing by collectors -func GetDevicesHandler(c *gin.Context) { - storageDevices, err := RetrieveStorageDevices() - +// filter devices that are detected by various collectors. +func RegisterDevices(c *gin.Context) { db := c.MustGet("DB").(*gorm.DB) - for _, dev := range storageDevices { + + var collectorDeviceWrapper dbModels.DeviceWrapper + err := c.BindJSON(&collectorDeviceWrapper) + if err != nil { + log.Error("Cannot parse detected devices") + c.JSON(http.StatusOK, gin.H{"success": false}) + } + + //TODO: filter devices here (remove excludes, force includes) + + for _, dev := range collectorDeviceWrapper.Data { //insert devices into DB if not already there. db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev) } @@ -70,9 +78,9 @@ func GetDevicesHandler(c *gin.Context) { "success": false, }) } else { - c.JSON(http.StatusOK, gin.H{ - "success": true, - "data": storageDevices, + c.JSON(http.StatusOK, dbModels.DeviceWrapper{ + Success: true, + Data: collectorDeviceWrapper.Data, }) } }