open-source-mirrors
/
scrutiny
mirror of https://github.com/AnalogJ/scrutiny
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11 from AnalogJHQ/refactor_collecttor

Browse Source 
moved hard drive device detection to collector (So we can run multipl…
pull/1/head
Jason Kulatunga 4 years ago committed by GitHub
parent c6a0033786 40f5cbef6c
commit fea5ad1ee4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 121 additions and 101 deletions
Show Stats Download Patch File Download Diff File

88
collector/pkg/collector/base.go
Unescape View File

@ -4,14 +4,87 @@ import (
"bytes"
"encoding/json"
"errors"
"fmt"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/jaypipes/ghw"
"io"
"net/http"
"os"
"os/exec"
"path"
"time"
)
var httpClient = &http.Client{Timeout: 10 * time.Second}
type BaseCollector struct{}
func (c *BaseCollector) detectStorageDevices() ([]models.Device, error) {
block, err := ghw.Block()
if err != nil {
fmt.Printf("Error getting block storage info: %v", err)
return nil, err
}
approvedDisks := []models.Device{}
for _, disk := range block.Disks {
// ignore optical drives and floppy disks
if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
fmt.Printf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
continue
}
// ignore removable disks
if disk.IsRemovable {
fmt.Printf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
continue
}
// ignore virtual disks & mobile phone storage devices
if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
fmt.Printf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// ignore NVMe devices (not currently supported) TBA
if disk.StorageController == ghw.STORAGE_CONTROLLER_NVME {
fmt.Printf(" => Ignore: NVMe storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
fmt.Printf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
continue
}
diskModel := models.Device{
WWN: disk.WWN,
Manufacturer: disk.Vendor,
ModelName: disk.Model,
InterfaceType: disk.StorageController.String(),
//InterfaceSpeed: string
SerialNumber: disk.SerialNumber,
Capacity: int64(disk.SizeBytes),
//Firmware string
//RotationSpeed int
DeviceName: disk.Name,
}
if len(diskModel.WWN) == 0 {
//(macOS and some other os's) do not provide a WWN, so we're going to fallback to
//diskname as identifier if WWN is not present
diskModel.WWN = disk.Name
}
approvedDisks = append(approvedDisks, diskModel)
}
return approvedDisks, nil
}
func (c *BaseCollector) getJson(url string, target interface{}) error {
r, err := httpClient.Get(url)
@ -23,6 +96,21 @@ func (c *BaseCollector) getJson(url string, target interface{}) error {
return json.NewDecoder(r.Body).Decode(target)
}
func (c *BaseCollector) postJson(url string, body interface{}, target interface{}) error {
requestBody, err := json.Marshal(body)
if err != nil {
return err
}
r, err := httpClient.Post(url, "application/json", bytes.NewBuffer(requestBody))
if err != nil {
return err
}
defer r.Body.Close()
return json.NewDecoder(r.Body).Decode(target)
}
func (c *BaseCollector) execCmd(cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error) {
cmd := exec.Command(cmdName, cmdArgs...)

15
collector/pkg/collector/metrics.go
Unescape View File

@ -6,16 +6,12 @@ import (
"github.com/analogj/scrutiny/collector/pkg/errors"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/sirupsen/logrus"
"net/http"
"net/url"
"os/exec"
"strings"
"sync"
"time"
)
var httpClient = &http.Client{Timeout: 10 * time.Second}
type MetricsCollector struct {
BaseCollector
@ -44,12 +40,15 @@ func (mc *MetricsCollector) Run() error {
}
apiEndpoint, _ := url.Parse(mc.apiEndpoint.String())
apiEndpoint.Path = "/api/devices"
apiEndpoint.Path = "/api/devices/register"
deviceRespWrapper := new(models.DeviceRespWrapper)
deviceRespWrapper := new(models.DeviceWrapper)
detectedStorageDevices, err := mc.detectStorageDevices()
fmt.Println("Getting devices")
err = mc.getJson(apiEndpoint.String(), &deviceRespWrapper)
fmt.Println("Sending detected devices to API, for filtering & validation")
err = mc.postJson(apiEndpoint.String(), models.DeviceWrapper{
Data: detectedStorageDevices,
}, &deviceRespWrapper)
if err != nil {
return err
}

14
collector/pkg/models/device.go
Unescape View File

@ -1,21 +1,23 @@
package models
type Device struct {
WWN string `json:"wwn" gorm:"primary_key"`
WWN string `json:"wwn"`
DeviceName string `json:"device_name"`
Manufacturer string `json:"manufacturer"`
ModelName string `json:"model_name"`
InterfaceType string `json:"interface_type"`
InterfaceSpeed string `json:"interface_speed"`
SerialNumber string `json:"serial_name"`
Capacity int64 `json:"capacity"`
SerialNumber string `json:"serial_number"`
Firmware string `json:"firmware"`
RotationSpeed int `json:"rotational_speed"`
Capacity int64 `json:"capacity"`
FormFactor string `json:"form_factor"`
SmartSupport bool `json:"smart_support"`
}
type DeviceRespWrapper struct {
Success bool `json:"success"`
Errors []error `json:"errors"`
type DeviceWrapper struct {
Success bool `json:"success,omitempty"`
Errors []error `json:"errors,omitempty"`
Data []Device `json:"data"`
}

2
webapp/backend/pkg/models/db/device.go
Unescape View File

@ -7,7 +7,7 @@ import (
"time"
)
type DeviceRespWrapper struct {
type DeviceWrapper struct {
Success bool `json:"success"`
Errors []error `json:"errors"`
Data []Device `json:"data"`

77
webapp/backend/pkg/web/disk.go
Unescape View File

@ -1,77 +0,0 @@
package web
import (
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/jaypipes/ghw"
)
func RetrieveStorageDevices() ([]db.Device, error) {
block, err := ghw.Block()
if err != nil {
fmt.Printf("Error getting block storage info: %v", err)
return nil, err
}
approvedDisks := []db.Device{}
for _, disk := range block.Disks {
//TODO: always allow if in approved list
fmt.Printf(" %v\n", disk)
// ignore optical drives and floppy disks
if disk.DriveType == ghw.DRIVE_TYPE_FDD || disk.DriveType == ghw.DRIVE_TYPE_ODD {
fmt.Printf(" => Ignore: Optical or floppy disk - (found %s)\n", disk.DriveType.String())
continue
}
// ignore removable disks
if disk.IsRemovable {
fmt.Printf(" => Ignore: Removable disk (%v)\n", disk.IsRemovable)
continue
}
// ignore virtual disks & mobile phone storage devices
if disk.StorageController == ghw.STORAGE_CONTROLLER_VIRTIO || disk.StorageController == ghw.STORAGE_CONTROLLER_MMC {
fmt.Printf(" => Ignore: Virtual/multi-media storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// ignore NVMe devices (not currently supported) TBA
if disk.StorageController == ghw.STORAGE_CONTROLLER_NVME {
fmt.Printf(" => Ignore: NVMe storage controller - (found %s)\n", disk.StorageController.String())
continue
}
// Skip unknown storage controllers, not usually S.M.A.R.T compatible.
if disk.StorageController == ghw.STORAGE_CONTROLLER_UNKNOWN {
fmt.Printf(" => Ignore: Unknown storage controller - (found %s)\n", disk.StorageController.String())
continue
}
//TODO: remove if in excluded list
diskModel := db.Device{
WWN: disk.WWN,
Manufacturer: disk.Vendor,
ModelName: disk.Model,
InterfaceType: disk.StorageController.String(),
//InterfaceSpeed: string
SerialNumber: disk.SerialNumber,
Capacity: int64(disk.SizeBytes),
//Firmware string
//RotationSpeed int
DeviceName: disk.Name,
}
if len(diskModel.WWN) == 0 {
//(macOS and some other os's) do not provide a WWN, so we're going to fallback to
//diskname as identifier if WWN is not present
diskModel.WWN = disk.Name
}
approvedDisks = append(approvedDisks, diskModel)
}
return approvedDisks, nil
}

26
webapp/backend/pkg/web/server.go
Unescape View File

@ -31,7 +31,7 @@ func (ae *AppEngine) Start() error {
})
//TODO: notifications
api.GET("/devices", GetDevicesHandler)
api.POST("/devices/register", RegisterDevices)
api.GET("/summary", GetDevicesSummary)
api.POST("/device/:wwn/smart", UploadDeviceSmartData)
api.POST("/device/:wwn/selftest", UploadDeviceSelfTestData)
@ -55,12 +55,20 @@ func (ae *AppEngine) Start() error {
return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port"))) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080")
}
// Get all active disks for processing by collectors
func GetDevicesHandler(c *gin.Context) {
storageDevices, err := RetrieveStorageDevices()
// filter devices that are detected by various collectors.
func RegisterDevices(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
for _, dev := range storageDevices {
var collectorDeviceWrapper dbModels.DeviceWrapper
err := c.BindJSON(&collectorDeviceWrapper)
if err != nil {
log.Error("Cannot parse detected devices")
c.JSON(http.StatusOK, gin.H{"success": false})
}
//TODO: filter devices here (remove excludes, force includes)
for _, dev := range collectorDeviceWrapper.Data {
//insert devices into DB if not already there.
db.Where(dbModels.Device{WWN: dev.WWN}).FirstOrCreate(&dev)
}
@ -70,9 +78,9 @@ func GetDevicesHandler(c *gin.Context) {
"success": false,
})
} else {
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": storageDevices,
c.JSON(http.StatusOK, dbModels.DeviceWrapper{
Success: true,
Data: collectorDeviceWrapper.Data,
})
}
}

Write Preview
Loading…
Cancel
Save