diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/device.go b/webapp/backend/pkg/database/migrations/m20201107210306/device.go index cac8b29..11600c3 100644 --- a/webapp/backend/pkg/database/migrations/m20201107210306/device.go +++ b/webapp/backend/pkg/database/migrations/m20201107210306/device.go @@ -29,3 +29,19 @@ type Device struct { DeviceType string `json:"device_type"` //device type is used for querying with -d/t flag, should only be used by collector. SmartResults []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"` } + +const DeviceProtocolAta = "ATA" +const DeviceProtocolScsi = "SCSI" +const DeviceProtocolNvme = "NVMe" + +func (dv *Device) IsAta() bool { + return dv.DeviceProtocol == DeviceProtocolAta +} + +func (dv *Device) IsScsi() bool { + return dv.DeviceProtocol == DeviceProtocolScsi +} + +func (dv *Device) IsNvme() bool { + return dv.DeviceProtocol == DeviceProtocolNvme +} diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go index 263e08d..19d5861 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go @@ -6,8 +6,10 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" influxdb2 "github.com/influxdata/influxdb-client-go/v2" + "github.com/influxdata/influxdb-client-go/v2/api" log "github.com/sirupsen/logrus" "strings" + "time" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -22,13 +24,9 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin } tags, fields := deviceSmartData.Flatten() - p := influxdb2.NewPoint("smart", - tags, - fields, - deviceSmartData.Date) // write point immediately - return deviceSmartData, sr.influxWriteApi.WritePoint(ctx, p) + return deviceSmartData, sr.saveDatapoint(sr.influxWriteApi, "smart", tags, fields, deviceSmartData.Date, ctx) } func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) { @@ -93,6 +91,17 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn // Helper Methods //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, measurement string, tags map[string]string, fields map[string]interface{}, date time.Time, ctx context.Context) error { + sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields)) + p := influxdb2.NewPoint(measurement, + tags, + fields, + date) + + // write point immediately + return influxWriteApi.WritePoint(ctx, p) +} + func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string) string { /* diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index c7cedd2..0c12186 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -2,11 +2,16 @@ package database import ( "context" + "fmt" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" "github.com/go-gormigrate/gormigrate/v2" _ "github.com/jinzhu/gorm/dialects/sqlite" "gorm.io/gorm" + "strconv" + "time" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -66,14 +71,170 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { //get a list of all devices: // get a list of all smart scans in the last 2 weeks: // get a list of associated smart attribute data: - // translate to a collector.SmartInfo object - // call scrutinyRepository.SaveSmartAttributes + // translate to a measurements.Smart{} object + // call CUSTOM INFLUXDB SAVE FUNCTION (taking bucket as parameter) // get a list of all smart scans in the last 9 weeks: // do same as above (select 1 scan per week) // get a list of all smart scans in the last 25 months: // do same as above (select 1 scan per month) // get a list of all smart scans: // do same as above (select 1 scan per year) + + preDevices := []m20201107210306.Device{} //pre-migration device information + if err = tx.Preload("SmartResults", func(db *gorm.DB) *gorm.DB { + return db.Order("smarts.created_at ASC") //OLD: .Limit(devicesCount) + }).Find(&preDevices).Error; err != nil { + sr.logger.Errorln("Could not get device summary from DB", err) + return err + } + + //weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary. + weeklyLookup := map[string]bool{} + monthlyLookup := map[string]bool{} + yearlyLookup := map[string]bool{} + + //calculate bucket oldest dates + today := time.Now() + dailyBucketMax := today.Add(-RETENTION_PERIOD_15_DAYS_IN_SECONDS * time.Second) //15 days + weeklyBucketMax := today.Add(-RETENTION_PERIOD_9_WEEKS_IN_SECONDS * time.Second) //9 weeks + monthlyBucketMax := today.Add(-RETENTION_PERIOD_25_MONTHS_IN_SECONDS * time.Second) //25 weeks + + for _, preDevice := range preDevices { + for _, preSmartResult := range preDevice.SmartResults { //pre-migration smart results + + //we're looping in ASC mode, so from oldest entry to most current. + + //TODO: skip any results that are outside of the range that we care about for each bucket. + + err, postSmartResults := m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(tx, preDevice, preSmartResult) + if err != nil { + return err + } + smartTags, smartFields := postSmartResults.Flatten() + + err, postSmartTemp := m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice, preSmartResult) + if err != nil { + return err + } + tempTags, tempFields := postSmartTemp.Flatten() + tempTags["device_wwn"] = preDevice.WWN + + year, week := postSmartResults.Date.ISOWeek() + month := postSmartResults.Date.Month() + + yearStr := strconv.Itoa(year) + yearMonthStr := fmt.Sprintf("%d-%d", year, month) + yearWeekStr := fmt.Sprintf("%d-%d", year, week) + + //write data to daily bucket if in the last 15 days + if postSmartResults.Date.After(dailyBucketMax) { + sr.logger.Debugf("device (%s) smart data added to bucket: daily", preDevice.WWN) + // write point immediately + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")), + "smart", + smartTags, + smartFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")), + "temp", + tempTags, + tempFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + } + + //write data to the weekly bucket if in the last 9 weeks, and week has not been processed yet + if _, weekExists := weeklyLookup[yearWeekStr]; !weekExists && postSmartResults.Date.After(weeklyBucketMax) { + sr.logger.Debugf("device (%s) smart data added to bucket: weekly", preDevice.WWN) + + //this week/year pair has not been processed + weeklyLookup[yearWeekStr] = true + // write point immediately + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))), + "smart", + smartTags, + smartFields, + postSmartResults.Date, ctx) + + if err != nil { + return err + } + + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))), + "temp", + tempTags, + tempFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + } + + //write data to the monthly bucket if in the last 9 weeks, and week has not been processed yet + if _, monthExists := monthlyLookup[yearMonthStr]; !monthExists && postSmartResults.Date.After(monthlyBucketMax) { + sr.logger.Debugf("device (%s) smart data added to bucket: monthly", preDevice.WWN) + //this month/year pair has not been processed + monthlyLookup[yearMonthStr] = true + // write point immediately + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))), + "smart", + smartTags, + smartFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))), + "temp", + tempTags, + tempFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + } + + if _, yearExists := yearlyLookup[yearStr]; !yearExists && year != today.Year() { + sr.logger.Debugf("device (%s) smart data added to bucket: yearly", preDevice.WWN) + //this year has not been processed + yearlyLookup[yearStr] = true + // write point immediately + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))), + "smart", + smartTags, + smartFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))), + "temp", + tempTags, + tempFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + } + } + } + return nil }, }, @@ -104,3 +265,183 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { sr.logger.Infoln("Database migration completed successfully") return nil } + +// Deprecated +func m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.SmartTemperature) { + //extract temperature data for every datapoint + postSmartTemp := measurements.SmartTemperature{ + Date: preSmartResult.TestDate, + Temp: preSmartResult.Temp, + } + + return nil, postSmartTemp +} + +// Deprecated +func m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(database *gorm.DB, preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.Smart) { + //create a measurements.Smart object (which we will then push to the InfluxDB) + postDeviceSmartData := measurements.Smart{ + Date: preSmartResult.TestDate, + DeviceWWN: preDevice.WWN, + DeviceProtocol: preDevice.DeviceProtocol, + Temp: preSmartResult.Temp, + PowerOnHours: preSmartResult.PowerOnHours, + PowerCycleCount: preSmartResult.PowerCycleCount, + + // this needs to be populated using measurements.Smart.ProcessAtaSmartInfo, ProcessScsiSmartInfo or ProcessNvmeSmartInfo + // because those functions will take into account thresholds (which we didn't consider correctly previously) + Attributes: map[string]measurements.SmartAttribute{}, + } + + result := database.Preload("AtaAttributes").Preload("NvmeAttributes").Preload("ScsiAttributes").Find(&preSmartResult) + if result.Error != nil { + return result.Error, postDeviceSmartData + } + + if preDevice.IsAta() { + preAtaSmartAttributesTable := []collector.AtaSmartAttributesTableItem{} + for _, preAtaAttribute := range preSmartResult.AtaAttributes { + preAtaSmartAttributesTable = append(preAtaSmartAttributesTable, collector.AtaSmartAttributesTableItem{ + ID: preAtaAttribute.AttributeId, + Name: preAtaAttribute.Name, + Value: int64(preAtaAttribute.Value), + Worst: int64(preAtaAttribute.Worst), + Thresh: int64(preAtaAttribute.Threshold), + WhenFailed: preAtaAttribute.WhenFailed, + Flags: struct { + Value int `json:"value"` + String string `json:"string"` + Prefailure bool `json:"prefailure"` + UpdatedOnline bool `json:"updated_online"` + Performance bool `json:"performance"` + ErrorRate bool `json:"error_rate"` + EventCount bool `json:"event_count"` + AutoKeep bool `json:"auto_keep"` + }{ + Value: 0, + String: "", + Prefailure: false, + UpdatedOnline: false, + Performance: false, + ErrorRate: false, + EventCount: false, + AutoKeep: false, + }, + Raw: struct { + Value int64 `json:"value"` + String string `json:"string"` + }{ + Value: preAtaAttribute.RawValue, + String: preAtaAttribute.RawString, + }, + }) + } + + postDeviceSmartData.ProcessAtaSmartInfo(preAtaSmartAttributesTable) + + } else if preDevice.IsNvme() { + //info collector.SmartInfo + postNvmeSmartHealthInformation := collector.NvmeSmartHealthInformationLog{} + + for _, preNvmeAttribute := range preSmartResult.NvmeAttributes { + switch preNvmeAttribute.AttributeId { + case "critical_warning": + postNvmeSmartHealthInformation.CriticalWarning = int64(preNvmeAttribute.Value) + case "temperature": + postNvmeSmartHealthInformation.Temperature = int64(preNvmeAttribute.Value) + case "available_spare": + postNvmeSmartHealthInformation.AvailableSpare = int64(preNvmeAttribute.Value) + case "available_spare_threshold": + postNvmeSmartHealthInformation.AvailableSpareThreshold = int64(preNvmeAttribute.Value) + case "percentage_used": + postNvmeSmartHealthInformation.PercentageUsed = int64(preNvmeAttribute.Value) + case "data_units_read": + postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value) + case "data_units_written": + postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value) + case "host_reads": + postNvmeSmartHealthInformation.HostReads = int64(preNvmeAttribute.Value) + case "host_writes": + postNvmeSmartHealthInformation.HostWrites = int64(preNvmeAttribute.Value) + case "controller_busy_time": + postNvmeSmartHealthInformation.ControllerBusyTime = int64(preNvmeAttribute.Value) + case "power_cycles": + postNvmeSmartHealthInformation.PowerCycles = int64(preNvmeAttribute.Value) + case "power_on_hours": + postNvmeSmartHealthInformation.PowerOnHours = int64(preNvmeAttribute.Value) + case "unsafe_shutdowns": + postNvmeSmartHealthInformation.UnsafeShutdowns = int64(preNvmeAttribute.Value) + case "media_errors": + postNvmeSmartHealthInformation.MediaErrors = int64(preNvmeAttribute.Value) + case "num_err_log_entries": + postNvmeSmartHealthInformation.NumErrLogEntries = int64(preNvmeAttribute.Value) + case "warning_temp_time": + postNvmeSmartHealthInformation.WarningTempTime = int64(preNvmeAttribute.Value) + case "critical_comp_time": + postNvmeSmartHealthInformation.CriticalCompTime = int64(preNvmeAttribute.Value) + } + } + + postDeviceSmartData.ProcessNvmeSmartInfo(postNvmeSmartHealthInformation) + + } else if preDevice.IsScsi() { + //info collector.SmartInfo + var postScsiGrownDefectList int64 + postScsiErrorCounterLog := collector.ScsiErrorCounterLog{ + Read: struct { + ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` + ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` + ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` + TotalErrorsCorrected int64 `json:"total_errors_corrected"` + CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` + GigabytesProcessed string `json:"gigabytes_processed"` + TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` + }{}, + Write: struct { + ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` + ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` + ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` + TotalErrorsCorrected int64 `json:"total_errors_corrected"` + CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` + GigabytesProcessed string `json:"gigabytes_processed"` + TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` + }{}, + } + + for _, preScsiAttribute := range preSmartResult.ScsiAttributes { + switch preScsiAttribute.AttributeId { + case "scsi_grown_defect_list": + postScsiGrownDefectList = int64(preScsiAttribute.Value) + case "read.errors_corrected_by_eccfast": + postScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value) + case "read.errors_corrected_by_eccdelayed": + postScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value) + case "read.errors_corrected_by_rereads_rewrites": + postScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value) + case "read.total_errors_corrected": + postScsiErrorCounterLog.Read.TotalErrorsCorrected = int64(preScsiAttribute.Value) + case "read.correction_algorithm_invocations": + postScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value) + case "read.total_uncorrected_errors": + postScsiErrorCounterLog.Read.TotalUncorrectedErrors = int64(preScsiAttribute.Value) + case "write.errors_corrected_by_eccfast": + postScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value) + case "write.errors_corrected_by_eccdelayed": + postScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value) + case "write.errors_corrected_by_rereads_rewrites": + postScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value) + case "write.total_errors_corrected": + postScsiErrorCounterLog.Write.TotalErrorsCorrected = int64(preScsiAttribute.Value) + case "write.correction_algorithm_invocations": + postScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value) + case "write.total_uncorrected_errors": + postScsiErrorCounterLog.Write.TotalUncorrectedErrors = int64(preScsiAttribute.Value) + } + } + postDeviceSmartData.ProcessScsiSmartInfo(postScsiGrownDefectList, postScsiErrorCounterLog) + } else { + return fmt.Errorf("Unknown device protocol: %s", preDevice.DeviceProtocol), postDeviceSmartData + } + + return nil, postDeviceSmartData +} diff --git a/webapp/backend/pkg/models/collector/smart.go b/webapp/backend/pkg/models/collector/smart.go index 8cb5653..4f7ae3d 100644 --- a/webapp/backend/pkg/models/collector/smart.go +++ b/webapp/backend/pkg/models/collector/smart.go @@ -134,29 +134,8 @@ type SmartInfo struct { Table []int64 `json:"table"` } `json:"ata_sct_temperature_history"` AtaSmartAttributes struct { - Revision int `json:"revision"` - Table []struct { - ID int `json:"id"` - Name string `json:"name"` - Value int64 `json:"value"` - Worst int64 `json:"worst"` - Thresh int64 `json:"thresh"` - WhenFailed string `json:"when_failed"` - Flags struct { - Value int `json:"value"` - String string `json:"string"` - Prefailure bool `json:"prefailure"` - UpdatedOnline bool `json:"updated_online"` - Performance bool `json:"performance"` - ErrorRate bool `json:"error_rate"` - EventCount bool `json:"event_count"` - AutoKeep bool `json:"auto_keep"` - } `json:"flags"` - Raw struct { - Value int64 `json:"value"` - String string `json:"string"` - } `json:"raw"` - } `json:"table"` + Revision int `json:"revision"` + Table []AtaSmartAttributesTableItem `json:"table"` } `json:"ata_smart_attributes"` AtaSmartErrorLog struct { Summary struct { @@ -250,49 +229,77 @@ type SmartInfo struct { } `json:"utilization"` FormattedLbaSize int `json:"formatted_lba_size"` } `json:"nvme_namespaces"` - NvmeSmartHealthInformationLog struct { - CriticalWarning int64 `json:"critical_warning"` - Temperature int64 `json:"temperature"` - AvailableSpare int64 `json:"available_spare"` - AvailableSpareThreshold int64 `json:"available_spare_threshold"` - PercentageUsed int64 `json:"percentage_used"` - DataUnitsRead int64 `json:"data_units_read"` - DataUnitsWritten int64 `json:"data_units_written"` - HostReads int64 `json:"host_reads"` - HostWrites int64 `json:"host_writes"` - ControllerBusyTime int64 `json:"controller_busy_time"` - PowerCycles int64 `json:"power_cycles"` - PowerOnHours int64 `json:"power_on_hours"` - UnsafeShutdowns int64 `json:"unsafe_shutdowns"` - MediaErrors int64 `json:"media_errors"` - NumErrLogEntries int64 `json:"num_err_log_entries"` - WarningTempTime int64 `json:"warning_temp_time"` - CriticalCompTime int64 `json:"critical_comp_time"` - } `json:"nvme_smart_health_information_log"` + NvmeSmartHealthInformationLog NvmeSmartHealthInformationLog `json:"nvme_smart_health_information_log"` // SCSI Protocol Specific Fields - Vendor string `json:"vendor"` - Product string `json:"product"` - ScsiVersion string `json:"scsi_version"` - ScsiGrownDefectList int64 `json:"scsi_grown_defect_list"` - ScsiErrorCounterLog struct { - Read struct { - ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` - ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` - ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` - TotalErrorsCorrected int64 `json:"total_errors_corrected"` - CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` - GigabytesProcessed string `json:"gigabytes_processed"` - TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` - } `json:"read"` - Write struct { - ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` - ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` - ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` - TotalErrorsCorrected int64 `json:"total_errors_corrected"` - CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` - GigabytesProcessed string `json:"gigabytes_processed"` - TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` - } `json:"write"` - } `json:"scsi_error_counter_log"` + Vendor string `json:"vendor"` + Product string `json:"product"` + ScsiVersion string `json:"scsi_version"` + ScsiGrownDefectList int64 `json:"scsi_grown_defect_list"` + ScsiErrorCounterLog ScsiErrorCounterLog `json:"scsi_error_counter_log"` +} + +//Primary Attribute Structs +type AtaSmartAttributesTableItem struct { + ID int `json:"id"` + Name string `json:"name"` + Value int64 `json:"value"` + Worst int64 `json:"worst"` + Thresh int64 `json:"thresh"` + WhenFailed string `json:"when_failed"` + Flags struct { + Value int `json:"value"` + String string `json:"string"` + Prefailure bool `json:"prefailure"` + UpdatedOnline bool `json:"updated_online"` + Performance bool `json:"performance"` + ErrorRate bool `json:"error_rate"` + EventCount bool `json:"event_count"` + AutoKeep bool `json:"auto_keep"` + } `json:"flags"` + Raw struct { + Value int64 `json:"value"` + String string `json:"string"` + } `json:"raw"` +} + +type NvmeSmartHealthInformationLog struct { + CriticalWarning int64 `json:"critical_warning"` + Temperature int64 `json:"temperature"` + AvailableSpare int64 `json:"available_spare"` + AvailableSpareThreshold int64 `json:"available_spare_threshold"` + PercentageUsed int64 `json:"percentage_used"` + DataUnitsRead int64 `json:"data_units_read"` + DataUnitsWritten int64 `json:"data_units_written"` + HostReads int64 `json:"host_reads"` + HostWrites int64 `json:"host_writes"` + ControllerBusyTime int64 `json:"controller_busy_time"` + PowerCycles int64 `json:"power_cycles"` + PowerOnHours int64 `json:"power_on_hours"` + UnsafeShutdowns int64 `json:"unsafe_shutdowns"` + MediaErrors int64 `json:"media_errors"` + NumErrLogEntries int64 `json:"num_err_log_entries"` + WarningTempTime int64 `json:"warning_temp_time"` + CriticalCompTime int64 `json:"critical_comp_time"` +} + +type ScsiErrorCounterLog struct { + Read struct { + ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` + ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` + ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` + TotalErrorsCorrected int64 `json:"total_errors_corrected"` + CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` + GigabytesProcessed string `json:"gigabytes_processed"` + TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` + } `json:"read"` + Write struct { + ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` + ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` + ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` + TotalErrorsCorrected int64 `json:"total_errors_corrected"` + CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` + GigabytesProcessed string `json:"gigabytes_processed"` + TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` + } `json:"write"` } diff --git a/webapp/backend/pkg/models/measurements/smart.go b/webapp/backend/pkg/models/measurements/smart.go index 37810f4..cc25593 100644 --- a/webapp/backend/pkg/models/measurements/smart.go +++ b/webapp/backend/pkg/models/measurements/smart.go @@ -125,20 +125,20 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er // process ATA/NVME/SCSI protocol data sm.Attributes = map[string]SmartAttribute{} if sm.DeviceProtocol == pkg.DeviceProtocolAta { - sm.ProcessAtaSmartInfo(info) + sm.ProcessAtaSmartInfo(info.AtaSmartAttributes.Table) } else if sm.DeviceProtocol == pkg.DeviceProtocolNvme { - sm.ProcessNvmeSmartInfo(info) + sm.ProcessNvmeSmartInfo(info.NvmeSmartHealthInformationLog) } else if sm.DeviceProtocol == pkg.DeviceProtocolScsi { - sm.ProcessScsiSmartInfo(info) + sm.ProcessScsiSmartInfo(info.ScsiGrownDefectList, info.ScsiErrorCounterLog) } return nil } //generate SmartAtaAttribute entries from Scrutiny Collector Smart data. -func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) { +func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTableItem) { sm.Status = pkg.DeviceStatusPassed - for _, collectorAttr := range info.AtaSmartAttributes.Table { + for _, collectorAttr := range tableItems { attrModel := SmartAtaAttribute{ AttributeId: collectorAttr.ID, Value: collectorAttr.Value, @@ -164,25 +164,25 @@ func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) { } //generate SmartNvmeAttribute entries from Scrutiny Collector Smart data. -func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) { +func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.NvmeSmartHealthInformationLog) { sm.Attributes = map[string]SmartAttribute{ - "critical_warning": (&SmartNvmeAttribute{AttributeId: "critical_warning", Value: info.NvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0}).PopulateAttributeStatus(), - "temperature": (&SmartNvmeAttribute{AttributeId: "temperature", Value: info.NvmeSmartHealthInformationLog.Temperature, Threshold: -1}).PopulateAttributeStatus(), - "available_spare": (&SmartNvmeAttribute{AttributeId: "available_spare", Value: info.NvmeSmartHealthInformationLog.AvailableSpare, Threshold: info.NvmeSmartHealthInformationLog.AvailableSpareThreshold}).PopulateAttributeStatus(), - "percentage_used": (&SmartNvmeAttribute{AttributeId: "percentage_used", Value: info.NvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100}).PopulateAttributeStatus(), - "data_units_read": (&SmartNvmeAttribute{AttributeId: "data_units_read", Value: info.NvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1}).PopulateAttributeStatus(), - "data_units_written": (&SmartNvmeAttribute{AttributeId: "data_units_written", Value: info.NvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1}).PopulateAttributeStatus(), - "host_reads": (&SmartNvmeAttribute{AttributeId: "host_reads", Value: info.NvmeSmartHealthInformationLog.HostReads, Threshold: -1}).PopulateAttributeStatus(), - "host_writes": (&SmartNvmeAttribute{AttributeId: "host_writes", Value: info.NvmeSmartHealthInformationLog.HostWrites, Threshold: -1}).PopulateAttributeStatus(), - "controller_busy_time": (&SmartNvmeAttribute{AttributeId: "controller_busy_time", Value: info.NvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1}).PopulateAttributeStatus(), - "power_cycles": (&SmartNvmeAttribute{AttributeId: "power_cycles", Value: info.NvmeSmartHealthInformationLog.PowerCycles, Threshold: -1}).PopulateAttributeStatus(), - "power_on_hours": (&SmartNvmeAttribute{AttributeId: "power_on_hours", Value: info.NvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1}).PopulateAttributeStatus(), - "unsafe_shutdowns": (&SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Value: info.NvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1}).PopulateAttributeStatus(), - "media_errors": (&SmartNvmeAttribute{AttributeId: "media_errors", Value: info.NvmeSmartHealthInformationLog.MediaErrors, Threshold: 0}).PopulateAttributeStatus(), - "num_err_log_entries": (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: info.NvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0}).PopulateAttributeStatus(), - "warning_temp_time": (&SmartNvmeAttribute{AttributeId: "warning_temp_time", Value: info.NvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1}).PopulateAttributeStatus(), - "critical_comp_time": (&SmartNvmeAttribute{AttributeId: "critical_comp_time", Value: info.NvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1}).PopulateAttributeStatus(), + "critical_warning": (&SmartNvmeAttribute{AttributeId: "critical_warning", Value: nvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0}).PopulateAttributeStatus(), + "temperature": (&SmartNvmeAttribute{AttributeId: "temperature", Value: nvmeSmartHealthInformationLog.Temperature, Threshold: -1}).PopulateAttributeStatus(), + "available_spare": (&SmartNvmeAttribute{AttributeId: "available_spare", Value: nvmeSmartHealthInformationLog.AvailableSpare, Threshold: nvmeSmartHealthInformationLog.AvailableSpareThreshold}).PopulateAttributeStatus(), + "percentage_used": (&SmartNvmeAttribute{AttributeId: "percentage_used", Value: nvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100}).PopulateAttributeStatus(), + "data_units_read": (&SmartNvmeAttribute{AttributeId: "data_units_read", Value: nvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1}).PopulateAttributeStatus(), + "data_units_written": (&SmartNvmeAttribute{AttributeId: "data_units_written", Value: nvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1}).PopulateAttributeStatus(), + "host_reads": (&SmartNvmeAttribute{AttributeId: "host_reads", Value: nvmeSmartHealthInformationLog.HostReads, Threshold: -1}).PopulateAttributeStatus(), + "host_writes": (&SmartNvmeAttribute{AttributeId: "host_writes", Value: nvmeSmartHealthInformationLog.HostWrites, Threshold: -1}).PopulateAttributeStatus(), + "controller_busy_time": (&SmartNvmeAttribute{AttributeId: "controller_busy_time", Value: nvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1}).PopulateAttributeStatus(), + "power_cycles": (&SmartNvmeAttribute{AttributeId: "power_cycles", Value: nvmeSmartHealthInformationLog.PowerCycles, Threshold: -1}).PopulateAttributeStatus(), + "power_on_hours": (&SmartNvmeAttribute{AttributeId: "power_on_hours", Value: nvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1}).PopulateAttributeStatus(), + "unsafe_shutdowns": (&SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Value: nvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1}).PopulateAttributeStatus(), + "media_errors": (&SmartNvmeAttribute{AttributeId: "media_errors", Value: nvmeSmartHealthInformationLog.MediaErrors, Threshold: 0}).PopulateAttributeStatus(), + "num_err_log_entries": (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: nvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0}).PopulateAttributeStatus(), + "warning_temp_time": (&SmartNvmeAttribute{AttributeId: "warning_temp_time", Value: nvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1}).PopulateAttributeStatus(), + "critical_comp_time": (&SmartNvmeAttribute{AttributeId: "critical_comp_time", Value: nvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1}).PopulateAttributeStatus(), } //find analyzed attribute status @@ -194,21 +194,21 @@ func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) { } //generate SmartScsiAttribute entries from Scrutiny Collector Smart data. -func (sm *Smart) ProcessScsiSmartInfo(info collector.SmartInfo) { +func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog collector.ScsiErrorCounterLog) { sm.Attributes = map[string]SmartAttribute{ - "scsi_grown_defect_list": (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: info.ScsiGrownDefectList, Threshold: 0}).PopulateAttributeStatus(), - "read_errors_corrected_by_eccfast": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(), - "read_errors_corrected_by_eccdelayed": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(), - "read_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_rereads_rewrites", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(), - "read_total_errors_corrected": (&SmartScsiAttribute{AttributeId: "read_total_errors_corrected", Value: info.ScsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(), - "read_correction_algorithm_invocations": (&SmartScsiAttribute{AttributeId: "read_correction_algorithm_invocations", Value: info.ScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(), - "read_total_uncorrected_errors": (&SmartScsiAttribute{AttributeId: "read_total_uncorrected_errors", Value: info.ScsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(), - "write_errors_corrected_by_eccfast": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccfast", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(), - "write_errors_corrected_by_eccdelayed": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccdelayed", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(), - "write_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_rereads_rewrites", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(), - "write_total_errors_corrected": (&SmartScsiAttribute{AttributeId: "write_total_errors_corrected", Value: info.ScsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(), - "write_correction_algorithm_invocations": (&SmartScsiAttribute{AttributeId: "write_correction_algorithm_invocations", Value: info.ScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(), - "write_total_uncorrected_errors": (&SmartScsiAttribute{AttributeId: "write_total_uncorrected_errors", Value: info.ScsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(), + "scsi_grown_defect_list": (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: defectGrownList, Threshold: 0}).PopulateAttributeStatus(), + "read_errors_corrected_by_eccfast": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(), + "read_errors_corrected_by_eccdelayed": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(), + "read_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(), + "read_total_errors_corrected": (&SmartScsiAttribute{AttributeId: "read_total_errors_corrected", Value: scsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(), + "read_correction_algorithm_invocations": (&SmartScsiAttribute{AttributeId: "read_correction_algorithm_invocations", Value: scsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(), + "read_total_uncorrected_errors": (&SmartScsiAttribute{AttributeId: "read_total_uncorrected_errors", Value: scsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(), + "write_errors_corrected_by_eccfast": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(), + "write_errors_corrected_by_eccdelayed": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(), + "write_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(), + "write_total_errors_corrected": (&SmartScsiAttribute{AttributeId: "write_total_errors_corrected", Value: scsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(), + "write_correction_algorithm_invocations": (&SmartScsiAttribute{AttributeId: "write_correction_algorithm_invocations", Value: scsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(), + "write_total_uncorrected_errors": (&SmartScsiAttribute{AttributeId: "write_total_uncorrected_errors", Value: scsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(), } //find analyzed attribute status