diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c184ef2..da29a48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,17 +76,19 @@ If you'd like to populate the database with some test data, you can run the fol > NOTE: you may need to update the `local_time` key within the JSON file, any timestamps older than ~3 weeks will be automatically ignored > (since the downsampling & retention policy takes effect at 2 weeks) +> This is done automatically by the `webapp/backend/pkg/models/testdata/helper.go` script ``` -docker run -p 8086:8086 --rm influxdb:2.0 +docker run -p 8086:8086 --rm influxdb:2.2 docker run --rm -p 8086:8086 \ + -e DOCKER_INFLUXDB_INIT_MODE=setup \ -e DOCKER_INFLUXDB_INIT_USERNAME=admin \ -e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \ -e DOCKER_INFLUXDB_INIT_ORG=scrutiny \ -e DOCKER_INFLUXDB_INIT_BUCKET=metrics \ - influxdb:2.0 + influxdb:2.2 # curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/web/testdata/register-devices-req.json localhost:8080/api/devices/register diff --git a/webapp/backend/pkg/database/scrutiny_repository.go b/webapp/backend/pkg/database/scrutiny_repository.go index 1a99257..cd50018 100644 --- a/webapp/backend/pkg/database/scrutiny_repository.go +++ b/webapp/backend/pkg/database/scrutiny_repository.go @@ -257,6 +257,37 @@ func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) err return nil } +/* + + +sourceBucket = "metrics" +rangeStart = -2w +rangeEnd = -1w +aggWindow = 1w +destBucket = "metrics_weekly" +destOrg = "scrutiny" + +smart_data = from(bucket: sourceBucket) +|> range(start: rangeStart, stop: rangeEnd) +|> filter(fn: (r) => r["_measurement"] == "smart" ) +|> filter(fn: (r) => r["_field"] !~ /(_measurement|device_protocol|device_wwn|attribute_id|raw_string|status_reason|when_failed)/) +|> yield(name: "last") + +smart_data +|> aggregateWindow(fn: mean, every: aggWindow) +|> to(bucket: destBucket, org: destOrg) + +temp_data = from(bucket: sourceBucket) +|> range(start: rangeStart, stop: rangeEnd) +|> filter(fn: (r) => r["_measurement"] == "temp") +|> toInt() +|> yield(name: "mean") + +temp_data +|> aggregateWindow(fn: mean, every: aggWindow) +|> to(bucket: destBucket, org: destOrg) + +*/ func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string { var sourceBucket string // the source of the data var destBucket string // the destination for the aggregated data @@ -284,7 +315,7 @@ func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string { aggWindow = "1y" } - return fmt.Sprintf(` + return fmt.Sprintf(`import "types" sourceBucket = "%s" rangeStart = %s rangeEnd = %s @@ -295,18 +326,24 @@ func (sr *scrutinyRepository) DownsampleScript(aggregationType string) string { smart_data = from(bucket: sourceBucket) |> range(start: rangeStart, stop: rangeEnd) |> filter(fn: (r) => r["_measurement"] == "smart" ) - |> filter(fn: (r) => r["_field"] !~ /(_measurement|device_protocol|device_wwn|attribute_id|raw_string|status_reason|when_failed)/) - |> yield(name: "last") + |> group(columns: ["device_wwn", "_field"]) - smart_data - |> aggregateWindow(fn: mean, every: aggWindow) + non_numeric_smart_data = smart_data + |> filter(fn: (r) => types.isType(v: r._value, type: "string") or types.isType(v: r._value, type: "bool")) + |> aggregateWindow(every: aggWindow, fn: last, createEmpty: false) + + numeric_smart_data = smart_data + |> filter(fn: (r) => types.isType(v: r._value, type: "int") or types.isType(v: r._value, type: "float")) + |> aggregateWindow(every: aggWindow, fn: mean, createEmpty: false) + + union(tables: [non_numeric_smart_data, numeric_smart_data]) |> to(bucket: destBucket, org: destOrg) temp_data = from(bucket: sourceBucket) |> range(start: rangeStart, stop: rangeEnd) |> filter(fn: (r) => r["_measurement"] == "temp") + |> group(columns: ["device_wwn"]) |> toInt() - |> yield(name: "mean") temp_data |> aggregateWindow(fn: mean, every: aggWindow) @@ -726,7 +763,9 @@ func (sr *scrutinyRepository) aggregateTempQuery(durationKey string) string { */ - partialQueryStr := []string{`import "influxdata/influxdb/schema"`} + partialQueryStr := []string{ + `import "influxdata/influxdb/schema"`, + } nestedDurationKeys := sr.lookupNestedDurationKeys(durationKey) diff --git a/webapp/backend/pkg/models/testdata/helper.go b/webapp/backend/pkg/models/testdata/helper.go index 559829c..771557c 100644 --- a/webapp/backend/pkg/models/testdata/helper.go +++ b/webapp/backend/pkg/models/testdata/helper.go @@ -32,7 +32,7 @@ func main() { log.Fatalf("ERROR %v", err) } defer file.Close() - _, err = SendPostRequest("http://localhost:8080/api/devices/register", file) + _, err = SendPostRequest("http://localhost:9090/api/devices/register", file) if err != nil { log.Fatalf("ERROR %v", err) } @@ -46,7 +46,7 @@ func main() { log.Fatalf("ERROR %v", err) } - _, err = SendPostRequest(fmt.Sprintf("http://localhost:8080/api/device/%s/smart", diskId), smartDataReader) + _, err = SendPostRequest(fmt.Sprintf("http://localhost:9090/api/device/%s/smart", diskId), smartDataReader) if err != nil { log.Fatalf("ERROR %v", err) }