From 9d85920f4906e2511fd5ac02f2eaed0d372779a7 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Tue, 3 May 2022 11:50:22 -0700 Subject: [PATCH 1/8] started working on migration code. --- go.mod | 2 + go.sum | 104 ++++++++++++++++++ .../migrations/m20201107210306/device.go | 30 +++++ .../migrations/m20201107210306/smart.go | 25 +++++ .../m20201107210306/smart_ata_attribute.go | 25 +++++ .../m20201107210306/smart_nvme_attribute.go | 21 ++++ .../m20201107210306/smart_scsci_attribute.go | 21 ++++ .../pkg/database/scrutiny_repository.go | 15 ++- .../scrutiny_repository_migrations.go | 96 ++++++++++++++++ 9 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 webapp/backend/pkg/database/migrations/m20201107210306/device.go create mode 100644 webapp/backend/pkg/database/migrations/m20201107210306/smart.go create mode 100644 webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go create mode 100644 webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go create mode 100644 webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go create mode 100644 webapp/backend/pkg/database/scrutiny_repository_migrations.go diff --git a/go.mod b/go.mod index 1f4714d..fce6bc7 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,13 @@ require ( github.com/containrrr/shoutrrr v0.4.4 github.com/fatih/color v1.10.0 github.com/gin-gonic/gin v1.6.3 + github.com/go-gormigrate/gormigrate/v2 v2.0.0 github.com/golang/mock v1.4.3 github.com/google/uuid v1.2.0 // indirect github.com/hashicorp/serf v0.8.2 // indirect github.com/influxdata/influxdb-client-go/v2 v2.2.3 github.com/jaypipes/ghw v0.6.1 + github.com/jinzhu/gorm v1.9.16 github.com/klauspost/compress v1.12.1 // indirect github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect github.com/mattn/go-sqlite3 v1.14.4 // indirect diff --git a/go.sum b/go.sum index 8ea4b43..f30e6c4 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/agnivade/wasmbrowsertest v0.3.1/go.mod h1:zQt6ZTdl338xxRaMW395qccVE2eQm0SjC/SDz0mPWQI= @@ -22,6 +23,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14 h1:wsrSjiqQtseStRIoLLxS4C5IEtXkazZVEPDHq8jW7r8= github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14/go.mod h1:lJQVqFKMV5/oDGYR2bra2OljcF3CvolAoyDRyOA4k4E= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= @@ -39,6 +41,7 @@ github.com/chromedp/cdproto v0.0.0-20190926234355-1b4886c6fad6/go.mod h1:0YChpVz github.com/chromedp/chromedp v0.3.1-0.20190619195644-fd957a4d2901/go.mod h1:mJdvfrVn594N9tfiPecUidF6W5jPRKHymqHfzbobPsM= github.com/chromedp/chromedp v0.4.0/go.mod h1:DC3QUn4mJ24dwjcaGQLoZrhm4X/uPHZ6spDbS2uFhm4= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a h1:6ZMiughZYF6fJjFIf2X3D7AfImJeXnTMJ9qC2v75WPw= github.com/containrrr/shoutrrr v0.0.0-20200828202222-1da53231b05a/go.mod h1:z3pUtEhu5zOpu+Q8wZWiEq+ZLL9hM0HiFNhttaI67Ks= github.com/containrrr/shoutrrr v0.4.4 h1:vHZ4E/76pKVY+Jyn/qhBz3X540Bn8NI5ppPHK4PyILY= @@ -49,20 +52,25 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deepmap/oapi-codegen v1.3.13 h1:9HKGCsdJqE4dnrQ8VerFS0/1ZOJPmAhN+g8xgp8y3K4= github.com/deepmap/oapi-codegen v1.3.13/go.mod h1:WAmG5dWY8/PYHt4vKxlt90NsbHMAOCiteYKZMiIRfOo= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= @@ -82,6 +90,8 @@ github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gormigrate/gormigrate/v2 v2.0.0 h1:e2A3Uznk4viUC4UuemuVgsNnvYZyOA8B3awlYk3UioU= +github.com/go-gormigrate/gormigrate/v2 v2.0.0/go.mod h1:YuVJ+D/dNt4HWrThTBnjgZuRbt7AuwINeg4q52ZE3Jw= github.com/go-interpreter/wagon v0.5.1-0.20190713202023-55a163980b6c/go.mod h1:5+b/MBYkclRZngKF5s6qrgWxSLgE9F5dFdO1hAueZLc= github.com/go-interpreter/wagon v0.6.0/go.mod h1:5+b/MBYkclRZngKF5s6qrgWxSLgE9F5dFdO1hAueZLc= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -97,13 +107,16 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -193,6 +206,46 @@ github.com/influxdata/influxdb-client-go/v2 v2.2.3 h1:082jdJ5t1CFeo0rpGQvKAK1mON github.com/influxdata/influxdb-client-go/v2 v2.2.3/go.mod h1:fa/d1lAdUHxuc1jedx30ZfNG573oQTQmUni3N6pcW+0= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jaypipes/ghw v0.6.1 h1:Ewt3mdpiyhWotGyzg1ursV/6SnToGcG4215X6rR2af8= @@ -200,10 +253,14 @@ github.com/jaypipes/ghw v0.6.1/go.mod h1:QOXppNRCLGYR1H+hu09FxZPqjNt09bqUZUnOL3R github.com/jaypipes/pcidb v0.5.0 h1:4W5gZ+G7QxydevI8/MmmKdnIPJpURqJ2JNXTzfLxF5c= github.com/jaypipes/pcidb v0.5.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= +github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= @@ -228,6 +285,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 h1:3tLzEnUizyN9YLWFTT9loC30lSBvh2y70LTDcZOTs1s= @@ -236,6 +294,11 @@ github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvf github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -246,18 +309,23 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA= github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI= @@ -327,16 +395,23 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -371,6 +446,7 @@ github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -393,22 +469,33 @@ github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.coder.com/go-tools v0.0.0-20190317003359-0c6a35b74a16/go.mod h1:iKV5yK9t+J5nG9O3uF6KYdPEz3dyfMyB15MN1rbQ8Qw= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180426230345-b49d69b5da94/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= @@ -432,6 +519,7 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -449,7 +537,9 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -479,6 +569,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190306220234-b354f8bf4d9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -488,6 +579,7 @@ golang.org/x/sys v0.0.0-20190618155005-516e3c20635f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -527,18 +619,23 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -584,6 +681,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ= @@ -598,8 +696,14 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gorm.io/driver/mysql v1.0.1/go.mod h1:KtqSthtg55lFp3S5kUXqlGaelnWpKitn4k1xZTnoiPw= +gorm.io/driver/postgres v1.0.0/go.mod h1:wtMFcOzmuA5QigNsgEIb7O5lhvH1tHAF1RbWmLWV4to= +gorm.io/driver/sqlite v1.1.1/go.mod h1:hm2olEcl8Tmsc6eZyxYSeznnsDaMqamBvEXLNtBg4cI= gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc= gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c= +gorm.io/driver/sqlserver v1.0.2/go.mod h1:gb0Y9QePGgqjzrVyTQUZeh9zkd5v0iz71cM1B4ZycEY= +gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.20.2 h1:bZzSEnq7NDGsrd+n3evOOedDrY5oLM5QPlCjZJUK2ro= gorm.io/gorm v1.20.2/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/device.go b/webapp/backend/pkg/database/migrations/m20201107210306/device.go new file mode 100644 index 0000000..250122d --- /dev/null +++ b/webapp/backend/pkg/database/migrations/m20201107210306/device.go @@ -0,0 +1,30 @@ +package m20201107210306 + +import ( + "time" +) + +type Device struct { + //GORM attributes, see: http://gorm.io/docs/conventions.html + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time + + WWN string `json:"wwn" gorm:"primary_key"` + HostId string `json:"host_id"` + + DeviceName string `json:"device_name"` + Manufacturer string `json:"manufacturer"` + ModelName string `json:"model_name"` + InterfaceType string `json:"interface_type"` + InterfaceSpeed string `json:"interface_speed"` + SerialNumber string `json:"serial_number"` + Firmware string `json:"firmware"` + RotationSpeed int `json:"rotational_speed"` + Capacity int64 `json:"capacity"` + FormFactor string `json:"form_factor"` + SmartSupport bool `json:"smart_support"` + DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI) + DeviceType string `json:"device_type"` //device type is used for querying with -d/t flag, should only be used by collector. + SmartResults []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"` +} diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/smart.go b/webapp/backend/pkg/database/migrations/m20201107210306/smart.go new file mode 100644 index 0000000..798da4d --- /dev/null +++ b/webapp/backend/pkg/database/migrations/m20201107210306/smart.go @@ -0,0 +1,25 @@ +package m20201107210306 + +import ( + "gorm.io/gorm" + "time" +) + +type Smart struct { + gorm.Model + + DeviceWWN string `json:"device_wwn"` + Device Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key + + TestDate time.Time `json:"date"` + SmartStatus string `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed + + //Metrics + Temp int64 `json:"temp"` + PowerOnHours int64 `json:"power_on_hours"` + PowerCycleCount int64 `json:"power_cycle_count"` + + AtaAttributes []SmartAtaAttribute `json:"ata_attributes" gorm:"foreignkey:SmartId"` + NvmeAttributes []SmartNvmeAttribute `json:"nvme_attributes" gorm:"foreignkey:SmartId"` + ScsiAttributes []SmartScsiAttribute `json:"scsi_attributes" gorm:"foreignkey:SmartId"` +} diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go b/webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go new file mode 100644 index 0000000..f1f10dd --- /dev/null +++ b/webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go @@ -0,0 +1,25 @@ +package m20201107210306 + +import "gorm.io/gorm" + +type SmartAtaAttribute struct { + gorm.Model + + SmartId int `json:"smart_id"` + Smart Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key + + AttributeId int `json:"attribute_id"` + Name string `json:"name"` + Value int `json:"value"` + Worst int `json:"worst"` + Threshold int `json:"thresh"` + RawValue int64 `json:"raw_value"` + RawString string `json:"raw_string"` + WhenFailed string `json:"when_failed"` + + TransformedValue int64 `json:"transformed_value"` + Status string `gorm:"-" json:"status,omitempty"` + StatusReason string `gorm:"-" json:"status_reason,omitempty"` + FailureRate float64 `gorm:"-" json:"failure_rate,omitempty"` + History []SmartAtaAttribute `gorm:"-" json:"history,omitempty"` +} diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go b/webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go new file mode 100644 index 0000000..97abdad --- /dev/null +++ b/webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go @@ -0,0 +1,21 @@ +package m20201107210306 + +import "gorm.io/gorm" + +type SmartNvmeAttribute struct { + gorm.Model + + SmartId int `json:"smart_id"` + Smart Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key + + AttributeId string `json:"attribute_id"` //json string from smartctl + Name string `json:"name"` + Value int `json:"value"` + Threshold int `json:"thresh"` + + TransformedValue int64 `json:"transformed_value"` + Status string `gorm:"-" json:"status,omitempty"` + StatusReason string `gorm:"-" json:"status_reason,omitempty"` + FailureRate float64 `gorm:"-" json:"failure_rate,omitempty"` + History []SmartNvmeAttribute `gorm:"-" json:"history,omitempty"` +} diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go b/webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go new file mode 100644 index 0000000..4b2df5e --- /dev/null +++ b/webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go @@ -0,0 +1,21 @@ +package m20201107210306 + +import "gorm.io/gorm" + +type SmartScsiAttribute struct { + gorm.Model + + SmartId int `json:"smart_id"` + Smart Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key + + AttributeId string `json:"attribute_id"` //json string from smartctl + Name string `json:"name"` + Value int `json:"value"` + Threshold int `json:"thresh"` + + TransformedValue int64 `json:"transformed_value"` + Status string `gorm:"-" json:"status,omitempty"` + StatusReason string `gorm:"-" json:"status_reason,omitempty"` + FailureRate float64 `gorm:"-" json:"failure_rate,omitempty"` + History []SmartScsiAttribute `gorm:"-" json:"history,omitempty"` +} diff --git a/webapp/backend/pkg/database/scrutiny_repository.go b/webapp/backend/pkg/database/scrutiny_repository.go index f442a20..41b89aa 100644 --- a/webapp/backend/pkg/database/scrutiny_repository.go +++ b/webapp/backend/pkg/database/scrutiny_repository.go @@ -57,17 +57,18 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Gorm/SQLite setup //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location")) + globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location")) database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{ //TODO: figure out how to log database queries again. //Logger: logger + DisableForeignKeyConstraintWhenMigrating: true, }) if err != nil { return nil, fmt.Errorf("Failed to connect to database! - %v", err) } + globalLogger.Infof("Successfully connected to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location")) //database.SetLogger() - database.AutoMigrate(&models.Device{}) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // InfluxDB setup @@ -143,6 +144,16 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field if err != nil { return nil, err } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // InfluxDB & SQLite migrations + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //database.AutoMigrate(&models.Device{}) + err = deviceRepo.Migrate(backgroundContext) + if err != nil { + return nil, err + } + return &deviceRepo, nil } diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go new file mode 100644 index 0000000..0e299fa --- /dev/null +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -0,0 +1,96 @@ +package database + +import ( + "context" + "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/go-gormigrate/gormigrate/v2" + _ "github.com/jinzhu/gorm/dialects/sqlite" + "gorm.io/gorm" +) + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// SQLite migrations +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +//database.AutoMigrate(&models.Device{}) + +func (sr *scrutinyRepository) Migrate(ctx context.Context) error { + + sr.logger.Infoln("Database migration starting") + + m := gormigrate.New(sr.gormClient, gormigrate.DefaultOptions, []*gormigrate.Migration{ + { + ID: "20201107210306", // v0.3.13 (pre-influxdb schema). 9fac3c6308dc6cb6cd5bbc43a68cd93e8fb20b87 + Migrate: func(tx *gorm.DB) error { + // it's a good practice to copy the struct inside the function, + + return tx.AutoMigrate( + &m20201107210306.Device{}, + &m20201107210306.Smart{}, + &m20201107210306.SmartAtaAttribute{}, + &m20201107210306.SmartNvmeAttribute{}, + &m20201107210306.SmartNvmeAttribute{}, + ) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable( + &m20201107210306.Device{}, + &m20201107210306.Smart{}, + &m20201107210306.SmartAtaAttribute{}, + &m20201107210306.SmartNvmeAttribute{}, + &m20201107210306.SmartNvmeAttribute{}, + "self_tests", + ) + }, + }, + { + ID: "20220503113100", // backwards compatible - influxdb schema + Migrate: func(tx *gorm.DB) error { + // delete unnecessary table. + err := tx.Migrator().DropTable("self_tests") + if err != nil { + return err + } + + //add columns to the Device schema, so we can start adding data to the database & influxdb + err = tx.Migrator().AddColumn(&models.Device{}, "Label") //Label string `json:"label"` + if err != nil { + return err + } + err = tx.Migrator().AddColumn(&models.Device{}, "DeviceStatus") //DeviceStatus pkg.DeviceStatus `json:"device_status"` + if err != nil { + return err + } + + //TODO: migrate the data from GORM to influxdb. + + return nil + }, + }, + //{ + // ID: "20220503120000", // v0.4.0 - influxdb schema + // Migrate: func(tx *gorm.DB) error { + // // delete unnecessary tables. + // err := tx.Migrator().DropTable( + // &m20201107210306.Smart{}, + // &m20201107210306.SmartAtaAttribute{}, + // &m20201107210306.SmartNvmeAttribute{}, + // &m20201107210306.SmartNvmeAttribute{}, + // ) + // if err != nil { + // return err + // } + // + // //migrate the device database to the final version + // return tx.AutoMigrate(models.Device{}) + // }, + //}, + }) + + if err := m.Migrate(); err != nil { + sr.logger.Errorf("Database migration failed with error: %w", err) + return err + } + sr.logger.Infoln("Database migration completed successfully") + return nil +} From 2750ccef4aff529f194aab2eaf0b91eadecf4970 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Tue, 3 May 2022 11:52:47 -0700 Subject: [PATCH 2/8] call out deprecated structs so they are not accidentally used via autocomplete. --- webapp/backend/pkg/database/migrations/m20201107210306/device.go | 1 + webapp/backend/pkg/database/migrations/m20201107210306/smart.go | 1 + .../database/migrations/m20201107210306/smart_ata_attribute.go | 1 + .../database/migrations/m20201107210306/smart_nvme_attribute.go | 1 + .../database/migrations/m20201107210306/smart_scsci_attribute.go | 1 + 5 files changed, 5 insertions(+) diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/device.go b/webapp/backend/pkg/database/migrations/m20201107210306/device.go index 250122d..cac8b29 100644 --- a/webapp/backend/pkg/database/migrations/m20201107210306/device.go +++ b/webapp/backend/pkg/database/migrations/m20201107210306/device.go @@ -4,6 +4,7 @@ import ( "time" ) +// Deprecated: m20201107210306.Device is deprecated, only used by db migrations type Device struct { //GORM attributes, see: http://gorm.io/docs/conventions.html CreatedAt time.Time diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/smart.go b/webapp/backend/pkg/database/migrations/m20201107210306/smart.go index 798da4d..f0e615b 100644 --- a/webapp/backend/pkg/database/migrations/m20201107210306/smart.go +++ b/webapp/backend/pkg/database/migrations/m20201107210306/smart.go @@ -5,6 +5,7 @@ import ( "time" ) +// Deprecated: m20201107210306.Smart is deprecated, only used by db migrations type Smart struct { gorm.Model diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go b/webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go index f1f10dd..d20bac4 100644 --- a/webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go +++ b/webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go @@ -2,6 +2,7 @@ package m20201107210306 import "gorm.io/gorm" +// Deprecated: m20201107210306.SmartAtaAttribute is deprecated, only used by db migrations type SmartAtaAttribute struct { gorm.Model diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go b/webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go index 97abdad..3fe944a 100644 --- a/webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go +++ b/webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go @@ -2,6 +2,7 @@ package m20201107210306 import "gorm.io/gorm" +// Deprecated: m20201107210306.SmartNvmeAttribute is deprecated, only used by db migrations type SmartNvmeAttribute struct { gorm.Model diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go b/webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go index 4b2df5e..2a5be46 100644 --- a/webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go +++ b/webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go @@ -2,6 +2,7 @@ package m20201107210306 import "gorm.io/gorm" +// Deprecated: m20201107210306.SmartScsiAttribute is deprecated, only used by db migrations type SmartScsiAttribute struct { gorm.Model From 7d963c96a6aaa43714637175b9e5e5b0f51c320b Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Tue, 3 May 2022 12:03:08 -0700 Subject: [PATCH 3/8] writing pseudocode algorithm for data migration. --- .../pkg/database/scrutiny_repository_migrations.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 0e299fa..c7cedd2 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -63,12 +63,22 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { } //TODO: migrate the data from GORM to influxdb. - + //get a list of all devices: + // get a list of all smart scans in the last 2 weeks: + // get a list of associated smart attribute data: + // translate to a collector.SmartInfo object + // call scrutinyRepository.SaveSmartAttributes + // get a list of all smart scans in the last 9 weeks: + // do same as above (select 1 scan per week) + // get a list of all smart scans in the last 25 months: + // do same as above (select 1 scan per month) + // get a list of all smart scans: + // do same as above (select 1 scan per year) return nil }, }, //{ - // ID: "20220503120000", // v0.4.0 - influxdb schema + // ID: "20220503120000", // cleanup - v0.4.0 - influxdb schema // Migrate: func(tx *gorm.DB) error { // // delete unnecessary tables. // err := tx.Migrator().DropTable( From 8fe0dbed6baa05ed5c425a41679c2816c407d7b8 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Tue, 3 May 2022 22:40:31 -0700 Subject: [PATCH 4/8] partially working. Some datapoints are failing with panic and are silently ignored. TODO must fix. --- .../migrations/m20201107210306/device.go | 16 + ...tiny_repository_device_smart_attributes.go | 19 +- .../scrutiny_repository_migrations.go | 345 +++++++++++++++++- webapp/backend/pkg/models/collector/smart.go | 139 +++---- .../backend/pkg/models/measurements/smart.go | 72 ++-- 5 files changed, 482 insertions(+), 109 deletions(-) diff --git a/webapp/backend/pkg/database/migrations/m20201107210306/device.go b/webapp/backend/pkg/database/migrations/m20201107210306/device.go index cac8b29..11600c3 100644 --- a/webapp/backend/pkg/database/migrations/m20201107210306/device.go +++ b/webapp/backend/pkg/database/migrations/m20201107210306/device.go @@ -29,3 +29,19 @@ type Device struct { DeviceType string `json:"device_type"` //device type is used for querying with -d/t flag, should only be used by collector. SmartResults []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"` } + +const DeviceProtocolAta = "ATA" +const DeviceProtocolScsi = "SCSI" +const DeviceProtocolNvme = "NVMe" + +func (dv *Device) IsAta() bool { + return dv.DeviceProtocol == DeviceProtocolAta +} + +func (dv *Device) IsScsi() bool { + return dv.DeviceProtocol == DeviceProtocolScsi +} + +func (dv *Device) IsNvme() bool { + return dv.DeviceProtocol == DeviceProtocolNvme +} diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go index 263e08d..19d5861 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go @@ -6,8 +6,10 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" influxdb2 "github.com/influxdata/influxdb-client-go/v2" + "github.com/influxdata/influxdb-client-go/v2/api" log "github.com/sirupsen/logrus" "strings" + "time" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -22,13 +24,9 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin } tags, fields := deviceSmartData.Flatten() - p := influxdb2.NewPoint("smart", - tags, - fields, - deviceSmartData.Date) // write point immediately - return deviceSmartData, sr.influxWriteApi.WritePoint(ctx, p) + return deviceSmartData, sr.saveDatapoint(sr.influxWriteApi, "smart", tags, fields, deviceSmartData.Date, ctx) } func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) { @@ -93,6 +91,17 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn // Helper Methods //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, measurement string, tags map[string]string, fields map[string]interface{}, date time.Time, ctx context.Context) error { + sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields)) + p := influxdb2.NewPoint(measurement, + tags, + fields, + date) + + // write point immediately + return influxWriteApi.WritePoint(ctx, p) +} + func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string) string { /* diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index c7cedd2..0c12186 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -2,11 +2,16 @@ package database import ( "context" + "fmt" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" "github.com/go-gormigrate/gormigrate/v2" _ "github.com/jinzhu/gorm/dialects/sqlite" "gorm.io/gorm" + "strconv" + "time" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -66,14 +71,170 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { //get a list of all devices: // get a list of all smart scans in the last 2 weeks: // get a list of associated smart attribute data: - // translate to a collector.SmartInfo object - // call scrutinyRepository.SaveSmartAttributes + // translate to a measurements.Smart{} object + // call CUSTOM INFLUXDB SAVE FUNCTION (taking bucket as parameter) // get a list of all smart scans in the last 9 weeks: // do same as above (select 1 scan per week) // get a list of all smart scans in the last 25 months: // do same as above (select 1 scan per month) // get a list of all smart scans: // do same as above (select 1 scan per year) + + preDevices := []m20201107210306.Device{} //pre-migration device information + if err = tx.Preload("SmartResults", func(db *gorm.DB) *gorm.DB { + return db.Order("smarts.created_at ASC") //OLD: .Limit(devicesCount) + }).Find(&preDevices).Error; err != nil { + sr.logger.Errorln("Could not get device summary from DB", err) + return err + } + + //weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary. + weeklyLookup := map[string]bool{} + monthlyLookup := map[string]bool{} + yearlyLookup := map[string]bool{} + + //calculate bucket oldest dates + today := time.Now() + dailyBucketMax := today.Add(-RETENTION_PERIOD_15_DAYS_IN_SECONDS * time.Second) //15 days + weeklyBucketMax := today.Add(-RETENTION_PERIOD_9_WEEKS_IN_SECONDS * time.Second) //9 weeks + monthlyBucketMax := today.Add(-RETENTION_PERIOD_25_MONTHS_IN_SECONDS * time.Second) //25 weeks + + for _, preDevice := range preDevices { + for _, preSmartResult := range preDevice.SmartResults { //pre-migration smart results + + //we're looping in ASC mode, so from oldest entry to most current. + + //TODO: skip any results that are outside of the range that we care about for each bucket. + + err, postSmartResults := m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(tx, preDevice, preSmartResult) + if err != nil { + return err + } + smartTags, smartFields := postSmartResults.Flatten() + + err, postSmartTemp := m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice, preSmartResult) + if err != nil { + return err + } + tempTags, tempFields := postSmartTemp.Flatten() + tempTags["device_wwn"] = preDevice.WWN + + year, week := postSmartResults.Date.ISOWeek() + month := postSmartResults.Date.Month() + + yearStr := strconv.Itoa(year) + yearMonthStr := fmt.Sprintf("%d-%d", year, month) + yearWeekStr := fmt.Sprintf("%d-%d", year, week) + + //write data to daily bucket if in the last 15 days + if postSmartResults.Date.After(dailyBucketMax) { + sr.logger.Debugf("device (%s) smart data added to bucket: daily", preDevice.WWN) + // write point immediately + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")), + "smart", + smartTags, + smartFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), sr.appConfig.GetString("web.influxdb.bucket")), + "temp", + tempTags, + tempFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + } + + //write data to the weekly bucket if in the last 9 weeks, and week has not been processed yet + if _, weekExists := weeklyLookup[yearWeekStr]; !weekExists && postSmartResults.Date.After(weeklyBucketMax) { + sr.logger.Debugf("device (%s) smart data added to bucket: weekly", preDevice.WWN) + + //this week/year pair has not been processed + weeklyLookup[yearWeekStr] = true + // write point immediately + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))), + "smart", + smartTags, + smartFields, + postSmartResults.Date, ctx) + + if err != nil { + return err + } + + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_weekly", sr.appConfig.GetString("web.influxdb.bucket"))), + "temp", + tempTags, + tempFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + } + + //write data to the monthly bucket if in the last 9 weeks, and week has not been processed yet + if _, monthExists := monthlyLookup[yearMonthStr]; !monthExists && postSmartResults.Date.After(monthlyBucketMax) { + sr.logger.Debugf("device (%s) smart data added to bucket: monthly", preDevice.WWN) + //this month/year pair has not been processed + monthlyLookup[yearMonthStr] = true + // write point immediately + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))), + "smart", + smartTags, + smartFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_monthly", sr.appConfig.GetString("web.influxdb.bucket"))), + "temp", + tempTags, + tempFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + } + + if _, yearExists := yearlyLookup[yearStr]; !yearExists && year != today.Year() { + sr.logger.Debugf("device (%s) smart data added to bucket: yearly", preDevice.WWN) + //this year has not been processed + yearlyLookup[yearStr] = true + // write point immediately + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))), + "smart", + smartTags, + smartFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + + err = sr.saveDatapoint( + sr.influxClient.WriteAPIBlocking(sr.appConfig.GetString("web.influxdb.org"), fmt.Sprintf("%s_yearly", sr.appConfig.GetString("web.influxdb.bucket"))), + "temp", + tempTags, + tempFields, + postSmartResults.Date, ctx) + if err != nil { + return err + } + } + } + } + return nil }, }, @@ -104,3 +265,183 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { sr.logger.Infoln("Database migration completed successfully") return nil } + +// Deprecated +func m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.SmartTemperature) { + //extract temperature data for every datapoint + postSmartTemp := measurements.SmartTemperature{ + Date: preSmartResult.TestDate, + Temp: preSmartResult.Temp, + } + + return nil, postSmartTemp +} + +// Deprecated +func m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(database *gorm.DB, preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.Smart) { + //create a measurements.Smart object (which we will then push to the InfluxDB) + postDeviceSmartData := measurements.Smart{ + Date: preSmartResult.TestDate, + DeviceWWN: preDevice.WWN, + DeviceProtocol: preDevice.DeviceProtocol, + Temp: preSmartResult.Temp, + PowerOnHours: preSmartResult.PowerOnHours, + PowerCycleCount: preSmartResult.PowerCycleCount, + + // this needs to be populated using measurements.Smart.ProcessAtaSmartInfo, ProcessScsiSmartInfo or ProcessNvmeSmartInfo + // because those functions will take into account thresholds (which we didn't consider correctly previously) + Attributes: map[string]measurements.SmartAttribute{}, + } + + result := database.Preload("AtaAttributes").Preload("NvmeAttributes").Preload("ScsiAttributes").Find(&preSmartResult) + if result.Error != nil { + return result.Error, postDeviceSmartData + } + + if preDevice.IsAta() { + preAtaSmartAttributesTable := []collector.AtaSmartAttributesTableItem{} + for _, preAtaAttribute := range preSmartResult.AtaAttributes { + preAtaSmartAttributesTable = append(preAtaSmartAttributesTable, collector.AtaSmartAttributesTableItem{ + ID: preAtaAttribute.AttributeId, + Name: preAtaAttribute.Name, + Value: int64(preAtaAttribute.Value), + Worst: int64(preAtaAttribute.Worst), + Thresh: int64(preAtaAttribute.Threshold), + WhenFailed: preAtaAttribute.WhenFailed, + Flags: struct { + Value int `json:"value"` + String string `json:"string"` + Prefailure bool `json:"prefailure"` + UpdatedOnline bool `json:"updated_online"` + Performance bool `json:"performance"` + ErrorRate bool `json:"error_rate"` + EventCount bool `json:"event_count"` + AutoKeep bool `json:"auto_keep"` + }{ + Value: 0, + String: "", + Prefailure: false, + UpdatedOnline: false, + Performance: false, + ErrorRate: false, + EventCount: false, + AutoKeep: false, + }, + Raw: struct { + Value int64 `json:"value"` + String string `json:"string"` + }{ + Value: preAtaAttribute.RawValue, + String: preAtaAttribute.RawString, + }, + }) + } + + postDeviceSmartData.ProcessAtaSmartInfo(preAtaSmartAttributesTable) + + } else if preDevice.IsNvme() { + //info collector.SmartInfo + postNvmeSmartHealthInformation := collector.NvmeSmartHealthInformationLog{} + + for _, preNvmeAttribute := range preSmartResult.NvmeAttributes { + switch preNvmeAttribute.AttributeId { + case "critical_warning": + postNvmeSmartHealthInformation.CriticalWarning = int64(preNvmeAttribute.Value) + case "temperature": + postNvmeSmartHealthInformation.Temperature = int64(preNvmeAttribute.Value) + case "available_spare": + postNvmeSmartHealthInformation.AvailableSpare = int64(preNvmeAttribute.Value) + case "available_spare_threshold": + postNvmeSmartHealthInformation.AvailableSpareThreshold = int64(preNvmeAttribute.Value) + case "percentage_used": + postNvmeSmartHealthInformation.PercentageUsed = int64(preNvmeAttribute.Value) + case "data_units_read": + postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value) + case "data_units_written": + postNvmeSmartHealthInformation.DataUnitsWritten = int64(preNvmeAttribute.Value) + case "host_reads": + postNvmeSmartHealthInformation.HostReads = int64(preNvmeAttribute.Value) + case "host_writes": + postNvmeSmartHealthInformation.HostWrites = int64(preNvmeAttribute.Value) + case "controller_busy_time": + postNvmeSmartHealthInformation.ControllerBusyTime = int64(preNvmeAttribute.Value) + case "power_cycles": + postNvmeSmartHealthInformation.PowerCycles = int64(preNvmeAttribute.Value) + case "power_on_hours": + postNvmeSmartHealthInformation.PowerOnHours = int64(preNvmeAttribute.Value) + case "unsafe_shutdowns": + postNvmeSmartHealthInformation.UnsafeShutdowns = int64(preNvmeAttribute.Value) + case "media_errors": + postNvmeSmartHealthInformation.MediaErrors = int64(preNvmeAttribute.Value) + case "num_err_log_entries": + postNvmeSmartHealthInformation.NumErrLogEntries = int64(preNvmeAttribute.Value) + case "warning_temp_time": + postNvmeSmartHealthInformation.WarningTempTime = int64(preNvmeAttribute.Value) + case "critical_comp_time": + postNvmeSmartHealthInformation.CriticalCompTime = int64(preNvmeAttribute.Value) + } + } + + postDeviceSmartData.ProcessNvmeSmartInfo(postNvmeSmartHealthInformation) + + } else if preDevice.IsScsi() { + //info collector.SmartInfo + var postScsiGrownDefectList int64 + postScsiErrorCounterLog := collector.ScsiErrorCounterLog{ + Read: struct { + ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` + ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` + ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` + TotalErrorsCorrected int64 `json:"total_errors_corrected"` + CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` + GigabytesProcessed string `json:"gigabytes_processed"` + TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` + }{}, + Write: struct { + ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` + ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` + ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` + TotalErrorsCorrected int64 `json:"total_errors_corrected"` + CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` + GigabytesProcessed string `json:"gigabytes_processed"` + TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` + }{}, + } + + for _, preScsiAttribute := range preSmartResult.ScsiAttributes { + switch preScsiAttribute.AttributeId { + case "scsi_grown_defect_list": + postScsiGrownDefectList = int64(preScsiAttribute.Value) + case "read.errors_corrected_by_eccfast": + postScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value) + case "read.errors_corrected_by_eccdelayed": + postScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value) + case "read.errors_corrected_by_rereads_rewrites": + postScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value) + case "read.total_errors_corrected": + postScsiErrorCounterLog.Read.TotalErrorsCorrected = int64(preScsiAttribute.Value) + case "read.correction_algorithm_invocations": + postScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value) + case "read.total_uncorrected_errors": + postScsiErrorCounterLog.Read.TotalUncorrectedErrors = int64(preScsiAttribute.Value) + case "write.errors_corrected_by_eccfast": + postScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast = int64(preScsiAttribute.Value) + case "write.errors_corrected_by_eccdelayed": + postScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed = int64(preScsiAttribute.Value) + case "write.errors_corrected_by_rereads_rewrites": + postScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites = int64(preScsiAttribute.Value) + case "write.total_errors_corrected": + postScsiErrorCounterLog.Write.TotalErrorsCorrected = int64(preScsiAttribute.Value) + case "write.correction_algorithm_invocations": + postScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations = int64(preScsiAttribute.Value) + case "write.total_uncorrected_errors": + postScsiErrorCounterLog.Write.TotalUncorrectedErrors = int64(preScsiAttribute.Value) + } + } + postDeviceSmartData.ProcessScsiSmartInfo(postScsiGrownDefectList, postScsiErrorCounterLog) + } else { + return fmt.Errorf("Unknown device protocol: %s", preDevice.DeviceProtocol), postDeviceSmartData + } + + return nil, postDeviceSmartData +} diff --git a/webapp/backend/pkg/models/collector/smart.go b/webapp/backend/pkg/models/collector/smart.go index 8cb5653..4f7ae3d 100644 --- a/webapp/backend/pkg/models/collector/smart.go +++ b/webapp/backend/pkg/models/collector/smart.go @@ -134,29 +134,8 @@ type SmartInfo struct { Table []int64 `json:"table"` } `json:"ata_sct_temperature_history"` AtaSmartAttributes struct { - Revision int `json:"revision"` - Table []struct { - ID int `json:"id"` - Name string `json:"name"` - Value int64 `json:"value"` - Worst int64 `json:"worst"` - Thresh int64 `json:"thresh"` - WhenFailed string `json:"when_failed"` - Flags struct { - Value int `json:"value"` - String string `json:"string"` - Prefailure bool `json:"prefailure"` - UpdatedOnline bool `json:"updated_online"` - Performance bool `json:"performance"` - ErrorRate bool `json:"error_rate"` - EventCount bool `json:"event_count"` - AutoKeep bool `json:"auto_keep"` - } `json:"flags"` - Raw struct { - Value int64 `json:"value"` - String string `json:"string"` - } `json:"raw"` - } `json:"table"` + Revision int `json:"revision"` + Table []AtaSmartAttributesTableItem `json:"table"` } `json:"ata_smart_attributes"` AtaSmartErrorLog struct { Summary struct { @@ -250,49 +229,77 @@ type SmartInfo struct { } `json:"utilization"` FormattedLbaSize int `json:"formatted_lba_size"` } `json:"nvme_namespaces"` - NvmeSmartHealthInformationLog struct { - CriticalWarning int64 `json:"critical_warning"` - Temperature int64 `json:"temperature"` - AvailableSpare int64 `json:"available_spare"` - AvailableSpareThreshold int64 `json:"available_spare_threshold"` - PercentageUsed int64 `json:"percentage_used"` - DataUnitsRead int64 `json:"data_units_read"` - DataUnitsWritten int64 `json:"data_units_written"` - HostReads int64 `json:"host_reads"` - HostWrites int64 `json:"host_writes"` - ControllerBusyTime int64 `json:"controller_busy_time"` - PowerCycles int64 `json:"power_cycles"` - PowerOnHours int64 `json:"power_on_hours"` - UnsafeShutdowns int64 `json:"unsafe_shutdowns"` - MediaErrors int64 `json:"media_errors"` - NumErrLogEntries int64 `json:"num_err_log_entries"` - WarningTempTime int64 `json:"warning_temp_time"` - CriticalCompTime int64 `json:"critical_comp_time"` - } `json:"nvme_smart_health_information_log"` + NvmeSmartHealthInformationLog NvmeSmartHealthInformationLog `json:"nvme_smart_health_information_log"` // SCSI Protocol Specific Fields - Vendor string `json:"vendor"` - Product string `json:"product"` - ScsiVersion string `json:"scsi_version"` - ScsiGrownDefectList int64 `json:"scsi_grown_defect_list"` - ScsiErrorCounterLog struct { - Read struct { - ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` - ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` - ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` - TotalErrorsCorrected int64 `json:"total_errors_corrected"` - CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` - GigabytesProcessed string `json:"gigabytes_processed"` - TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` - } `json:"read"` - Write struct { - ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` - ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` - ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` - TotalErrorsCorrected int64 `json:"total_errors_corrected"` - CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` - GigabytesProcessed string `json:"gigabytes_processed"` - TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` - } `json:"write"` - } `json:"scsi_error_counter_log"` + Vendor string `json:"vendor"` + Product string `json:"product"` + ScsiVersion string `json:"scsi_version"` + ScsiGrownDefectList int64 `json:"scsi_grown_defect_list"` + ScsiErrorCounterLog ScsiErrorCounterLog `json:"scsi_error_counter_log"` +} + +//Primary Attribute Structs +type AtaSmartAttributesTableItem struct { + ID int `json:"id"` + Name string `json:"name"` + Value int64 `json:"value"` + Worst int64 `json:"worst"` + Thresh int64 `json:"thresh"` + WhenFailed string `json:"when_failed"` + Flags struct { + Value int `json:"value"` + String string `json:"string"` + Prefailure bool `json:"prefailure"` + UpdatedOnline bool `json:"updated_online"` + Performance bool `json:"performance"` + ErrorRate bool `json:"error_rate"` + EventCount bool `json:"event_count"` + AutoKeep bool `json:"auto_keep"` + } `json:"flags"` + Raw struct { + Value int64 `json:"value"` + String string `json:"string"` + } `json:"raw"` +} + +type NvmeSmartHealthInformationLog struct { + CriticalWarning int64 `json:"critical_warning"` + Temperature int64 `json:"temperature"` + AvailableSpare int64 `json:"available_spare"` + AvailableSpareThreshold int64 `json:"available_spare_threshold"` + PercentageUsed int64 `json:"percentage_used"` + DataUnitsRead int64 `json:"data_units_read"` + DataUnitsWritten int64 `json:"data_units_written"` + HostReads int64 `json:"host_reads"` + HostWrites int64 `json:"host_writes"` + ControllerBusyTime int64 `json:"controller_busy_time"` + PowerCycles int64 `json:"power_cycles"` + PowerOnHours int64 `json:"power_on_hours"` + UnsafeShutdowns int64 `json:"unsafe_shutdowns"` + MediaErrors int64 `json:"media_errors"` + NumErrLogEntries int64 `json:"num_err_log_entries"` + WarningTempTime int64 `json:"warning_temp_time"` + CriticalCompTime int64 `json:"critical_comp_time"` +} + +type ScsiErrorCounterLog struct { + Read struct { + ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` + ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` + ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` + TotalErrorsCorrected int64 `json:"total_errors_corrected"` + CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` + GigabytesProcessed string `json:"gigabytes_processed"` + TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` + } `json:"read"` + Write struct { + ErrorsCorrectedByEccfast int64 `json:"errors_corrected_by_eccfast"` + ErrorsCorrectedByEccdelayed int64 `json:"errors_corrected_by_eccdelayed"` + ErrorsCorrectedByRereadsRewrites int64 `json:"errors_corrected_by_rereads_rewrites"` + TotalErrorsCorrected int64 `json:"total_errors_corrected"` + CorrectionAlgorithmInvocations int64 `json:"correction_algorithm_invocations"` + GigabytesProcessed string `json:"gigabytes_processed"` + TotalUncorrectedErrors int64 `json:"total_uncorrected_errors"` + } `json:"write"` } diff --git a/webapp/backend/pkg/models/measurements/smart.go b/webapp/backend/pkg/models/measurements/smart.go index 37810f4..cc25593 100644 --- a/webapp/backend/pkg/models/measurements/smart.go +++ b/webapp/backend/pkg/models/measurements/smart.go @@ -125,20 +125,20 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er // process ATA/NVME/SCSI protocol data sm.Attributes = map[string]SmartAttribute{} if sm.DeviceProtocol == pkg.DeviceProtocolAta { - sm.ProcessAtaSmartInfo(info) + sm.ProcessAtaSmartInfo(info.AtaSmartAttributes.Table) } else if sm.DeviceProtocol == pkg.DeviceProtocolNvme { - sm.ProcessNvmeSmartInfo(info) + sm.ProcessNvmeSmartInfo(info.NvmeSmartHealthInformationLog) } else if sm.DeviceProtocol == pkg.DeviceProtocolScsi { - sm.ProcessScsiSmartInfo(info) + sm.ProcessScsiSmartInfo(info.ScsiGrownDefectList, info.ScsiErrorCounterLog) } return nil } //generate SmartAtaAttribute entries from Scrutiny Collector Smart data. -func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) { +func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTableItem) { sm.Status = pkg.DeviceStatusPassed - for _, collectorAttr := range info.AtaSmartAttributes.Table { + for _, collectorAttr := range tableItems { attrModel := SmartAtaAttribute{ AttributeId: collectorAttr.ID, Value: collectorAttr.Value, @@ -164,25 +164,25 @@ func (sm *Smart) ProcessAtaSmartInfo(info collector.SmartInfo) { } //generate SmartNvmeAttribute entries from Scrutiny Collector Smart data. -func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) { +func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.NvmeSmartHealthInformationLog) { sm.Attributes = map[string]SmartAttribute{ - "critical_warning": (&SmartNvmeAttribute{AttributeId: "critical_warning", Value: info.NvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0}).PopulateAttributeStatus(), - "temperature": (&SmartNvmeAttribute{AttributeId: "temperature", Value: info.NvmeSmartHealthInformationLog.Temperature, Threshold: -1}).PopulateAttributeStatus(), - "available_spare": (&SmartNvmeAttribute{AttributeId: "available_spare", Value: info.NvmeSmartHealthInformationLog.AvailableSpare, Threshold: info.NvmeSmartHealthInformationLog.AvailableSpareThreshold}).PopulateAttributeStatus(), - "percentage_used": (&SmartNvmeAttribute{AttributeId: "percentage_used", Value: info.NvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100}).PopulateAttributeStatus(), - "data_units_read": (&SmartNvmeAttribute{AttributeId: "data_units_read", Value: info.NvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1}).PopulateAttributeStatus(), - "data_units_written": (&SmartNvmeAttribute{AttributeId: "data_units_written", Value: info.NvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1}).PopulateAttributeStatus(), - "host_reads": (&SmartNvmeAttribute{AttributeId: "host_reads", Value: info.NvmeSmartHealthInformationLog.HostReads, Threshold: -1}).PopulateAttributeStatus(), - "host_writes": (&SmartNvmeAttribute{AttributeId: "host_writes", Value: info.NvmeSmartHealthInformationLog.HostWrites, Threshold: -1}).PopulateAttributeStatus(), - "controller_busy_time": (&SmartNvmeAttribute{AttributeId: "controller_busy_time", Value: info.NvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1}).PopulateAttributeStatus(), - "power_cycles": (&SmartNvmeAttribute{AttributeId: "power_cycles", Value: info.NvmeSmartHealthInformationLog.PowerCycles, Threshold: -1}).PopulateAttributeStatus(), - "power_on_hours": (&SmartNvmeAttribute{AttributeId: "power_on_hours", Value: info.NvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1}).PopulateAttributeStatus(), - "unsafe_shutdowns": (&SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Value: info.NvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1}).PopulateAttributeStatus(), - "media_errors": (&SmartNvmeAttribute{AttributeId: "media_errors", Value: info.NvmeSmartHealthInformationLog.MediaErrors, Threshold: 0}).PopulateAttributeStatus(), - "num_err_log_entries": (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: info.NvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0}).PopulateAttributeStatus(), - "warning_temp_time": (&SmartNvmeAttribute{AttributeId: "warning_temp_time", Value: info.NvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1}).PopulateAttributeStatus(), - "critical_comp_time": (&SmartNvmeAttribute{AttributeId: "critical_comp_time", Value: info.NvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1}).PopulateAttributeStatus(), + "critical_warning": (&SmartNvmeAttribute{AttributeId: "critical_warning", Value: nvmeSmartHealthInformationLog.CriticalWarning, Threshold: 0}).PopulateAttributeStatus(), + "temperature": (&SmartNvmeAttribute{AttributeId: "temperature", Value: nvmeSmartHealthInformationLog.Temperature, Threshold: -1}).PopulateAttributeStatus(), + "available_spare": (&SmartNvmeAttribute{AttributeId: "available_spare", Value: nvmeSmartHealthInformationLog.AvailableSpare, Threshold: nvmeSmartHealthInformationLog.AvailableSpareThreshold}).PopulateAttributeStatus(), + "percentage_used": (&SmartNvmeAttribute{AttributeId: "percentage_used", Value: nvmeSmartHealthInformationLog.PercentageUsed, Threshold: 100}).PopulateAttributeStatus(), + "data_units_read": (&SmartNvmeAttribute{AttributeId: "data_units_read", Value: nvmeSmartHealthInformationLog.DataUnitsRead, Threshold: -1}).PopulateAttributeStatus(), + "data_units_written": (&SmartNvmeAttribute{AttributeId: "data_units_written", Value: nvmeSmartHealthInformationLog.DataUnitsWritten, Threshold: -1}).PopulateAttributeStatus(), + "host_reads": (&SmartNvmeAttribute{AttributeId: "host_reads", Value: nvmeSmartHealthInformationLog.HostReads, Threshold: -1}).PopulateAttributeStatus(), + "host_writes": (&SmartNvmeAttribute{AttributeId: "host_writes", Value: nvmeSmartHealthInformationLog.HostWrites, Threshold: -1}).PopulateAttributeStatus(), + "controller_busy_time": (&SmartNvmeAttribute{AttributeId: "controller_busy_time", Value: nvmeSmartHealthInformationLog.ControllerBusyTime, Threshold: -1}).PopulateAttributeStatus(), + "power_cycles": (&SmartNvmeAttribute{AttributeId: "power_cycles", Value: nvmeSmartHealthInformationLog.PowerCycles, Threshold: -1}).PopulateAttributeStatus(), + "power_on_hours": (&SmartNvmeAttribute{AttributeId: "power_on_hours", Value: nvmeSmartHealthInformationLog.PowerOnHours, Threshold: -1}).PopulateAttributeStatus(), + "unsafe_shutdowns": (&SmartNvmeAttribute{AttributeId: "unsafe_shutdowns", Value: nvmeSmartHealthInformationLog.UnsafeShutdowns, Threshold: -1}).PopulateAttributeStatus(), + "media_errors": (&SmartNvmeAttribute{AttributeId: "media_errors", Value: nvmeSmartHealthInformationLog.MediaErrors, Threshold: 0}).PopulateAttributeStatus(), + "num_err_log_entries": (&SmartNvmeAttribute{AttributeId: "num_err_log_entries", Value: nvmeSmartHealthInformationLog.NumErrLogEntries, Threshold: 0}).PopulateAttributeStatus(), + "warning_temp_time": (&SmartNvmeAttribute{AttributeId: "warning_temp_time", Value: nvmeSmartHealthInformationLog.WarningTempTime, Threshold: -1}).PopulateAttributeStatus(), + "critical_comp_time": (&SmartNvmeAttribute{AttributeId: "critical_comp_time", Value: nvmeSmartHealthInformationLog.CriticalCompTime, Threshold: -1}).PopulateAttributeStatus(), } //find analyzed attribute status @@ -194,21 +194,21 @@ func (sm *Smart) ProcessNvmeSmartInfo(info collector.SmartInfo) { } //generate SmartScsiAttribute entries from Scrutiny Collector Smart data. -func (sm *Smart) ProcessScsiSmartInfo(info collector.SmartInfo) { +func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog collector.ScsiErrorCounterLog) { sm.Attributes = map[string]SmartAttribute{ - "scsi_grown_defect_list": (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: info.ScsiGrownDefectList, Threshold: 0}).PopulateAttributeStatus(), - "read_errors_corrected_by_eccfast": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(), - "read_errors_corrected_by_eccdelayed": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(), - "read_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_rereads_rewrites", Value: info.ScsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(), - "read_total_errors_corrected": (&SmartScsiAttribute{AttributeId: "read_total_errors_corrected", Value: info.ScsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(), - "read_correction_algorithm_invocations": (&SmartScsiAttribute{AttributeId: "read_correction_algorithm_invocations", Value: info.ScsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(), - "read_total_uncorrected_errors": (&SmartScsiAttribute{AttributeId: "read_total_uncorrected_errors", Value: info.ScsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(), - "write_errors_corrected_by_eccfast": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccfast", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(), - "write_errors_corrected_by_eccdelayed": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccdelayed", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(), - "write_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_rereads_rewrites", Value: info.ScsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(), - "write_total_errors_corrected": (&SmartScsiAttribute{AttributeId: "write_total_errors_corrected", Value: info.ScsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(), - "write_correction_algorithm_invocations": (&SmartScsiAttribute{AttributeId: "write_correction_algorithm_invocations", Value: info.ScsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(), - "write_total_uncorrected_errors": (&SmartScsiAttribute{AttributeId: "write_total_uncorrected_errors", Value: info.ScsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(), + "scsi_grown_defect_list": (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: defectGrownList, Threshold: 0}).PopulateAttributeStatus(), + "read_errors_corrected_by_eccfast": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(), + "read_errors_corrected_by_eccdelayed": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(), + "read_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(), + "read_total_errors_corrected": (&SmartScsiAttribute{AttributeId: "read_total_errors_corrected", Value: scsiErrorCounterLog.Read.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(), + "read_correction_algorithm_invocations": (&SmartScsiAttribute{AttributeId: "read_correction_algorithm_invocations", Value: scsiErrorCounterLog.Read.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(), + "read_total_uncorrected_errors": (&SmartScsiAttribute{AttributeId: "read_total_uncorrected_errors", Value: scsiErrorCounterLog.Read.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(), + "write_errors_corrected_by_eccfast": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(), + "write_errors_corrected_by_eccdelayed": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(), + "write_errors_corrected_by_rereads_rewrites": (&SmartScsiAttribute{AttributeId: "write_errors_corrected_by_rereads_rewrites", Value: scsiErrorCounterLog.Write.ErrorsCorrectedByRereadsRewrites, Threshold: 0}).PopulateAttributeStatus(), + "write_total_errors_corrected": (&SmartScsiAttribute{AttributeId: "write_total_errors_corrected", Value: scsiErrorCounterLog.Write.TotalErrorsCorrected, Threshold: -1}).PopulateAttributeStatus(), + "write_correction_algorithm_invocations": (&SmartScsiAttribute{AttributeId: "write_correction_algorithm_invocations", Value: scsiErrorCounterLog.Write.CorrectionAlgorithmInvocations, Threshold: -1}).PopulateAttributeStatus(), + "write_total_uncorrected_errors": (&SmartScsiAttribute{AttributeId: "write_total_uncorrected_errors", Value: scsiErrorCounterLog.Write.TotalUncorrectedErrors, Threshold: 0}).PopulateAttributeStatus(), } //find analyzed attribute status From fc5a9ba15e15e3e593975eb273f2dcff830c43bb Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 4 May 2022 20:13:11 -0700 Subject: [PATCH 5/8] fixed device processing in details page. Summary query is still broken. --- .../backend/pkg/database/scrutiny_repository.go | 2 +- ...scrutiny_repository_device_smart_attributes.go | 2 +- .../database/scrutiny_repository_migrations.go | 15 ++++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/webapp/backend/pkg/database/scrutiny_repository.go b/webapp/backend/pkg/database/scrutiny_repository.go index 41b89aa..a285091 100644 --- a/webapp/backend/pkg/database/scrutiny_repository.go +++ b/webapp/backend/pkg/database/scrutiny_repository.go @@ -368,7 +368,7 @@ func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []str return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR} case DURATION_KEY_FOREVER: //data stored before the last year - return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR} + return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR, DURATION_KEY_FOREVER} } return []string{DURATION_KEY_WEEK} } diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go index 19d5861..eaa1852 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go @@ -92,7 +92,7 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, measurement string, tags map[string]string, fields map[string]interface{}, date time.Time, ctx context.Context) error { - sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields)) + //sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields)) p := influxdb2.NewPoint(measurement, tags, fields, diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 0c12186..5f6496a 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -88,11 +88,6 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { return err } - //weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary. - weeklyLookup := map[string]bool{} - monthlyLookup := map[string]bool{} - yearlyLookup := map[string]bool{} - //calculate bucket oldest dates today := time.Now() dailyBucketMax := today.Add(-RETENTION_PERIOD_15_DAYS_IN_SECONDS * time.Second) //15 days @@ -100,6 +95,12 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { monthlyBucketMax := today.Add(-RETENTION_PERIOD_25_MONTHS_IN_SECONDS * time.Second) //25 weeks for _, preDevice := range preDevices { + sr.logger.Infof("\n====================================\n\nBegin processing device %s\n\n====================================\n", preDevice.WWN) + + //weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary. + weeklyLookup := map[string]bool{} + monthlyLookup := map[string]bool{} + yearlyLookup := map[string]bool{} for _, preSmartResult := range preDevice.SmartResults { //pre-migration smart results //we're looping in ASC mode, so from oldest entry to most current. @@ -183,6 +184,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { //write data to the monthly bucket if in the last 9 weeks, and week has not been processed yet if _, monthExists := monthlyLookup[yearMonthStr]; !monthExists && postSmartResults.Date.After(monthlyBucketMax) { sr.logger.Debugf("device (%s) smart data added to bucket: monthly", preDevice.WWN) + //this month/year pair has not been processed monthlyLookup[yearMonthStr] = true // write point immediately @@ -209,6 +211,7 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { if _, yearExists := yearlyLookup[yearStr]; !yearExists && year != today.Year() { sr.logger.Debugf("device (%s) smart data added to bucket: yearly", preDevice.WWN) + //this year has not been processed yearlyLookup[yearStr] = true // write point immediately @@ -233,6 +236,8 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { } } } + fmt.Printf("finished processing device %s. weekly: %d, monthly: %d, yearly: %d", preDevice.WWN, len(weeklyLookup), len(monthlyLookup), len(yearlyLookup)) + } return nil From 702518579b98c5f00cad985100f098304149d608 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 4 May 2022 20:40:48 -0700 Subject: [PATCH 6/8] fixed summary query. --- .../pkg/database/scrutiny_repository.go | 49 +++++++++++++++---- ...tiny_repository_device_smart_attributes.go | 41 +++++++++------- 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/webapp/backend/pkg/database/scrutiny_repository.go b/webapp/backend/pkg/database/scrutiny_repository.go index a285091..c35923f 100644 --- a/webapp/backend/pkg/database/scrutiny_repository.go +++ b/webapp/backend/pkg/database/scrutiny_repository.go @@ -252,15 +252,46 @@ func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*model // Get parser flux query result //appConfig.GetString("web.influxdb.bucket") queryStr := fmt.Sprintf(` - import "influxdata/influxdb/schema" - from(bucket: "%s") - |> range(start: -1y, stop: now()) - |> filter(fn: (r) => r["_measurement"] == "smart" ) - |> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date") - |> last() - |> schema.fieldsAsCols() - |> group(columns: ["device_wwn"]) - |> yield(name: "last") + import "influxdata/influxdb/schema" + bucketBaseName = "%s" + + dailyData = from(bucket: bucketBaseName) + |> range(start: -10y, stop: now()) + |> filter(fn: (r) => r["_measurement"] == "smart" ) + |> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date") + |> last() + |> schema.fieldsAsCols() + |> group(columns: ["device_wwn"]) + + weeklyData = from(bucket: bucketBaseName + "_weekly") + |> range(start: -10y, stop: now()) + |> filter(fn: (r) => r["_measurement"] == "smart" ) + |> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date") + |> last() + |> schema.fieldsAsCols() + |> group(columns: ["device_wwn"]) + + monthlyData = from(bucket: bucketBaseName + "_monthly") + |> range(start: -10y, stop: now()) + |> filter(fn: (r) => r["_measurement"] == "smart" ) + |> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date") + |> last() + |> schema.fieldsAsCols() + |> group(columns: ["device_wwn"]) + + yearlyData = from(bucket: bucketBaseName + "_yearly") + |> range(start: -10y, stop: now()) + |> filter(fn: (r) => r["_measurement"] == "smart" ) + |> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date") + |> last() + |> schema.fieldsAsCols() + |> group(columns: ["device_wwn"]) + + union(tables: [dailyData, weeklyData, monthlyData, yearlyData]) + |> sort(columns: ["_time"], desc: false) + |> group(columns: ["device_wwn"]) + |> last(column: "device_wwn") + |> yield(name: "last") `, sr.appConfig.GetString("web.influxdb.bucket"), ) diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go index eaa1852..0479986 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go @@ -108,25 +108,32 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration import "influxdata/influxdb/schema" weekData = from(bucket: "metrics") - |> range(start: -1w, stop: now()) - |> filter(fn: (r) => r["_measurement"] == "smart" ) - |> filter(fn: (r) => r["device_wwn"] == "%s" ) - |> aggregateWindow(every: 1h, fn: mean, createEmpty: false) - |> group(columns: ["device_wwn"]) - |> toInt() + |> range(start: -1w, stop: now()) + |> filter(fn: (r) => r["_measurement"] == "smart" ) + |> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" ) + |> schema.fieldsAsCols() monthData = from(bucket: "metrics_weekly") - |> range(start: -1mo, stop: now()) - |> filter(fn: (r) => r["_measurement"] == "smart" ) - |> filter(fn: (r) => r["device_wwn"] == "%s" ) - |> aggregateWindow(every: 1h, fn: mean, createEmpty: false) - |> group(columns: ["device_wwn"]) - |> toInt() - - union(tables: [weekData, monthData]) - |> group(columns: ["device_wwn"]) - |> sort(columns: ["_time"], desc: false) - |> schema.fieldsAsCols() + |> range(start: -1mo, stop: -1w) + |> filter(fn: (r) => r["_measurement"] == "smart" ) + |> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" ) + |> schema.fieldsAsCols() + + yearData = from(bucket: "metrics_monthly") + |> range(start: -1y, stop: -1mo) + |> filter(fn: (r) => r["_measurement"] == "smart" ) + |> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" ) + |> schema.fieldsAsCols() + + foreverData = from(bucket: "metrics_yearly") + |> range(start: -10y, stop: -1y) + |> filter(fn: (r) => r["_measurement"] == "smart" ) + |> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" ) + |> schema.fieldsAsCols() + + union(tables: [weekData, monthData, yearData, foreverData]) + |> sort(columns: ["_time"], desc: false) + |> yield(name: "last") */ From 5f12fbb51095ddd5ce507335a7c514f5b3665e9b Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 4 May 2022 20:50:17 -0700 Subject: [PATCH 7/8] enable final migration cleanup. --- .../scrutiny_repository_migrations.go | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 5f6496a..f431145 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -95,7 +95,8 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { monthlyBucketMax := today.Add(-RETENTION_PERIOD_25_MONTHS_IN_SECONDS * time.Second) //25 weeks for _, preDevice := range preDevices { - sr.logger.Infof("\n====================================\n\nBegin processing device %s\n\n====================================\n", preDevice.WWN) + sr.logger.Debugf("====================================") + sr.logger.Infof("begin processing device: %s", preDevice.WWN) //weekly, monthly, yearly lookup storage, so we don't add more data to the buckets than necessary. weeklyLookup := map[string]bool{} @@ -105,8 +106,6 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { //we're looping in ASC mode, so from oldest entry to most current. - //TODO: skip any results that are outside of the range that we care about for each bucket. - err, postSmartResults := m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(tx, preDevice, preSmartResult) if err != nil { return err @@ -236,31 +235,31 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { } } } - fmt.Printf("finished processing device %s. weekly: %d, monthly: %d, yearly: %d", preDevice.WWN, len(weeklyLookup), len(monthlyLookup), len(yearlyLookup)) + sr.logger.Infof("finished processing device %s. weekly: %d, monthly: %d, yearly: %d", preDevice.WWN, len(weeklyLookup), len(monthlyLookup), len(yearlyLookup)) } return nil }, }, - //{ - // ID: "20220503120000", // cleanup - v0.4.0 - influxdb schema - // Migrate: func(tx *gorm.DB) error { - // // delete unnecessary tables. - // err := tx.Migrator().DropTable( - // &m20201107210306.Smart{}, - // &m20201107210306.SmartAtaAttribute{}, - // &m20201107210306.SmartNvmeAttribute{}, - // &m20201107210306.SmartNvmeAttribute{}, - // ) - // if err != nil { - // return err - // } - // - // //migrate the device database to the final version - // return tx.AutoMigrate(models.Device{}) - // }, - //}, + { + ID: "20220503120000", // cleanup - v0.4.0 - influxdb schema + Migrate: func(tx *gorm.DB) error { + // delete unnecessary tables. + err := tx.Migrator().DropTable( + &m20201107210306.Smart{}, + &m20201107210306.SmartAtaAttribute{}, + &m20201107210306.SmartNvmeAttribute{}, + &m20201107210306.SmartScsiAttribute{}, + ) + if err != nil { + return err + } + + //migrate the device database to the final version + return tx.AutoMigrate(models.Device{}) + }, + }, }) if err := m.Migrate(); err != nil { From 1ced2198c7044061c96bf89413ee5e5343aac5f4 Mon Sep 17 00:00:00 2001 From: Jason Kulatunga Date: Wed, 4 May 2022 21:04:58 -0700 Subject: [PATCH 8/8] cleanup log messages. --- .../scrutiny_repository_device_smart_attributes.go | 7 ------- webapp/backend/pkg/models/measurements/smart.go | 13 +------------ 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go index 0479986..18960cb 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go @@ -32,8 +32,6 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) { // Get SMartResults from InfluxDB - fmt.Println("GetDeviceDetails from INFLUXDB") - //TODO: change the filter startrange to a real number. // Get parser flux query result @@ -45,18 +43,13 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn result, err := sr.influxQueryApi.Query(ctx, queryStr) if err == nil { - fmt.Println("GetDeviceDetails NO EROR") - // Use Next() to iterate over query result lines for result.Next() { - fmt.Println("GetDeviceDetails NEXT") - // Observe when there is new grouping key producing new table if result.TableChanged() { //fmt.Printf("table: %s\n", result.TableMetadata().String()) } - fmt.Printf("DECODINIG TABLE VALUES: %v", result.Record().Values()) smartData, err := measurements.NewSmartFromInfluxDB(result.Record().Values()) if err != nil { return nil, err diff --git a/webapp/backend/pkg/models/measurements/smart.go b/webapp/backend/pkg/models/measurements/smart.go index cc25593..c0523b1 100644 --- a/webapp/backend/pkg/models/measurements/smart.go +++ b/webapp/backend/pkg/models/measurements/smart.go @@ -61,12 +61,7 @@ func NewSmartFromInfluxDB(attrs map[string]interface{}) (*Smart, error) { Attributes: map[string]SmartAttribute{}, } - log.Printf("Prefetched Smart: %v\n", sm) - - //two steps (because we dont know the for key, val := range attrs { - log.Printf("Found Attribute (%s = %v)\n", key, val) - switch key { case "temp": sm.Temp = val.(int64) @@ -100,13 +95,7 @@ func NewSmartFromInfluxDB(attrs map[string]interface{}) (*Smart, error) { } - log.Printf("########NUMBER OF ATTRIBUTES: %v", len(sm.Attributes)) - log.Printf("########SMART: %v", sm) - - //panic("ERROR HERE.") - - //log.Printf("Sm.Attributes: %v", sm.Attributes) - //log.Printf("sm.Attributes[attributeId]: %v", sm.Attributes[attributeId]) + log.Printf("Found Smart Device (%s) Attributes (%v)", sm.DeviceWWN, len(sm.Attributes)) return &sm, nil }