From bc2e9cffdad12fb6bd7a6cb4e90d3b38fc238a4a Mon Sep 17 00:00:00 2001 From: Harvey Tindall Date: Thu, 21 Dec 2023 18:17:03 +0000 Subject: [PATCH] backups: move code to own files --- api-backups.go | 117 ++++++++++++++++++++++++++++ api.go | 109 -------------------------- backups.go | 173 ++++++++++++++++++++++++++++++++++++++++++ daemon.go | 168 ---------------------------------------- html/admin.html | 1 + lang/admin/en-us.json | 1 + main.go | 3 + 7 files changed, 295 insertions(+), 277 deletions(-) create mode 100644 api-backups.go create mode 100644 backups.go diff --git a/api-backups.go b/api-backups.go new file mode 100644 index 0000000..5010e51 --- /dev/null +++ b/api-backups.go @@ -0,0 +1,117 @@ +package main + +import ( + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/gin-gonic/gin" +) + +// @Summary Creates a backup of the database. +// @Router /backups [post] +// @Success 200 {object} CreateBackupDTO +// @Security Bearer +// @tags Backups +func (app *appContext) CreateBackup(gc *gin.Context) { + backup := app.makeBackup() + gc.JSON(200, backup) +} + +// @Summary Download a specific backup file. Requires auth, so can't be accessed plainly in the browser. +// @Param fname path string true "backup filename" +// @Router /backups/{fname} [get] +// @Produce octet-stream +// @Produce json +// @Success 200 {body} file +// @Failure 400 {object} boolResponse +// @Security Bearer +// @tags Backups +func (app *appContext) GetBackup(gc *gin.Context) { + fname := gc.Param("fname") + // Hopefully this is enough to ensure the path isn't malicious. Hidden behind bearer auth anyway so shouldn't matter too much I guess. + ok := strings.HasPrefix(fname, BACKUP_PREFIX) && strings.HasSuffix(fname, BACKUP_SUFFIX) + t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(fname, BACKUP_PREFIX), BACKUP_SUFFIX)) + if !ok || err != nil || t.IsZero() { + app.debug.Printf("Ignoring backup DL request due to fname: %v\n", err) + respondBool(400, false, gc) + return + } + path := app.config.Section("backups").Key("path").String() + fullpath := filepath.Join(path, fname) + gc.FileAttachment(fullpath, fname) +} + +// @Summary Get a list of backups. +// @Router /backups [get] +// @Produce json +// @Success 200 {object} GetBackupsDTO +// @Security Bearer +// @tags Backups +func (app *appContext) GetBackups(gc *gin.Context) { + path := app.config.Section("backups").Key("path").String() + backups := app.getBackups() + sort.Sort(backups) + resp := GetBackupsDTO{} + resp.Backups = make([]CreateBackupDTO, backups.count) + + for i, item := range backups.files[:backups.count] { + resp.Backups[i].Name = item.Name() + fullpath := filepath.Join(path, item.Name()) + resp.Backups[i].Path = fullpath + resp.Backups[i].Date = backups.dates[i].Unix() + fstat, err := os.Stat(fullpath) + if err == nil { + resp.Backups[i].Size = fileSize(fstat.Size()) + } + } + gc.JSON(200, resp) +} + +// @Summary Restore a backup file stored locally to the server. +// @Param fname path string true "backup filename" +// @Router /backups/restore/{fname} [post] +// @Produce json +// @Failure 400 {object} boolResponse +// @Security Bearer +// @tags Backups +func (app *appContext) RestoreLocalBackup(gc *gin.Context) { + fname := gc.Param("fname") + // Hopefully this is enough to ensure the path isn't malicious. Hidden behind bearer auth anyway so shouldn't matter too much I guess. + ok := strings.HasPrefix(fname, BACKUP_PREFIX) && strings.HasSuffix(fname, BACKUP_SUFFIX) + t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(fname, BACKUP_PREFIX), BACKUP_SUFFIX)) + if !ok || err != nil || t.IsZero() { + app.debug.Printf("Ignoring backup DL request due to fname: %v\n", err) + respondBool(400, false, gc) + return + } + path := app.config.Section("backups").Key("path").String() + fullpath := filepath.Join(path, fname) + LOADBAK = fullpath + app.restart(gc) +} + +// @Summary Restore a backup file uploaded by the user. +// @Param file formData file true ".bak file" +// @Router /backups/restore [post] +// @Produce json +// @Failure 400 {object} boolResponse +// @Security Bearer +// @tags Backups +func (app *appContext) RestoreBackup(gc *gin.Context) { + file, err := gc.FormFile("backups-file") + if err != nil { + app.err.Printf("Failed to get file from form data: %v\n", err) + respondBool(400, false, gc) + return + } + app.debug.Printf("Got uploaded file \"%s\"\n", file.Filename) + path := app.config.Section("backups").Key("path").String() + fullpath := filepath.Join(path, "jfa-go-upload-bak-"+time.Now().Local().Format(BACKUP_DATEFMT)+BACKUP_SUFFIX) + gc.SaveUploadedFile(file, fullpath) + app.debug.Printf("Saved to \"%s\"\n", fullpath) + LOADBAK = fullpath + app.restart(gc) +} diff --git a/api.go b/api.go index 9ff95c8..1b926e0 100644 --- a/api.go +++ b/api.go @@ -1,9 +1,6 @@ package main import ( - "os" - "path/filepath" - "sort" "strings" "time" @@ -548,109 +545,3 @@ func (app *appContext) Restart() error { time.Sleep(time.Second) return nil } - -// @Summary Creates a backup of the database. -// @Router /backups [post] -// @Success 200 {object} CreateBackupDTO -// @Security Bearer -// @tags Other -func (app *appContext) CreateBackup(gc *gin.Context) { - backup := app.makeBackup() - gc.JSON(200, backup) -} - -// @Summary Download a specific backup file. Requires auth, so can't be accessed plainly in the browser. -// @Param fname path string true "backup filename" -// @Router /backups/{fname} [get] -// @Produce octet-stream -// @Produce json -// @Success 200 {body} file -// @Failure 400 {object} boolResponse -// @Security Bearer -// @tags Other -func (app *appContext) GetBackup(gc *gin.Context) { - fname := gc.Param("fname") - // Hopefully this is enough to ensure the path isn't malicious. Hidden behind bearer auth anyway so shouldn't matter too much I guess. - ok := strings.HasPrefix(fname, BACKUP_PREFIX) && strings.HasSuffix(fname, BACKUP_SUFFIX) - t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(fname, BACKUP_PREFIX), BACKUP_SUFFIX)) - if !ok || err != nil || t.IsZero() { - app.debug.Printf("Ignoring backup DL request due to fname: %v\n", err) - respondBool(400, false, gc) - return - } - path := app.config.Section("backups").Key("path").String() - fullpath := filepath.Join(path, fname) - gc.FileAttachment(fullpath, fname) -} - -// @Summary Get a list of backups. -// @Router /backups [get] -// @Produce json -// @Success 200 {object} GetBackupsDTO -// @Security Bearer -// @tags Other -func (app *appContext) GetBackups(gc *gin.Context) { - path := app.config.Section("backups").Key("path").String() - backups := app.getBackups() - sort.Sort(backups) - resp := GetBackupsDTO{} - resp.Backups = make([]CreateBackupDTO, backups.count) - - for i, item := range backups.files[:backups.count] { - resp.Backups[i].Name = item.Name() - fullpath := filepath.Join(path, item.Name()) - resp.Backups[i].Path = fullpath - resp.Backups[i].Date = backups.dates[i].Unix() - fstat, err := os.Stat(fullpath) - if err == nil { - resp.Backups[i].Size = fileSize(fstat.Size()) - } - } - gc.JSON(200, resp) -} - -// @Summary Restore a backup file stored locally to the server. -// @Param fname path string true "backup filename" -// @Router /backups/restore/{fname} [post] -// @Produce json -// @Failure 400 {object} boolResponse -// @Security Bearer -// @tags Other -func (app *appContext) RestoreLocalBackup(gc *gin.Context) { - fname := gc.Param("fname") - // Hopefully this is enough to ensure the path isn't malicious. Hidden behind bearer auth anyway so shouldn't matter too much I guess. - ok := strings.HasPrefix(fname, BACKUP_PREFIX) && strings.HasSuffix(fname, BACKUP_SUFFIX) - t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(fname, BACKUP_PREFIX), BACKUP_SUFFIX)) - if !ok || err != nil || t.IsZero() { - app.debug.Printf("Ignoring backup DL request due to fname: %v\n", err) - respondBool(400, false, gc) - return - } - path := app.config.Section("backups").Key("path").String() - fullpath := filepath.Join(path, fname) - LOADBAK = fullpath - app.restart(gc) -} - -// @Summary Restore a backup file uploaded by the user. -// @Param file formData file true ".bak file" -// @Router /backups/restore [post] -// @Produce json -// @Failure 400 {object} boolResponse -// @Security Bearer -// @tags Other -func (app *appContext) RestoreBackup(gc *gin.Context) { - file, err := gc.FormFile("backups-file") - if err != nil { - app.err.Printf("Failed to get file from form data: %v\n", err) - respondBool(400, false, gc) - return - } - app.debug.Printf("Got uploaded file \"%s\"\n", file.Filename) - path := app.config.Section("backups").Key("path").String() - fullpath := filepath.Join(path, "jfa-go-upload-bak-"+time.Now().Local().Format(BACKUP_DATEFMT)+BACKUP_SUFFIX) - gc.SaveUploadedFile(file, fullpath) - app.debug.Printf("Saved to \"%s\"\n", fullpath) - LOADBAK = fullpath - app.restart(gc) -} diff --git a/backups.go b/backups.go new file mode 100644 index 0000000..139e311 --- /dev/null +++ b/backups.go @@ -0,0 +1,173 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +type BackupList struct { + files []os.DirEntry + dates []time.Time + count int +} + +func (bl BackupList) Len() int { return len(bl.files) } +func (bl BackupList) Swap(i, j int) { + bl.files[i], bl.files[j] = bl.files[j], bl.files[i] + bl.dates[i], bl.dates[j] = bl.dates[j], bl.dates[i] +} + +func (bl BackupList) Less(i, j int) bool { + // Push non-backup files to the end of the array, + // Since they didn't have a date parsed. + if bl.dates[i].IsZero() { + return false + } + if bl.dates[j].IsZero() { + return true + } + // Sort by oldest first + return bl.dates[j].After(bl.dates[i]) +} + +// Get human-readable file size from f.Size() result. +// https://programming.guide/go/formatting-byte-size-to-human-readable-format.html +func fileSize(l int64) string { + const unit = 1000 + if l < unit { + return fmt.Sprintf("%dB", l) + } + div, exp := int64(unit), 0 + for n := l / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%c", float64(l)/float64(div), "KMGTPE"[exp]) +} + +func (app *appContext) getBackups() *BackupList { + path := app.config.Section("backups").Key("path").String() + err := os.MkdirAll(path, 0755) + if err != nil { + app.err.Printf("Failed to create backup directory \"%s\": %v\n", path, err) + return nil + } + items, err := os.ReadDir(path) + if err != nil { + app.err.Printf("Failed to read backup directory \"%s\": %v\n", path, err) + return nil + } + backups := &BackupList{} + backups.files = items + backups.dates = make([]time.Time, len(items)) + backups.count = 0 + for i, item := range items { + if item.IsDir() || !(strings.HasSuffix(item.Name(), BACKUP_SUFFIX)) { + continue + } + t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(item.Name(), BACKUP_PREFIX), BACKUP_SUFFIX)) + if err != nil { + app.debug.Printf("Failed to parse backup filename \"%s\": %v\n", item.Name(), err) + continue + } + backups.dates[i] = t + backups.count++ + } + return backups +} + +func (app *appContext) makeBackup() (fileDetails CreateBackupDTO) { + toKeep := app.config.Section("backups").Key("keep_n_backups").MustInt(20) + fname := BACKUP_PREFIX + time.Now().Local().Format(BACKUP_DATEFMT) + BACKUP_SUFFIX + path := app.config.Section("backups").Key("path").String() + backups := app.getBackups() + if backups == nil { + return + } + toDelete := backups.count + 1 - toKeep + // fmt.Printf("toDelete: %d, backCount: %d, keep: %d, length: %d\n", toDelete, backups.count, toKeep, len(backups.files)) + if toDelete > 0 && toDelete <= backups.count { + sort.Sort(backups) + for _, item := range backups.files[:toDelete] { + fullpath := filepath.Join(path, item.Name()) + app.debug.Printf("Deleting old backup \"%s\"\n", item.Name()) + err := os.Remove(fullpath) + if err != nil { + app.err.Printf("Failed to delete old backup \"%s\": %v\n", fullpath, err) + return + } + } + } + fullpath := filepath.Join(path, fname) + f, err := os.Create(fullpath) + if err != nil { + app.err.Printf("Failed to open backup file \"%s\": %v\n", fullpath, err) + return + } + defer f.Close() + _, err = app.storage.db.Badger().Backup(f, 0) + if err != nil { + app.err.Printf("Failed to create backup: %v\n", err) + return + } + + fstat, err := f.Stat() + if err != nil { + app.err.Printf("Failed to get info on new backup: %v\n", err) + return + } + fileDetails.Size = fileSize(fstat.Size()) + fileDetails.Name = fname + fileDetails.Path = fullpath + // fmt.Printf("Created backup %+v\n", fileDetails) + return +} + +func (app *appContext) loadPendingBackup() { + if LOADBAK == "" { + return + } + oldPath := filepath.Join(app.dataPath, "db-"+string(time.Now().Unix())+"-pre-"+filepath.Base(LOADBAK)) + app.info.Printf("Moving existing database to \"%s\"\n", oldPath) + err := os.Rename(app.storage.db_path, oldPath) + if err != nil { + app.err.Fatalf("Failed to move existing database: %v\n", err) + } + + app.ConnectDB() + defer app.storage.db.Close() + + f, err := os.Open(LOADBAK) + if err != nil { + app.err.Fatalf("Failed to open backup file \"%s\": %v\n", LOADBAK, err) + } + err = app.storage.db.Badger().Load(f, 256) + f.Close() + if err != nil { + app.err.Fatalf("Failed to restore backup file \"%s\": %v\n", LOADBAK, err) + } + app.info.Printf("Restored backup \"%s\".", LOADBAK) + LOADBAK = "" +} + +func newBackupDaemon(app *appContext) *housekeepingDaemon { + interval := time.Duration(app.config.Section("backups").Key("every_n_minutes").MustInt(1440)) * time.Minute + daemon := housekeepingDaemon{ + Stopped: false, + ShutdownChannel: make(chan string), + Interval: interval, + period: interval, + app: app, + } + daemon.jobs = []func(app *appContext){ + func(app *appContext) { + app.debug.Println("Backups: Creating backup") + app.makeBackup() + }, + } + return &daemon +} diff --git a/daemon.go b/daemon.go index 5c9b3f2..9dc8c7d 100644 --- a/daemon.go +++ b/daemon.go @@ -1,11 +1,6 @@ package main import ( - "fmt" - "os" - "path/filepath" - "sort" - "strings" "time" "github.com/dgraph-io/badger/v3" @@ -85,151 +80,6 @@ func (app *appContext) clearTelegram() { } } -type BackupList struct { - files []os.DirEntry - dates []time.Time - count int -} - -func (bl BackupList) Len() int { return len(bl.files) } -func (bl BackupList) Swap(i, j int) { - bl.files[i], bl.files[j] = bl.files[j], bl.files[i] - bl.dates[i], bl.dates[j] = bl.dates[j], bl.dates[i] -} - -func (bl BackupList) Less(i, j int) bool { - // Push non-backup files to the end of the array, - // Since they didn't have a date parsed. - if bl.dates[i].IsZero() { - return false - } - if bl.dates[j].IsZero() { - return true - } - // Sort by oldest first - return bl.dates[j].After(bl.dates[i]) -} - -// Get human-readable file size from f.Size() result. -// https://programming.guide/go/formatting-byte-size-to-human-readable-format.html -func fileSize(l int64) string { - const unit = 1000 - if l < unit { - return fmt.Sprintf("%dB", l) - } - div, exp := int64(unit), 0 - for n := l / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f%c", float64(l)/float64(div), "KMGTPE"[exp]) -} - -func (app *appContext) getBackups() *BackupList { - path := app.config.Section("backups").Key("path").String() - err := os.MkdirAll(path, 0755) - if err != nil { - app.err.Printf("Failed to create backup directory \"%s\": %v\n", path, err) - return nil - } - items, err := os.ReadDir(path) - if err != nil { - app.err.Printf("Failed to read backup directory \"%s\": %v\n", path, err) - return nil - } - backups := &BackupList{} - backups.files = items - backups.dates = make([]time.Time, len(items)) - backups.count = 0 - for i, item := range items { - if item.IsDir() || !(strings.HasSuffix(item.Name(), BACKUP_SUFFIX)) { - continue - } - t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(item.Name(), BACKUP_PREFIX), BACKUP_SUFFIX)) - if err != nil { - app.debug.Printf("Failed to parse backup filename \"%s\": %v\n", item.Name(), err) - continue - } - backups.dates[i] = t - backups.count++ - } - return backups -} - -func (app *appContext) makeBackup() (fileDetails CreateBackupDTO) { - toKeep := app.config.Section("backups").Key("keep_n_backups").MustInt(20) - fname := BACKUP_PREFIX + time.Now().Local().Format(BACKUP_DATEFMT) + BACKUP_SUFFIX - path := app.config.Section("backups").Key("path").String() - backups := app.getBackups() - if backups == nil { - return - } - toDelete := backups.count + 1 - toKeep - // fmt.Printf("toDelete: %d, backCount: %d, keep: %d, length: %d\n", toDelete, backups.count, toKeep, len(backups.files)) - if toDelete > 0 && toDelete <= backups.count { - sort.Sort(backups) - for _, item := range backups.files[:toDelete] { - fullpath := filepath.Join(path, item.Name()) - app.debug.Printf("Deleting old backup \"%s\"\n", item.Name()) - err := os.Remove(fullpath) - if err != nil { - app.err.Printf("Failed to delete old backup \"%s\": %v\n", fullpath, err) - return - } - } - } - fullpath := filepath.Join(path, fname) - f, err := os.Create(fullpath) - if err != nil { - app.err.Printf("Failed to open backup file \"%s\": %v\n", fullpath, err) - return - } - defer f.Close() - _, err = app.storage.db.Badger().Backup(f, 0) - if err != nil { - app.err.Printf("Failed to create backup: %v\n", err) - return - } - - fstat, err := f.Stat() - if err != nil { - app.err.Printf("Failed to get info on new backup: %v\n", err) - return - } - fileDetails.Size = fileSize(fstat.Size()) - fileDetails.Name = fname - fileDetails.Path = fullpath - // fmt.Printf("Created backup %+v\n", fileDetails) - return -} - -func (app *appContext) loadPendingBackup() { - if LOADBAK == "" { - return - } - oldPath := filepath.Join(app.dataPath, "db-"+string(time.Now().Unix())+"-pre-"+filepath.Base(LOADBAK)) - app.info.Printf("Moving existing database to \"%s\"\n", oldPath) - err := os.Rename(app.storage.db_path, oldPath) - if err != nil { - app.err.Fatalf("Failed to move existing database: %v\n", err) - } - - app.ConnectDB() - defer app.storage.db.Close() - - f, err := os.Open(LOADBAK) - if err != nil { - app.err.Fatalf("Failed to open backup file \"%s\": %v\n", LOADBAK, err) - } - err = app.storage.db.Badger().Load(f, 256) - f.Close() - if err != nil { - app.err.Fatalf("Failed to restore backup file \"%s\": %v\n", LOADBAK, err) - } - app.info.Printf("Restored backup \"%s\".", LOADBAK) - LOADBAK = "" -} - func (app *appContext) clearActivities() { app.debug.Println("Housekeeping: Cleaning up Activity log...") keepCount := app.config.Section("activity_log").Key("keep_n_records").MustInt(1000) @@ -272,24 +122,6 @@ type housekeepingDaemon struct { app *appContext } -func newBackupDaemon(app *appContext) *housekeepingDaemon { - interval := time.Duration(app.config.Section("backups").Key("every_n_minutes").MustInt(1440)) * time.Minute - daemon := housekeepingDaemon{ - Stopped: false, - ShutdownChannel: make(chan string), - Interval: interval, - period: interval, - app: app, - } - daemon.jobs = []func(app *appContext){ - func(app *appContext) { - app.debug.Println("Backups: Creating backup") - app.makeBackup() - }, - } - return &daemon -} - func newInviteDaemon(interval time.Duration, app *appContext) *housekeepingDaemon { daemon := housekeepingDaemon{ Stopped: false, diff --git a/html/admin.html b/html/admin.html index 1a9486a..6dc6ab9 100644 --- a/html/admin.html +++ b/html/admin.html @@ -332,6 +332,7 @@
{{ .strings.backups }} ×

{{ .strings.backupsDescription }}

+

{{ .strings.backupsCopy }}

{{ .strings.backupsFormatNote }}

diff --git a/lang/admin/en-us.json b/lang/admin/en-us.json index d4580fd..a625c1c 100644 --- a/lang/admin/en-us.json +++ b/lang/admin/en-us.json @@ -184,6 +184,7 @@ "backups": "Backups", "backupsDescription": "Backups of the database can be made, restored, or downloaded from here.", "backupsFormatNote": "Only backup files with the standard name format will be shown here. To use any other, upload the backup manually.", + "backupsCopy": "When applying a backup, a copy of the original \"db\" folder will be made next to it, in case anything goes wrong.", "backupDownloadRestore": "Download / Restore", "backupUpload": "Upload backup", "backupDownload": "Download backup", diff --git a/main.go b/main.go index 52ca32f..52c96c8 100644 --- a/main.go +++ b/main.go @@ -658,6 +658,9 @@ func flagPassed(name string) (found bool) { // @tag.name Ombi // @tag.description Ombi related operations. +// @tag.name Backups +// @tag.description Database backup/restore operations. + // @tag.name Other // @tag.description Things that dont fit elsewhere.