backups: move code to own files

backups
Harvey Tindall 11 months ago
parent ade032241a
commit bc2e9cffda
No known key found for this signature in database
GPG Key ID: BBC65952848FB1A2

@ -0,0 +1,117 @@
package main
import (
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/gin-gonic/gin"
)
// @Summary Creates a backup of the database.
// @Router /backups [post]
// @Success 200 {object} CreateBackupDTO
// @Security Bearer
// @tags Backups
func (app *appContext) CreateBackup(gc *gin.Context) {
backup := app.makeBackup()
gc.JSON(200, backup)
}
// @Summary Download a specific backup file. Requires auth, so can't be accessed plainly in the browser.
// @Param fname path string true "backup filename"
// @Router /backups/{fname} [get]
// @Produce octet-stream
// @Produce json
// @Success 200 {body} file
// @Failure 400 {object} boolResponse
// @Security Bearer
// @tags Backups
func (app *appContext) GetBackup(gc *gin.Context) {
fname := gc.Param("fname")
// Hopefully this is enough to ensure the path isn't malicious. Hidden behind bearer auth anyway so shouldn't matter too much I guess.
ok := strings.HasPrefix(fname, BACKUP_PREFIX) && strings.HasSuffix(fname, BACKUP_SUFFIX)
t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(fname, BACKUP_PREFIX), BACKUP_SUFFIX))
if !ok || err != nil || t.IsZero() {
app.debug.Printf("Ignoring backup DL request due to fname: %v\n", err)
respondBool(400, false, gc)
return
}
path := app.config.Section("backups").Key("path").String()
fullpath := filepath.Join(path, fname)
gc.FileAttachment(fullpath, fname)
}
// @Summary Get a list of backups.
// @Router /backups [get]
// @Produce json
// @Success 200 {object} GetBackupsDTO
// @Security Bearer
// @tags Backups
func (app *appContext) GetBackups(gc *gin.Context) {
path := app.config.Section("backups").Key("path").String()
backups := app.getBackups()
sort.Sort(backups)
resp := GetBackupsDTO{}
resp.Backups = make([]CreateBackupDTO, backups.count)
for i, item := range backups.files[:backups.count] {
resp.Backups[i].Name = item.Name()
fullpath := filepath.Join(path, item.Name())
resp.Backups[i].Path = fullpath
resp.Backups[i].Date = backups.dates[i].Unix()
fstat, err := os.Stat(fullpath)
if err == nil {
resp.Backups[i].Size = fileSize(fstat.Size())
}
}
gc.JSON(200, resp)
}
// @Summary Restore a backup file stored locally to the server.
// @Param fname path string true "backup filename"
// @Router /backups/restore/{fname} [post]
// @Produce json
// @Failure 400 {object} boolResponse
// @Security Bearer
// @tags Backups
func (app *appContext) RestoreLocalBackup(gc *gin.Context) {
fname := gc.Param("fname")
// Hopefully this is enough to ensure the path isn't malicious. Hidden behind bearer auth anyway so shouldn't matter too much I guess.
ok := strings.HasPrefix(fname, BACKUP_PREFIX) && strings.HasSuffix(fname, BACKUP_SUFFIX)
t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(fname, BACKUP_PREFIX), BACKUP_SUFFIX))
if !ok || err != nil || t.IsZero() {
app.debug.Printf("Ignoring backup DL request due to fname: %v\n", err)
respondBool(400, false, gc)
return
}
path := app.config.Section("backups").Key("path").String()
fullpath := filepath.Join(path, fname)
LOADBAK = fullpath
app.restart(gc)
}
// @Summary Restore a backup file uploaded by the user.
// @Param file formData file true ".bak file"
// @Router /backups/restore [post]
// @Produce json
// @Failure 400 {object} boolResponse
// @Security Bearer
// @tags Backups
func (app *appContext) RestoreBackup(gc *gin.Context) {
file, err := gc.FormFile("backups-file")
if err != nil {
app.err.Printf("Failed to get file from form data: %v\n", err)
respondBool(400, false, gc)
return
}
app.debug.Printf("Got uploaded file \"%s\"\n", file.Filename)
path := app.config.Section("backups").Key("path").String()
fullpath := filepath.Join(path, "jfa-go-upload-bak-"+time.Now().Local().Format(BACKUP_DATEFMT)+BACKUP_SUFFIX)
gc.SaveUploadedFile(file, fullpath)
app.debug.Printf("Saved to \"%s\"\n", fullpath)
LOADBAK = fullpath
app.restart(gc)
}

109
api.go

@ -1,9 +1,6 @@
package main
import (
"os"
"path/filepath"
"sort"
"strings"
"time"
@ -548,109 +545,3 @@ func (app *appContext) Restart() error {
time.Sleep(time.Second)
return nil
}
// @Summary Creates a backup of the database.
// @Router /backups [post]
// @Success 200 {object} CreateBackupDTO
// @Security Bearer
// @tags Other
func (app *appContext) CreateBackup(gc *gin.Context) {
backup := app.makeBackup()
gc.JSON(200, backup)
}
// @Summary Download a specific backup file. Requires auth, so can't be accessed plainly in the browser.
// @Param fname path string true "backup filename"
// @Router /backups/{fname} [get]
// @Produce octet-stream
// @Produce json
// @Success 200 {body} file
// @Failure 400 {object} boolResponse
// @Security Bearer
// @tags Other
func (app *appContext) GetBackup(gc *gin.Context) {
fname := gc.Param("fname")
// Hopefully this is enough to ensure the path isn't malicious. Hidden behind bearer auth anyway so shouldn't matter too much I guess.
ok := strings.HasPrefix(fname, BACKUP_PREFIX) && strings.HasSuffix(fname, BACKUP_SUFFIX)
t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(fname, BACKUP_PREFIX), BACKUP_SUFFIX))
if !ok || err != nil || t.IsZero() {
app.debug.Printf("Ignoring backup DL request due to fname: %v\n", err)
respondBool(400, false, gc)
return
}
path := app.config.Section("backups").Key("path").String()
fullpath := filepath.Join(path, fname)
gc.FileAttachment(fullpath, fname)
}
// @Summary Get a list of backups.
// @Router /backups [get]
// @Produce json
// @Success 200 {object} GetBackupsDTO
// @Security Bearer
// @tags Other
func (app *appContext) GetBackups(gc *gin.Context) {
path := app.config.Section("backups").Key("path").String()
backups := app.getBackups()
sort.Sort(backups)
resp := GetBackupsDTO{}
resp.Backups = make([]CreateBackupDTO, backups.count)
for i, item := range backups.files[:backups.count] {
resp.Backups[i].Name = item.Name()
fullpath := filepath.Join(path, item.Name())
resp.Backups[i].Path = fullpath
resp.Backups[i].Date = backups.dates[i].Unix()
fstat, err := os.Stat(fullpath)
if err == nil {
resp.Backups[i].Size = fileSize(fstat.Size())
}
}
gc.JSON(200, resp)
}
// @Summary Restore a backup file stored locally to the server.
// @Param fname path string true "backup filename"
// @Router /backups/restore/{fname} [post]
// @Produce json
// @Failure 400 {object} boolResponse
// @Security Bearer
// @tags Other
func (app *appContext) RestoreLocalBackup(gc *gin.Context) {
fname := gc.Param("fname")
// Hopefully this is enough to ensure the path isn't malicious. Hidden behind bearer auth anyway so shouldn't matter too much I guess.
ok := strings.HasPrefix(fname, BACKUP_PREFIX) && strings.HasSuffix(fname, BACKUP_SUFFIX)
t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(fname, BACKUP_PREFIX), BACKUP_SUFFIX))
if !ok || err != nil || t.IsZero() {
app.debug.Printf("Ignoring backup DL request due to fname: %v\n", err)
respondBool(400, false, gc)
return
}
path := app.config.Section("backups").Key("path").String()
fullpath := filepath.Join(path, fname)
LOADBAK = fullpath
app.restart(gc)
}
// @Summary Restore a backup file uploaded by the user.
// @Param file formData file true ".bak file"
// @Router /backups/restore [post]
// @Produce json
// @Failure 400 {object} boolResponse
// @Security Bearer
// @tags Other
func (app *appContext) RestoreBackup(gc *gin.Context) {
file, err := gc.FormFile("backups-file")
if err != nil {
app.err.Printf("Failed to get file from form data: %v\n", err)
respondBool(400, false, gc)
return
}
app.debug.Printf("Got uploaded file \"%s\"\n", file.Filename)
path := app.config.Section("backups").Key("path").String()
fullpath := filepath.Join(path, "jfa-go-upload-bak-"+time.Now().Local().Format(BACKUP_DATEFMT)+BACKUP_SUFFIX)
gc.SaveUploadedFile(file, fullpath)
app.debug.Printf("Saved to \"%s\"\n", fullpath)
LOADBAK = fullpath
app.restart(gc)
}

@ -0,0 +1,173 @@
package main
import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
type BackupList struct {
files []os.DirEntry
dates []time.Time
count int
}
func (bl BackupList) Len() int { return len(bl.files) }
func (bl BackupList) Swap(i, j int) {
bl.files[i], bl.files[j] = bl.files[j], bl.files[i]
bl.dates[i], bl.dates[j] = bl.dates[j], bl.dates[i]
}
func (bl BackupList) Less(i, j int) bool {
// Push non-backup files to the end of the array,
// Since they didn't have a date parsed.
if bl.dates[i].IsZero() {
return false
}
if bl.dates[j].IsZero() {
return true
}
// Sort by oldest first
return bl.dates[j].After(bl.dates[i])
}
// Get human-readable file size from f.Size() result.
// https://programming.guide/go/formatting-byte-size-to-human-readable-format.html
func fileSize(l int64) string {
const unit = 1000
if l < unit {
return fmt.Sprintf("%dB", l)
}
div, exp := int64(unit), 0
for n := l / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f%c", float64(l)/float64(div), "KMGTPE"[exp])
}
func (app *appContext) getBackups() *BackupList {
path := app.config.Section("backups").Key("path").String()
err := os.MkdirAll(path, 0755)
if err != nil {
app.err.Printf("Failed to create backup directory \"%s\": %v\n", path, err)
return nil
}
items, err := os.ReadDir(path)
if err != nil {
app.err.Printf("Failed to read backup directory \"%s\": %v\n", path, err)
return nil
}
backups := &BackupList{}
backups.files = items
backups.dates = make([]time.Time, len(items))
backups.count = 0
for i, item := range items {
if item.IsDir() || !(strings.HasSuffix(item.Name(), BACKUP_SUFFIX)) {
continue
}
t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(item.Name(), BACKUP_PREFIX), BACKUP_SUFFIX))
if err != nil {
app.debug.Printf("Failed to parse backup filename \"%s\": %v\n", item.Name(), err)
continue
}
backups.dates[i] = t
backups.count++
}
return backups
}
func (app *appContext) makeBackup() (fileDetails CreateBackupDTO) {
toKeep := app.config.Section("backups").Key("keep_n_backups").MustInt(20)
fname := BACKUP_PREFIX + time.Now().Local().Format(BACKUP_DATEFMT) + BACKUP_SUFFIX
path := app.config.Section("backups").Key("path").String()
backups := app.getBackups()
if backups == nil {
return
}
toDelete := backups.count + 1 - toKeep
// fmt.Printf("toDelete: %d, backCount: %d, keep: %d, length: %d\n", toDelete, backups.count, toKeep, len(backups.files))
if toDelete > 0 && toDelete <= backups.count {
sort.Sort(backups)
for _, item := range backups.files[:toDelete] {
fullpath := filepath.Join(path, item.Name())
app.debug.Printf("Deleting old backup \"%s\"\n", item.Name())
err := os.Remove(fullpath)
if err != nil {
app.err.Printf("Failed to delete old backup \"%s\": %v\n", fullpath, err)
return
}
}
}
fullpath := filepath.Join(path, fname)
f, err := os.Create(fullpath)
if err != nil {
app.err.Printf("Failed to open backup file \"%s\": %v\n", fullpath, err)
return
}
defer f.Close()
_, err = app.storage.db.Badger().Backup(f, 0)
if err != nil {
app.err.Printf("Failed to create backup: %v\n", err)
return
}
fstat, err := f.Stat()
if err != nil {
app.err.Printf("Failed to get info on new backup: %v\n", err)
return
}
fileDetails.Size = fileSize(fstat.Size())
fileDetails.Name = fname
fileDetails.Path = fullpath
// fmt.Printf("Created backup %+v\n", fileDetails)
return
}
func (app *appContext) loadPendingBackup() {
if LOADBAK == "" {
return
}
oldPath := filepath.Join(app.dataPath, "db-"+string(time.Now().Unix())+"-pre-"+filepath.Base(LOADBAK))
app.info.Printf("Moving existing database to \"%s\"\n", oldPath)
err := os.Rename(app.storage.db_path, oldPath)
if err != nil {
app.err.Fatalf("Failed to move existing database: %v\n", err)
}
app.ConnectDB()
defer app.storage.db.Close()
f, err := os.Open(LOADBAK)
if err != nil {
app.err.Fatalf("Failed to open backup file \"%s\": %v\n", LOADBAK, err)
}
err = app.storage.db.Badger().Load(f, 256)
f.Close()
if err != nil {
app.err.Fatalf("Failed to restore backup file \"%s\": %v\n", LOADBAK, err)
}
app.info.Printf("Restored backup \"%s\".", LOADBAK)
LOADBAK = ""
}
func newBackupDaemon(app *appContext) *housekeepingDaemon {
interval := time.Duration(app.config.Section("backups").Key("every_n_minutes").MustInt(1440)) * time.Minute
daemon := housekeepingDaemon{
Stopped: false,
ShutdownChannel: make(chan string),
Interval: interval,
period: interval,
app: app,
}
daemon.jobs = []func(app *appContext){
func(app *appContext) {
app.debug.Println("Backups: Creating backup")
app.makeBackup()
},
}
return &daemon
}

@ -1,11 +1,6 @@
package main
import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/dgraph-io/badger/v3"
@ -85,151 +80,6 @@ func (app *appContext) clearTelegram() {
}
}
type BackupList struct {
files []os.DirEntry
dates []time.Time
count int
}
func (bl BackupList) Len() int { return len(bl.files) }
func (bl BackupList) Swap(i, j int) {
bl.files[i], bl.files[j] = bl.files[j], bl.files[i]
bl.dates[i], bl.dates[j] = bl.dates[j], bl.dates[i]
}
func (bl BackupList) Less(i, j int) bool {
// Push non-backup files to the end of the array,
// Since they didn't have a date parsed.
if bl.dates[i].IsZero() {
return false
}
if bl.dates[j].IsZero() {
return true
}
// Sort by oldest first
return bl.dates[j].After(bl.dates[i])
}
// Get human-readable file size from f.Size() result.
// https://programming.guide/go/formatting-byte-size-to-human-readable-format.html
func fileSize(l int64) string {
const unit = 1000
if l < unit {
return fmt.Sprintf("%dB", l)
}
div, exp := int64(unit), 0
for n := l / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f%c", float64(l)/float64(div), "KMGTPE"[exp])
}
func (app *appContext) getBackups() *BackupList {
path := app.config.Section("backups").Key("path").String()
err := os.MkdirAll(path, 0755)
if err != nil {
app.err.Printf("Failed to create backup directory \"%s\": %v\n", path, err)
return nil
}
items, err := os.ReadDir(path)
if err != nil {
app.err.Printf("Failed to read backup directory \"%s\": %v\n", path, err)
return nil
}
backups := &BackupList{}
backups.files = items
backups.dates = make([]time.Time, len(items))
backups.count = 0
for i, item := range items {
if item.IsDir() || !(strings.HasSuffix(item.Name(), BACKUP_SUFFIX)) {
continue
}
t, err := time.Parse(BACKUP_DATEFMT, strings.TrimSuffix(strings.TrimPrefix(item.Name(), BACKUP_PREFIX), BACKUP_SUFFIX))
if err != nil {
app.debug.Printf("Failed to parse backup filename \"%s\": %v\n", item.Name(), err)
continue
}
backups.dates[i] = t
backups.count++
}
return backups
}
func (app *appContext) makeBackup() (fileDetails CreateBackupDTO) {
toKeep := app.config.Section("backups").Key("keep_n_backups").MustInt(20)
fname := BACKUP_PREFIX + time.Now().Local().Format(BACKUP_DATEFMT) + BACKUP_SUFFIX
path := app.config.Section("backups").Key("path").String()
backups := app.getBackups()
if backups == nil {
return
}
toDelete := backups.count + 1 - toKeep
// fmt.Printf("toDelete: %d, backCount: %d, keep: %d, length: %d\n", toDelete, backups.count, toKeep, len(backups.files))
if toDelete > 0 && toDelete <= backups.count {
sort.Sort(backups)
for _, item := range backups.files[:toDelete] {
fullpath := filepath.Join(path, item.Name())
app.debug.Printf("Deleting old backup \"%s\"\n", item.Name())
err := os.Remove(fullpath)
if err != nil {
app.err.Printf("Failed to delete old backup \"%s\": %v\n", fullpath, err)
return
}
}
}
fullpath := filepath.Join(path, fname)
f, err := os.Create(fullpath)
if err != nil {
app.err.Printf("Failed to open backup file \"%s\": %v\n", fullpath, err)
return
}
defer f.Close()
_, err = app.storage.db.Badger().Backup(f, 0)
if err != nil {
app.err.Printf("Failed to create backup: %v\n", err)
return
}
fstat, err := f.Stat()
if err != nil {
app.err.Printf("Failed to get info on new backup: %v\n", err)
return
}
fileDetails.Size = fileSize(fstat.Size())
fileDetails.Name = fname
fileDetails.Path = fullpath
// fmt.Printf("Created backup %+v\n", fileDetails)
return
}
func (app *appContext) loadPendingBackup() {
if LOADBAK == "" {
return
}
oldPath := filepath.Join(app.dataPath, "db-"+string(time.Now().Unix())+"-pre-"+filepath.Base(LOADBAK))
app.info.Printf("Moving existing database to \"%s\"\n", oldPath)
err := os.Rename(app.storage.db_path, oldPath)
if err != nil {
app.err.Fatalf("Failed to move existing database: %v\n", err)
}
app.ConnectDB()
defer app.storage.db.Close()
f, err := os.Open(LOADBAK)
if err != nil {
app.err.Fatalf("Failed to open backup file \"%s\": %v\n", LOADBAK, err)
}
err = app.storage.db.Badger().Load(f, 256)
f.Close()
if err != nil {
app.err.Fatalf("Failed to restore backup file \"%s\": %v\n", LOADBAK, err)
}
app.info.Printf("Restored backup \"%s\".", LOADBAK)
LOADBAK = ""
}
func (app *appContext) clearActivities() {
app.debug.Println("Housekeeping: Cleaning up Activity log...")
keepCount := app.config.Section("activity_log").Key("keep_n_records").MustInt(1000)
@ -272,24 +122,6 @@ type housekeepingDaemon struct {
app *appContext
}
func newBackupDaemon(app *appContext) *housekeepingDaemon {
interval := time.Duration(app.config.Section("backups").Key("every_n_minutes").MustInt(1440)) * time.Minute
daemon := housekeepingDaemon{
Stopped: false,
ShutdownChannel: make(chan string),
Interval: interval,
period: interval,
app: app,
}
daemon.jobs = []func(app *appContext){
func(app *appContext) {
app.debug.Println("Backups: Creating backup")
app.makeBackup()
},
}
return &daemon
}
func newInviteDaemon(interval time.Duration, app *appContext) *housekeepingDaemon {
daemon := housekeepingDaemon{
Stopped: false,

@ -332,6 +332,7 @@
<div class="card relative mx-auto my-[10%] w-4/5 lg:w-1/3">
<span class="heading">{{ .strings.backups }} <span class="modal-close">&times;</span></span>
<p class="content my-4">{{ .strings.backupsDescription }}</p>
<p class="content my-4">{{ .strings.backupsCopy }}</p>
<p class="content my-4">{{ .strings.backupsFormatNote }}</p>
<div class="row col flex">
<button class="button ~info @low mr-2 mt-4 mb-4" id="settings-backups-backup">{{ .strings.backupNow }}</button>

@ -184,6 +184,7 @@
"backups": "Backups",
"backupsDescription": "Backups of the database can be made, restored, or downloaded from here.",
"backupsFormatNote": "Only backup files with the standard name format will be shown here. To use any other, upload the backup manually.",
"backupsCopy": "When applying a backup, a copy of the original \"db\" folder will be made next to it, in case anything goes wrong.",
"backupDownloadRestore": "Download / Restore",
"backupUpload": "Upload backup",
"backupDownload": "Download backup",

@ -658,6 +658,9 @@ func flagPassed(name string) (found bool) {
// @tag.name Ombi
// @tag.description Ombi related operations.
// @tag.name Backups
// @tag.description Database backup/restore operations.
// @tag.name Other
// @tag.description Things that dont fit elsewhere.

Loading…
Cancel
Save