Added PRAGMA settings support when connecting to SQLITE db.

When a transaction cannot lock the database, because it is already locked by another one,
SQLite by default throws an error: database is locked. This behavior is usually not appropriate when
concurrent access is needed, typically when multiple processes write to the same database.
PRAGMA busy_timeout lets you set a timeout or a handler for these events. When setting a timeout,
SQLite will try the transaction multiple times within this timeout.
https://rsqlite.r-dbi.org/reference/sqlitesetbusyhandler

retrying for 30000 milliseconds, 30seconds - this would be unreasonable for a distributed multi-tenant application,
but should be fine for local usage.

added mechanism for global settings (PRAGMA and DB level instructions).

fixes #341
pull/351/head
Jason Kulatunga 2 years ago
parent 3f272b36d4
commit a1b0108503

@ -62,7 +62,20 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field
// Gorm/SQLite setup // Gorm/SQLite setup
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location")) globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
// When a transaction cannot lock the database, because it is already locked by another one,
// SQLite by default throws an error: database is locked. This behavior is usually not appropriate when
// concurrent access is needed, typically when multiple processes write to the same database.
// PRAGMA busy_timeout lets you set a timeout or a handler for these events. When setting a timeout,
// SQLite will try the transaction multiple times within this timeout.
// fixes #341
// https://rsqlite.r-dbi.org/reference/sqlitesetbusyhandler
// retrying for 30000 milliseconds, 30seconds - this would be unreasonable for a distributed multi-tenant application,
// but should be fine for local usage.
pragmaStr := sqlitePragmaString(map[string]string{
"busy_timeout": "30000",
})
database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")+pragmaStr), &gorm.Config{
//TODO: figure out how to log database queries again. //TODO: figure out how to log database queries again.
//Logger: logger //Logger: logger
DisableForeignKeyConstraintWhenMigrating: true, DisableForeignKeyConstraintWhenMigrating: true,
@ -450,3 +463,16 @@ func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []str
} }
return []string{DURATION_KEY_WEEK} return []string{DURATION_KEY_WEEK}
} }
func sqlitePragmaString(pragmas map[string]string) string {
q := url.Values{}
for key, val := range pragmas {
q.Add("_pragma", key+"="+val)
}
queryStr := q.Encode()
if len(queryStr) > 0 {
return "?" + queryStr
}
return ""
}

@ -355,6 +355,30 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
return err return err
} }
sr.logger.Infoln("Database migration completed successfully") sr.logger.Infoln("Database migration completed successfully")
//these migrations cannot be done within a transaction, so they are done as a separate group, with `UseTransaction = false`
sr.logger.Infoln("SQLite global configuration migrations starting. Please wait....")
globalMigrateOptions := gormigrate.DefaultOptions
globalMigrateOptions.UseTransaction = false
gm := gormigrate.New(sr.gormClient, globalMigrateOptions, []*gormigrate.Migration{
{
ID: "g20220802211500",
Migrate: func(tx *gorm.DB) error {
//shrink the Database (maybe necessary after 20220503113100)
if err := tx.Exec("VACUUM;").Error; err != nil {
return err
}
return nil
},
},
})
if err := gm.Migrate(); err != nil {
sr.logger.Errorf("SQLite global configuration migrations failed with error. \n Please open a github issue at https://github.com/AnalogJ/scrutiny and attach a copy of your scrutiny.db file. \n %v", err)
return err
}
sr.logger.Infoln("SQLite global configuration migrations completed successfully")
return nil return nil
} }

Loading…
Cancel
Save