From df764ce8b4ed5c065cb592067c994b375fe10c5e Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 26 Sep 2021 12:27:50 -0500 Subject: [PATCH] New: Postgres Support --- frontend/src/System/Status/About/About.js | 19 +++-- .../Configuration/ConfigFileProvider.cs | 12 +++ .../Datastore/BasicRepository.cs | 25 +++++-- .../Datastore/ConnectionStringFactory.cs | 29 ++++++- src/NzbDrone.Core/Datastore/Database.cs | 45 ++++++++++- src/NzbDrone.Core/Datastore/DbFactory.cs | 18 ++++- .../Datastore/Extensions/BuilderExtensions.cs | 2 +- src/NzbDrone.Core/Datastore/LogDatabase.cs | 6 +- src/NzbDrone.Core/Datastore/MainDatabase.cs | 6 +- .../Migration/005_update_notifiarr.cs | 2 +- .../Datastore/Migration/007_history_failed.cs | 3 +- .../Datastore/Migration/008_redacted_api.cs | 4 +- .../Migration/013_desi_gazelle_to_unit3d.cs | 2 +- .../Framework/MigrationController.cs | 16 +++- src/NzbDrone.Core/Datastore/TableMapper.cs | 75 +++++++++++++++++-- .../Housekeepers/CleanupAdditionalUsers.cs | 8 +- .../CleanupOrphanedDownloadClientStatus.cs | 14 ++-- .../CleanupOrphanedHistoryItems.cs | 12 +-- .../CleanupOrphanedIndexerStatus.cs | 12 +-- .../Housekeepers/CleanupUnusedTags.cs | 13 +++- .../FixFutureRunScheduledTasks.cs | 8 +- .../Instrumentation/DatabaseTarget.cs | 73 +++++++++++++----- .../Messaging/Commands/CommandRepository.cs | 2 +- src/NzbDrone.Core/Prowlarr.Core.csproj | 2 + .../System/SystemController.cs | 3 +- 25 files changed, 320 insertions(+), 91 deletions(-) diff --git a/frontend/src/System/Status/About/About.js b/frontend/src/System/Status/About/About.js index d69d0b5d0..43b0deb9b 100644 --- a/frontend/src/System/Status/About/About.js +++ b/frontend/src/System/Status/About/About.js @@ -20,10 +20,11 @@ class About extends Component { packageVersion, packageAuthor, isNetCore, - isMono, isDocker, runtimeVersion, migrationVersion, + databaseVersion, + databaseType, appData, startupPath, mode, @@ -48,14 +49,6 @@ class About extends Component { /> } - { - isMono && - - } - { isNetCore && + + GetValue("LogLevel", "info").ToLowerInvariant(); public string ConsoleLogLevel => GetValue("ConsoleLogLevel", string.Empty, persist: false); + public string PostgresHost => GetValue("PostgresHost", string.Empty, persist: false); + public string PostgresUser => GetValue("PostgresUser", string.Empty, persist: false); + public string PostgresPassword => GetValue("PostgresPassword", string.Empty, persist: false); + public string PostgresMainDb => GetValue("PostgresMainDb", "prowlarr-main", persist: false); + public string PostgresLogDb => GetValue("PostgresLogDb", "prowlarr-log", persist: false); + public int PostgresPort => GetValueInt("PostgresPort", 5436, persist: false); public bool LogSql => GetValueBoolean("LogSql", false, persist: false); public int LogRotate => GetValueInt("LogRotate", 50, persist: false); public bool FilterSentryEvents => GetValueBoolean("FilterSentryEvents", true, persist: false); diff --git a/src/NzbDrone.Core/Datastore/BasicRepository.cs b/src/NzbDrone.Core/Datastore/BasicRepository.cs index 4bf0d0c83..c03ecae4b 100644 --- a/src/NzbDrone.Core/Datastore/BasicRepository.cs +++ b/src/NzbDrone.Core/Datastore/BasicRepository.cs @@ -78,7 +78,7 @@ namespace NzbDrone.Core.Datastore { using (var conn = _database.OpenConnection()) { - return conn.ExecuteScalar($"SELECT COUNT(*) FROM {_table}"); + return conn.ExecuteScalar($"SELECT COUNT(*) FROM \"{_table}\""); } } @@ -167,14 +167,22 @@ namespace NzbDrone.Core.Datastore } } - return $"INSERT INTO {_table} ({sbColumnList.ToString()}) VALUES ({sbParameterList.ToString()}); SELECT last_insert_rowid() id"; + if (_database.DatabaseType == DatabaseType.SQLite) + { + return $"INSERT INTO {_table} ({sbColumnList.ToString()}) VALUES ({sbParameterList.ToString()}); SELECT last_insert_rowid() id"; + } + else + { + return $"INSERT INTO \"{_table}\" ({sbColumnList.ToString()}) VALUES ({sbParameterList.ToString()}) RETURNING \"Id\""; + } } private TModel Insert(IDbConnection connection, IDbTransaction transaction, TModel model) { SqlBuilderExtensions.LogQuery(_insertSql, model); var multi = connection.QueryMultiple(_insertSql, model, transaction); - var id = (int)multi.Read().First().id; + var multiRead = multi.Read(); + var id = (int)(multiRead.First().id ?? multiRead.First().Id); _keyProperty.SetValue(model, id); _database.ApplyLazyLoad(model); @@ -287,7 +295,7 @@ namespace NzbDrone.Core.Datastore { using (var conn = _database.OpenConnection()) { - conn.Execute($"DELETE FROM [{_table}]"); + conn.Execute($"DELETE FROM \"{_table}\""); } if (vacuum) @@ -346,7 +354,7 @@ namespace NzbDrone.Core.Datastore private string GetUpdateSql(List propertiesToUpdate) { var sb = new StringBuilder(); - sb.AppendFormat("UPDATE {0} SET ", _table); + sb.AppendFormat("UPDATE \"{0}\" SET ", _table); for (var i = 0; i < propertiesToUpdate.Count; i++) { @@ -414,9 +422,12 @@ namespace NzbDrone.Core.Datastore pagingSpec.SortKey = $"{_table}.{_keyProperty.Name}"; } + var sortKey = TableMapping.Mapper.GetSortKey(pagingSpec.SortKey); + var sortDirection = pagingSpec.SortDirection == SortDirection.Descending ? "DESC" : "ASC"; - var pagingOffset = (pagingSpec.Page - 1) * pagingSpec.PageSize; - builder.OrderBy($"{pagingSpec.SortKey} {sortDirection} LIMIT {pagingSpec.PageSize} OFFSET {pagingOffset}"); + + var pagingOffset = Math.Max(pagingSpec.Page - 1, 0) * pagingSpec.PageSize; + builder.OrderBy($"\"{sortKey}\" {sortDirection} LIMIT {pagingSpec.PageSize} OFFSET {pagingOffset}"); return queryFunc(builder).ToList(); } diff --git a/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs b/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs index 9bdd133f7..961d060f8 100644 --- a/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs +++ b/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs @@ -1,7 +1,9 @@ using System; using System.Data.SQLite; +using Npgsql; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration; namespace NzbDrone.Core.Datastore { @@ -14,10 +16,17 @@ namespace NzbDrone.Core.Datastore public class ConnectionStringFactory : IConnectionStringFactory { - public ConnectionStringFactory(IAppFolderInfo appFolderInfo) + private readonly IConfigFileProvider _configFileProvider; + + public ConnectionStringFactory(IAppFolderInfo appFolderInfo, IConfigFileProvider configFileProvider) { - MainDbConnectionString = GetConnectionString(appFolderInfo.GetDatabase()); - LogDbConnectionString = GetConnectionString(appFolderInfo.GetLogDatabase()); + _configFileProvider = configFileProvider; + + MainDbConnectionString = _configFileProvider.PostgresHost.IsNotNullOrWhiteSpace() ? GetPostgresConnectionString(_configFileProvider.PostgresMainDb) : + GetConnectionString(appFolderInfo.GetDatabase()); + + LogDbConnectionString = _configFileProvider.PostgresHost.IsNotNullOrWhiteSpace() ? GetPostgresConnectionString(_configFileProvider.PostgresLogDb) : + GetConnectionString(appFolderInfo.GetLogDatabase()); } public string MainDbConnectionString { get; private set; } @@ -48,5 +57,19 @@ namespace NzbDrone.Core.Datastore return connectionBuilder.ConnectionString; } + + private string GetPostgresConnectionString(string dbName) + { + var connectionBuilder = new NpgsqlConnectionStringBuilder(); + + connectionBuilder.Database = dbName; + connectionBuilder.Host = _configFileProvider.PostgresHost; + connectionBuilder.Username = _configFileProvider.PostgresUser; + connectionBuilder.Password = _configFileProvider.PostgresPassword; + connectionBuilder.Port = _configFileProvider.PostgresPort; + connectionBuilder.Enlist = false; + + return connectionBuilder.ConnectionString; + } } } diff --git a/src/NzbDrone.Core/Datastore/Database.cs b/src/NzbDrone.Core/Datastore/Database.cs index a9b6e807f..fd262f90d 100644 --- a/src/NzbDrone.Core/Datastore/Database.cs +++ b/src/NzbDrone.Core/Datastore/Database.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Data; +using System.Text.RegularExpressions; using Dapper; using NLog; using NzbDrone.Common.Instrumentation; @@ -11,6 +12,7 @@ namespace NzbDrone.Core.Datastore IDbConnection OpenConnection(); Version Version { get; } int Migration { get; } + DatabaseType DatabaseType { get; } void Vacuum(); } @@ -32,13 +34,44 @@ namespace NzbDrone.Core.Datastore return _datamapperFactory(); } + public DatabaseType DatabaseType + { + get + { + using (var db = _datamapperFactory()) + { + if (db.ConnectionString.Contains(".db")) + { + return DatabaseType.SQLite; + } + else + { + return DatabaseType.PostgreSQL; + } + } + } + } + public Version Version { get { using (var db = _datamapperFactory()) { - var version = db.QueryFirstOrDefault("SELECT sqlite_version()"); + string version; + + try + { + version = db.QueryFirstOrDefault("SHOW server_version"); + + //Postgres can return extra info about operating system on version call, ignore this + version = Regex.Replace(version, @"\(.*?\)", ""); + } + catch + { + version = db.QueryFirstOrDefault("SELECT sqlite_version()"); + } + return new Version(version); } } @@ -50,7 +83,7 @@ namespace NzbDrone.Core.Datastore { using (var db = _datamapperFactory()) { - return db.QueryFirstOrDefault("SELECT version from VersionInfo ORDER BY version DESC LIMIT 1"); + return db.QueryFirstOrDefault("SELECT \"Version\" from \"VersionInfo\" ORDER BY \"Version\" DESC LIMIT 1"); } } } @@ -73,4 +106,10 @@ namespace NzbDrone.Core.Datastore } } } + + public enum DatabaseType + { + SQLite, + PostgreSQL + } } diff --git a/src/NzbDrone.Core/Datastore/DbFactory.cs b/src/NzbDrone.Core/Datastore/DbFactory.cs index bd115d6d8..c62442d1b 100644 --- a/src/NzbDrone.Core/Datastore/DbFactory.cs +++ b/src/NzbDrone.Core/Datastore/DbFactory.cs @@ -1,10 +1,13 @@ using System; +using System.Data.Common; using System.Data.SQLite; using NLog; +using Npgsql; using NzbDrone.Common.Disk; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Exceptions; using NzbDrone.Common.Instrumentation; +using NzbDrone.Core.Configuration; using NzbDrone.Core.Datastore.Migration.Framework; namespace NzbDrone.Core.Datastore @@ -85,10 +88,19 @@ namespace NzbDrone.Core.Datastore var db = new Database(migrationContext.MigrationType.ToString(), () => { - var conn = SQLiteFactory.Instance.CreateConnection(); - conn.ConnectionString = connectionString; - conn.Open(); + DbConnection conn; + if (connectionString.Contains(".db")) + { + conn = SQLiteFactory.Instance.CreateConnection(); + conn.ConnectionString = connectionString; + } + else + { + conn = new NpgsqlConnection(connectionString); + } + + conn.Open(); return conn; }); diff --git a/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs b/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs index 97ce5d731..dc3e442f2 100644 --- a/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs +++ b/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs @@ -20,7 +20,7 @@ namespace NzbDrone.Core.Datastore public static SqlBuilder Select(this SqlBuilder builder, params Type[] types) { - return builder.Select(types.Select(x => TableMapping.Mapper.TableNameMapping(x) + ".*").Join(", ")); + return builder.Select(types.Select(x => $"\"{TableMapping.Mapper.TableNameMapping(x)}\".*").Join(", ")); } public static SqlBuilder SelectDistinct(this SqlBuilder builder, params Type[] types) diff --git a/src/NzbDrone.Core/Datastore/LogDatabase.cs b/src/NzbDrone.Core/Datastore/LogDatabase.cs index f992c8bbe..e29252d2a 100644 --- a/src/NzbDrone.Core/Datastore/LogDatabase.cs +++ b/src/NzbDrone.Core/Datastore/LogDatabase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Data; namespace NzbDrone.Core.Datastore @@ -10,10 +10,12 @@ namespace NzbDrone.Core.Datastore public class LogDatabase : ILogDatabase { private readonly IDatabase _database; + private readonly DatabaseType _databaseType; public LogDatabase(IDatabase database) { _database = database; + _databaseType = _database == null ? DatabaseType.SQLite : _database.DatabaseType; } public IDbConnection OpenConnection() @@ -25,6 +27,8 @@ namespace NzbDrone.Core.Datastore public int Migration => _database.Migration; + public DatabaseType DatabaseType => _databaseType; + public void Vacuum() { _database.Vacuum(); diff --git a/src/NzbDrone.Core/Datastore/MainDatabase.cs b/src/NzbDrone.Core/Datastore/MainDatabase.cs index 4a9d3298c..e3e237441 100644 --- a/src/NzbDrone.Core/Datastore/MainDatabase.cs +++ b/src/NzbDrone.Core/Datastore/MainDatabase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Data; namespace NzbDrone.Core.Datastore @@ -10,10 +10,12 @@ namespace NzbDrone.Core.Datastore public class MainDatabase : IMainDatabase { private readonly IDatabase _database; + private readonly DatabaseType _databaseType; public MainDatabase(IDatabase database) { _database = database; + _databaseType = _database == null ? DatabaseType.SQLite : _database.DatabaseType; } public IDbConnection OpenConnection() @@ -25,6 +27,8 @@ namespace NzbDrone.Core.Datastore public int Migration => _database.Migration; + public DatabaseType DatabaseType => _databaseType; + public void Vacuum() { _database.Vacuum(); diff --git a/src/NzbDrone.Core/Datastore/Migration/005_update_notifiarr.cs b/src/NzbDrone.Core/Datastore/Migration/005_update_notifiarr.cs index 5f7606ba2..91a239d5f 100644 --- a/src/NzbDrone.Core/Datastore/Migration/005_update_notifiarr.cs +++ b/src/NzbDrone.Core/Datastore/Migration/005_update_notifiarr.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Execute.Sql("UPDATE Notifications SET Implementation = Replace(Implementation, 'DiscordNotifier', 'Notifiarr'),ConfigContract = Replace(ConfigContract, 'DiscordNotifierSettings', 'NotifiarrSettings') WHERE Implementation = 'DiscordNotifier';"); + Execute.Sql("UPDATE \"Notifications\" SET \"Implementation\" = Replace(\"Implementation\", 'DiscordNotifier', 'Notifiarr'),\"ConfigContract\" = Replace(\"ConfigContract\", 'DiscordNotifierSettings', 'NotifiarrSettings') WHERE \"Implementation\" = 'DiscordNotifier';"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/007_history_failed.cs b/src/NzbDrone.Core/Datastore/Migration/007_history_failed.cs index 287a6c18e..845da3ab9 100644 --- a/src/NzbDrone.Core/Datastore/Migration/007_history_failed.cs +++ b/src/NzbDrone.Core/Datastore/Migration/007_history_failed.cs @@ -11,7 +11,8 @@ namespace NzbDrone.Core.Datastore.Migration Alter.Table("History") .AddColumn("Successful").AsBoolean().NotNullable().WithDefaultValue(true); - Execute.Sql("UPDATE History SET Successful = (json_extract(History.Data,'$.successful') == 'True' );"); + // Postgres added after this, not needed + IfDatabase("sqlite").Execute.Sql("UPDATE \"History\" SET \"Successful\" = (json_extract(\"History\".\"Data\",'$.successful') == 'True' );"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/008_redacted_api.cs b/src/NzbDrone.Core/Datastore/Migration/008_redacted_api.cs index ac6c4ba38..796e74d13 100644 --- a/src/NzbDrone.Core/Datastore/Migration/008_redacted_api.cs +++ b/src/NzbDrone.Core/Datastore/Migration/008_redacted_api.cs @@ -19,7 +19,7 @@ namespace NzbDrone.Core.Datastore.Migration using (var cmd = conn.CreateCommand()) { cmd.Transaction = tran; - cmd.CommandText = "SELECT Id, Settings FROM Indexers WHERE Implementation = 'Redacted'"; + cmd.CommandText = "SELECT \"Id\", \"Settings\" FROM \"Indexers\" WHERE \"Implementation\" = 'Redacted'"; using (var reader = cmd.ExecuteReader()) { @@ -48,7 +48,7 @@ namespace NzbDrone.Core.Datastore.Migration using (var updateCmd = conn.CreateCommand()) { updateCmd.Transaction = tran; - updateCmd.CommandText = "UPDATE Indexers SET Settings = ?, ConfigContract = ?, Enable = 0 WHERE Id = ?"; + updateCmd.CommandText = "UPDATE \"Indexers\" SET \"Settings\" = ?, \"ConfigContract\" = ?, \"Enable\" = 0 WHERE \"Id\" = ?"; updateCmd.AddParameter(settings); updateCmd.AddParameter("RedactedSettings"); updateCmd.AddParameter(id); diff --git a/src/NzbDrone.Core/Datastore/Migration/013_desi_gazelle_to_unit3d.cs b/src/NzbDrone.Core/Datastore/Migration/013_desi_gazelle_to_unit3d.cs index c066fa8a8..be02bcf1f 100644 --- a/src/NzbDrone.Core/Datastore/Migration/013_desi_gazelle_to_unit3d.cs +++ b/src/NzbDrone.Core/Datastore/Migration/013_desi_gazelle_to_unit3d.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Update.Table("Indexers").Set(new { ConfigContract = "Unit3dSettings", Enable = 0 }).Where(new { Implementation = "DesiTorrents" }); + Update.Table("Indexers").Set(new { ConfigContract = "Unit3dSettings", Enable = true }).Where(new { Implementation = "DesiTorrents" }); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs index 35c5c6b59..1249dfd8b 100644 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics; using System.Reflection; using FluentMigrator.Runner; +using FluentMigrator.Runner.Generators; using FluentMigrator.Runner.Initialization; using FluentMigrator.Runner.Processors; using Microsoft.Extensions.DependencyInjection; @@ -34,11 +35,16 @@ namespace NzbDrone.Core.Datastore.Migration.Framework _logger.Info("*** Migrating {0} ***", connectionString); - var serviceProvider = new ServiceCollection() + ServiceProvider serviceProvider; + + var db = connectionString.Contains(".db") ? "sqlite" : "postgres"; + + serviceProvider = new ServiceCollection() .AddLogging(b => b.AddNLog()) .AddFluentMigratorCore() .ConfigureRunner( builder => builder + .AddPostgres() .AddNzbDroneSQLite() .WithGlobalConnectionString(connectionString) .WithMigrationsIn(Assembly.GetExecutingAssembly())) @@ -48,6 +54,14 @@ namespace NzbDrone.Core.Datastore.Migration.Framework opt.PreviewOnly = false; opt.Timeout = TimeSpan.FromSeconds(60); }) + .Configure(cfg => + { + cfg.ProcessorId = db; + }) + .Configure(cfg => + { + cfg.GeneratorId = db; + }) .BuildServiceProvider(); using (var scope = serviceProvider.CreateScope()) diff --git a/src/NzbDrone.Core/Datastore/TableMapper.cs b/src/NzbDrone.Core/Datastore/TableMapper.cs index 0dbfca678..bbb84b9cb 100644 --- a/src/NzbDrone.Core/Datastore/TableMapper.cs +++ b/src/NzbDrone.Core/Datastore/TableMapper.cs @@ -8,6 +8,8 @@ namespace NzbDrone.Core.Datastore { public class TableMapper { + private readonly HashSet _allowedOrderBy = new HashSet(StringComparer.OrdinalIgnoreCase); + public TableMapper() { IgnoreList = new Dictionary>(); @@ -27,12 +29,12 @@ namespace NzbDrone.Core.Datastore if (IgnoreList.TryGetValue(type, out var list)) { - return new ColumnMapper(list, LazyLoadList[type]); + return new ColumnMapper(list, LazyLoadList[type], _allowedOrderBy); } IgnoreList[type] = new List(); LazyLoadList[type] = new List(); - return new ColumnMapper(IgnoreList[type], LazyLoadList[type]); + return new ColumnMapper(IgnoreList[type], LazyLoadList[type], _allowedOrderBy); } public List ExcludeProperties(Type x) @@ -40,6 +42,64 @@ namespace NzbDrone.Core.Datastore return IgnoreList.ContainsKey(x) ? IgnoreList[x] : new List(); } + public bool IsValidSortKey(string sortKey) + { + string table = null; + + if (sortKey.Contains('.')) + { + var split = sortKey.Split('.'); + if (split.Length != 2) + { + return false; + } + + table = split[0]; + sortKey = split[1]; + } + + if (table != null && !TableMap.Values.Contains(table, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + + if (!_allowedOrderBy.Contains(sortKey)) + { + return false; + } + + return true; + } + + public string GetSortKey(string sortKey) + { + string table = null; + + if (sortKey.Contains('.')) + { + var split = sortKey.Split('.'); + if (split.Length != 2) + { + return sortKey; + } + + table = split[0]; + sortKey = split[1]; + } + + if (table != null && !TableMap.Values.Contains(table, StringComparer.OrdinalIgnoreCase)) + { + return sortKey; + } + + if (!_allowedOrderBy.Contains(sortKey)) + { + return sortKey; + } + + return _allowedOrderBy.First(x => x.Equals(sortKey, StringComparison.OrdinalIgnoreCase)); + } + public string TableNameMapping(Type x) { return TableMap.ContainsKey(x) ? TableMap[x] : null; @@ -47,17 +107,17 @@ namespace NzbDrone.Core.Datastore public string SelectTemplate(Type x) { - return $"SELECT /**select**/ FROM {TableMap[x]} /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/ /**orderby**/"; + return $"SELECT /**select**/ FROM \"{TableMap[x]}\" /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/ /**orderby**/"; } public string DeleteTemplate(Type x) { - return $"DELETE FROM {TableMap[x]} /**where**/"; + return $"DELETE FROM \"{TableMap[x]}\" /**where**/"; } public string PageCountTemplate(Type x) { - return $"SELECT /**select**/ FROM {TableMap[x]} /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/"; + return $"SELECT /**select**/ FROM \"{TableMap[x]}\" /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/"; } } @@ -72,17 +132,20 @@ namespace NzbDrone.Core.Datastore { private readonly List _ignoreList; private readonly List _lazyLoadList; + private readonly HashSet _allowedOrderBy; - public ColumnMapper(List ignoreList, List lazyLoadList) + public ColumnMapper(List ignoreList, List lazyLoadList, HashSet allowedOrderBy) { _ignoreList = ignoreList; _lazyLoadList = lazyLoadList; + _allowedOrderBy = allowedOrderBy; } public ColumnMapper AutoMapPropertiesWhere(Func predicate) { var properties = typeof(T).GetProperties(); _ignoreList.AddRange(properties.Where(x => !predicate(x))); + _allowedOrderBy.UnionWith(properties.Where(x => predicate(x)).Select(x => x.Name)); return this; } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalUsers.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalUsers.cs index 4460ac83c..8b36317ec 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalUsers.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalUsers.cs @@ -1,4 +1,4 @@ -using Dapper; +using Dapper; using NzbDrone.Core.Datastore; namespace NzbDrone.Core.Housekeeping.Housekeepers @@ -16,9 +16,9 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM Users - WHERE ID NOT IN ( - SELECT ID FROM Users + mapper.Execute(@"DELETE FROM ""Users"" + WHERE ""Id"" NOT IN ( + SELECT ""Id"" FROM ""Users"" LIMIT 1)"); } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs index a5f584797..5a6af9fd8 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs @@ -1,4 +1,4 @@ -using Dapper; +using Dapper; using NzbDrone.Core.Datastore; namespace NzbDrone.Core.Housekeeping.Housekeepers @@ -16,12 +16,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { var mapper = _database.OpenConnection(); - mapper.Execute(@"DELETE FROM DownloadClientStatus - WHERE Id IN ( - SELECT DownloadClientStatus.Id FROM DownloadClientStatus - LEFT OUTER JOIN DownloadClients - ON DownloadClientStatus.ProviderId = DownloadClients.Id - WHERE DownloadClients.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""DownloadClientStatus"" + WHERE ""Id"" IN ( + SELECT ""DownloadClientStatus"".""Id"" FROM ""DownloadClientStatus"" + LEFT OUTER JOIN ""DownloadClients"" + ON ""DownloadClientStatus"".""ProviderId"" = ""DownloadClients"".""Id"" + WHERE ""DownloadClients"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedHistoryItems.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedHistoryItems.cs index ad5a8e8ab..e6d8d789d 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedHistoryItems.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedHistoryItems.cs @@ -21,12 +21,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM History - WHERE Id IN ( - SELECT History.Id FROM History - LEFT OUTER JOIN Indexers - ON History.IndexerId = Indexers.Id - WHERE Indexers.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""History"" + WHERE ""Id"" IN ( + SELECT ""History"".""Id"" FROM ""History"" + LEFT OUTER JOIN ""Indexers"" + ON ""History"".""IndexerId"" = ""Indexers"".""Id"" + WHERE ""Indexers"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs index 039db9b52..059f059e4 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs @@ -16,12 +16,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM IndexerStatus - WHERE Id IN ( - SELECT IndexerStatus.Id FROM IndexerStatus - LEFT OUTER JOIN Indexers - ON IndexerStatus.ProviderId = Indexers.Id - WHERE Indexers.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""IndexerStatus"" + WHERE ""Id"" IN ( + SELECT ""IndexerStatus"".""Id"" FROM ""IndexerStatus"" + LEFT OUTER JOIN ""Indexers"" + ON ""IndexerStatus"".""ProviderId"" = ""Indexers"".""Id"" + WHERE ""Indexers"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupUnusedTags.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupUnusedTags.cs index 4ee4afd78..1df062918 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupUnusedTags.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupUnusedTags.cs @@ -24,15 +24,22 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers .Distinct() .ToArray(); - var usedTagsList = string.Join(",", usedTags.Select(d => d.ToString()).ToArray()); + if (usedTags.Length > 0) + { + var usedTagsList = string.Join(",", usedTags.Select(d => d.ToString()).ToArray()); - mapper.Execute($"DELETE FROM Tags WHERE NOT Id IN ({usedTagsList})"); + mapper.Execute($"DELETE FROM \"Tags\" WHERE NOT \"Id\" IN ({usedTagsList})"); + } + else + { + mapper.Execute($"DELETE FROM \"Tags\""); + } } } private int[] GetUsedTags(string table, IDbConnection mapper) { - return mapper.Query>($"SELECT DISTINCT Tags FROM {table} WHERE NOT Tags = '[]'") + return mapper.Query>($"SELECT DISTINCT \"Tags\" FROM \"{table}\" WHERE NOT \"Tags\" = '[]'") .SelectMany(x => x) .Distinct() .ToArray(); diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureRunScheduledTasks.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureRunScheduledTasks.cs index 82a7af7fa..6b45ed0c4 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureRunScheduledTasks.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureRunScheduledTasks.cs @@ -1,4 +1,4 @@ -using System; +using System; using Dapper; using NLog; using NzbDrone.Common.EnvironmentInfo; @@ -26,9 +26,9 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"UPDATE ScheduledTasks - SET LastExecution = @time - WHERE LastExecution > @time", + mapper.Execute(@"UPDATE ""ScheduledTasks"" + SET ""LastExecution"" = @time + WHERE ""LastExecution"" > @time", new { time = DateTime.UtcNow }); } } diff --git a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs index f761f3f7c..72ca2da08 100644 --- a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs +++ b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs @@ -1,9 +1,11 @@ -using System.Data; +using System; +using System.Data; using System.Data.SQLite; using NLog; using NLog.Common; using NLog.Config; using NLog.Targets; +using Npgsql; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Datastore; using NzbDrone.Core.Lifecycle; @@ -13,7 +15,7 @@ namespace NzbDrone.Core.Instrumentation { public class DatabaseTarget : TargetWithLayout, IHandle { - private const string INSERT_COMMAND = "INSERT INTO [Logs]([Message],[Time],[Logger],[Exception],[ExceptionType],[Level]) " + + private const string INSERT_COMMAND = "INSERT INTO \"Logs\" (\"Message\",\"Time\",\"Logger\",\"Exception\",\"ExceptionType\",\"Level\") " + "VALUES(@Message,@Time,@Logger,@Exception,@ExceptionType,@Level)"; private readonly IConnectionStringFactory _connectionStringFactory; @@ -83,23 +85,16 @@ namespace NzbDrone.Core.Instrumentation log.Level = logEvent.Level.Name; - using (var connection = - SQLiteFactory.Instance.CreateConnection()) + var connectionString = _connectionStringFactory.LogDbConnectionString; + + //TODO: Probably need more robust way to differentiate what's being used + if (connectionString.Contains(".db")) { - connection.ConnectionString = _connectionStringFactory.LogDbConnectionString; - connection.Open(); - using (var sqlCommand = connection.CreateCommand()) - { - sqlCommand.CommandText = INSERT_COMMAND; - sqlCommand.Parameters.Add(new SQLiteParameter("Message", DbType.String) { Value = log.Message }); - sqlCommand.Parameters.Add(new SQLiteParameter("Time", DbType.DateTime) { Value = log.Time.ToUniversalTime() }); - sqlCommand.Parameters.Add(new SQLiteParameter("Logger", DbType.String) { Value = log.Logger }); - sqlCommand.Parameters.Add(new SQLiteParameter("Exception", DbType.String) { Value = log.Exception }); - sqlCommand.Parameters.Add(new SQLiteParameter("ExceptionType", DbType.String) { Value = log.ExceptionType }); - sqlCommand.Parameters.Add(new SQLiteParameter("Level", DbType.String) { Value = log.Level }); - - sqlCommand.ExecuteNonQuery(); - } + WriteSqliteLog(log, connectionString); + } + else + { + WritePostgresLog(log, connectionString); } } catch (SQLiteException ex) @@ -109,6 +104,48 @@ namespace NzbDrone.Core.Instrumentation } } + private void WritePostgresLog(Log log, string connectionString) + { + using (var connection = + new NpgsqlConnection(connectionString)) + { + connection.Open(); + using (var sqlCommand = connection.CreateCommand()) + { + sqlCommand.CommandText = INSERT_COMMAND; + sqlCommand.Parameters.Add(new NpgsqlParameter("Message", DbType.String) { Value = log.Message }); + sqlCommand.Parameters.Add(new NpgsqlParameter("Time", DbType.DateTime) { Value = log.Time.ToUniversalTime() }); + sqlCommand.Parameters.Add(new NpgsqlParameter("Logger", DbType.String) { Value = log.Logger }); + sqlCommand.Parameters.Add(new NpgsqlParameter("Exception", DbType.String) { Value = log.Exception == null ? DBNull.Value : log.Exception }); + sqlCommand.Parameters.Add(new NpgsqlParameter("ExceptionType", DbType.String) { Value = log.ExceptionType == null ? DBNull.Value : log.ExceptionType }); + sqlCommand.Parameters.Add(new NpgsqlParameter("Level", DbType.String) { Value = log.Level }); + + sqlCommand.ExecuteNonQuery(); + } + } + } + + private void WriteSqliteLog(Log log, string connectionString) + { + using (var connection = + SQLiteFactory.Instance.CreateConnection()) + { + connection.ConnectionString = connectionString; + connection.Open(); + using (var sqlCommand = connection.CreateCommand()) + { + sqlCommand.CommandText = INSERT_COMMAND; + sqlCommand.Parameters.Add(new SQLiteParameter("Message", DbType.String) { Value = log.Message }); + sqlCommand.Parameters.Add(new SQLiteParameter("Time", DbType.DateTime) { Value = log.Time.ToUniversalTime() }); + sqlCommand.Parameters.Add(new SQLiteParameter("Logger", DbType.String) { Value = log.Logger }); + sqlCommand.Parameters.Add(new SQLiteParameter("Exception", DbType.String) { Value = log.Exception }); + sqlCommand.Parameters.Add(new SQLiteParameter("ExceptionType", DbType.String) { Value = log.ExceptionType }); + sqlCommand.Parameters.Add(new SQLiteParameter("Level", DbType.String) { Value = log.Level }); + sqlCommand.ExecuteNonQuery(); + } + } + } + public void Handle(ApplicationShutdownRequested message) { if (LogManager.Configuration?.LoggingRules?.Contains(Rule) == true) diff --git a/src/NzbDrone.Core/Messaging/Commands/CommandRepository.cs b/src/NzbDrone.Core/Messaging/Commands/CommandRepository.cs index 1bf894ecc..d41f2fbba 100644 --- a/src/NzbDrone.Core/Messaging/Commands/CommandRepository.cs +++ b/src/NzbDrone.Core/Messaging/Commands/CommandRepository.cs @@ -31,7 +31,7 @@ namespace NzbDrone.Core.Messaging.Commands public void OrphanStarted() { - var sql = @"UPDATE Commands SET Status = @Orphaned, EndedAt = @Ended WHERE Status = @Started"; + var sql = @"UPDATE ""Commands"" SET ""Status"" = @Orphaned, ""EndedAt"" = @Ended WHERE ""Status"" = @Started"; var args = new { Orphaned = (int)CommandStatus.Orphaned, diff --git a/src/NzbDrone.Core/Prowlarr.Core.csproj b/src/NzbDrone.Core/Prowlarr.Core.csproj index 6d89467d0..99a4adc4d 100644 --- a/src/NzbDrone.Core/Prowlarr.Core.csproj +++ b/src/NzbDrone.Core/Prowlarr.Core.csproj @@ -9,9 +9,11 @@ + + diff --git a/src/Prowlarr.Api.V1/System/SystemController.cs b/src/Prowlarr.Api.V1/System/SystemController.cs index ec5e2f28c..66d058b71 100644 --- a/src/Prowlarr.Api.V1/System/SystemController.cs +++ b/src/Prowlarr.Api.V1/System/SystemController.cs @@ -77,7 +77,8 @@ namespace Prowlarr.Api.V1.System Mode = _runtimeInfo.Mode, Branch = _configFileProvider.Branch, Authentication = _configFileProvider.AuthenticationMethod, - SqliteVersion = _database.Version, + DatabaseType = _database.DatabaseType, + DatabaseVersion = _database.Version, MigrationVersion = _database.Migration, UrlBase = _configFileProvider.UrlBase, RuntimeVersion = _platformInfo.Version,