From 46c2e0ba82b7625f13b9d28f6384cac95d37b496 Mon Sep 17 00:00:00 2001 From: Robin Dadswell <19610103+RobinDadswell@users.noreply.github.com> Date: Sat, 28 May 2022 21:45:24 +0100 Subject: [PATCH] New: Postgres Support Co-Authored-By: Qstick <376117+Qstick@users.noreply.github.com> Co-authored-by: ta264 (cherry picked from commit 80b1aa9a2c81617bdda7ef551c19a2f114e49204) --- azure-pipelines.yml | 109 +++++ frontend/src/System/Status/About/About.js | 9 + src/Directory.Packages.props | 2 + .../AutomationTest.cs | 2 +- .../CleanseLogMessageFixture.cs | 1 + .../ServiceFactoryFixture.cs | 5 +- .../Instrumentation/CleanseLogMessage.cs | 1 + .../Processes/ProcessProvider.cs | 13 +- .../Datastore/DatabaseFixture.cs | 2 +- .../Datastore/LazyLoadingFixture.cs | 6 +- .../Datastore/WhereBuilderPostgresFixture.cs | 211 ++++++++++ ...ixture.cs => WhereBuilderSqliteFixture.cs} | 16 +- src/NzbDrone.Core.Test/Framework/DbTest.cs | 85 +++- .../Framework/TestDatabase.cs | 3 + .../AlbumRepositoryFixture.cs | 17 +- .../ArtistRepositoryFixture.cs | 11 +- .../MusicTests/RefreshArtistServiceFixture.cs | 6 +- .../AuthorStats/AuthorStatisticsRepository.cs | 18 +- src/NzbDrone.Core/Backup/BackupService.cs | 7 +- .../Blocklisting/BlocklistRepository.cs | 2 +- .../Books/Repositories/AuthorRepository.cs | 4 +- .../Books/Repositories/BookRepository.cs | 40 +- .../Books/Repositories/EditionRepository.cs | 6 +- .../Books/Repositories/SeriesRepository.cs | 4 +- .../Books/Services/BookService.cs | 4 +- .../Books/Services/EditionService.cs | 4 +- .../Books/Services/RefreshAuthorService.cs | 2 +- .../Books/Services/RefreshBookService.cs | 2 +- .../Books/Services/RefreshSeriesService.cs | 2 +- .../Books/Services/SeriesService.cs | 4 +- .../Configuration/ConfigFileProvider.cs | 23 +- .../Datastore/BasicRepository.cs | 21 +- src/NzbDrone.Core/Datastore/CacheDatabase.cs | 6 +- .../Datastore/ConnectionStringFactory.cs | 33 +- src/NzbDrone.Core/Datastore/Database.cs | 43 +- src/NzbDrone.Core/Datastore/DbFactory.cs | 17 +- .../Datastore/Extensions/BuilderExtensions.cs | 37 +- src/NzbDrone.Core/Datastore/LogDatabase.cs | 4 + src/NzbDrone.Core/Datastore/MainDatabase.cs | 4 + .../Datastore/Migration/001_initial_setup.cs | 55 +-- .../Migration/002_import_list_search.cs | 2 +- ...006_remove_chown_and_folderchmod_config.cs | 8 +- .../Migration/007_update_notifiarr.cs | 2 +- .../Migration/009_update_author_sort_name.cs | 4 +- .../Migration/011_update_audio_qualities.cs | 4 +- .../012_add_bookfile_part_naming_token.cs | 2 +- .../013_update_author_sort_name_again.cs | 4 +- .../Datastore/Migration/015_fix_indexes.cs | 4 +- .../017_import_list_monitor_existing.cs | 2 +- .../Migration/020_add_download_history.cs | 16 +- .../021_add_on_delete_to_notifications.cs | 8 +- .../Framework/MigrationController.cs | 16 +- .../Datastore/PostgresOptions.cs | 27 ++ src/NzbDrone.Core/Datastore/SqlBuilder.cs | 8 + src/NzbDrone.Core/Datastore/TableMapper.cs | 37 +- src/NzbDrone.Core/Datastore/TableMapping.cs | 18 +- src/NzbDrone.Core/Datastore/WhereBuilder.cs | 384 +---------------- .../Datastore/WhereBuilderPostgres.cs | 387 ++++++++++++++++++ .../Datastore/WhereBuilderSqlite.cs | 387 ++++++++++++++++++ .../History/HistoryRepository.cs | 6 +- .../CleanupAbsolutePathMetadataFiles.cs | 30 +- .../CleanupAdditionalNamingSpecs.cs | 6 +- .../Housekeepers/CleanupAdditionalUsers.cs | 6 +- ...ownloadClientUnavailablePendingReleases.cs | 31 +- .../CleanupDuplicateMetadataFiles.cs | 36 +- .../CleanupOrphanedAuthorMetadata.cs | 12 +- .../Housekeepers/CleanupOrphanedBlocklist.cs | 12 +- .../Housekeepers/CleanupOrphanedBookFiles.cs | 14 +- .../Housekeepers/CleanupOrphanedBooks.cs | 12 +- .../CleanupOrphanedDownloadClientStatus.cs | 17 +- .../Housekeepers/CleanupOrphanedEditions.cs | 12 +- .../CleanupOrphanedHistoryItems.cs | 26 +- .../CleanupOrphanedImportListStatus.cs | 12 +- .../CleanupOrphanedIndexerStatus.cs | 12 +- .../CleanupOrphanedMetadataFiles.cs | 60 +-- .../CleanupOrphanedPendingReleases.cs | 12 +- .../CleanupOrphanedSeriesBookLinks.cs | 24 +- .../Housekeepers/CleanupUnusedTags.cs | 21 +- .../FixFutureRunScheduledTasks.cs | 6 +- .../Housekeepers/TrimHttpCache.cs | 2 +- .../Instrumentation/DatabaseTarget.cs | 68 ++- src/NzbDrone.Core/Localization/Core/en.json | 1 + .../MediaFiles/MediaFileRepository.cs | 8 +- .../Messaging/Commands/CommandRepository.cs | 2 +- src/NzbDrone.Core/Readarr.Core.csproj | 2 + .../Update/UpdatePackageProvider.cs | 6 +- src/NzbDrone.Host.Test/ContainerFixture.cs | 3 + src/NzbDrone.Host/Bootstrap.cs | 8 +- .../IntegrationTest.cs | 33 +- .../Datastore/PostgresDatabase.cs | 69 ++++ .../Datastore/SqliteDatabase.cs | 14 + src/NzbDrone.Test.Common/NzbDroneRunner.cs | 23 +- src/Readarr.Api.V1/System/SystemController.cs | 3 +- src/postgres.runsettings | 11 + 94 files changed, 2057 insertions(+), 724 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Datastore/WhereBuilderPostgresFixture.cs rename src/NzbDrone.Core.Test/Datastore/{WhereBuilderFixture.cs => WhereBuilderSqliteFixture.cs} (92%) create mode 100644 src/NzbDrone.Core/Datastore/PostgresOptions.cs create mode 100644 src/NzbDrone.Core/Datastore/WhereBuilderPostgres.cs create mode 100644 src/NzbDrone.Core/Datastore/WhereBuilderSqlite.cs create mode 100644 src/NzbDrone.Test.Common/Datastore/PostgresDatabase.cs create mode 100644 src/NzbDrone.Test.Common/Datastore/SqliteDatabase.cs create mode 100644 src/postgres.runsettings diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7be680632..3a1c58275 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -529,6 +529,56 @@ stages: testResultsFiles: '**/TestResult.xml' testRunTitle: '$(testName) Unit Tests' failTaskOnFailedTests: true + + - job: Unit_LinuxCore_Postgres + displayName: Unit Native LinuxCore with Postgres Database + variables: + pattern: 'Readarr.*.linux-core-x64.tar.gz' + artifactName: LinuxCoreTests + Readarr__Postgres__Host: 'localhost' + Readarr__Postgres__Port: '5432' + Readarr__Postgres__User: 'readarr' + Readarr__Postgres__Password: 'readarr' + + pool: + vmImage: ${{ variables.linuxImage }} + + timeoutInMinutes: 10 + + steps: + - task: UseDotNet@2 + displayName: 'Install .net core' + inputs: + version: $(dotnetVersion) + - checkout: none + - task: DownloadPipelineArtifact@2 + displayName: Download Test Artifact + inputs: + buildType: 'current' + artifactName: 'linux-x64-Tests' + targetPath: $(testsFolder) + - bash: find ${TESTSFOLDER} -name "Readarr.Test.Dummy" -exec chmod a+x {} \; + displayName: Make Test Dummy Executable + condition: and(succeeded(), ne(variables['osName'], 'Windows')) + - bash: | + docker run -d --name=postgres14 \ + -e POSTGRES_PASSWORD=readarr \ + -e POSTGRES_USER=readarr \ + -p 5432:5432/tcp \ + postgres:14 + displayName: Start postgres + - bash: | + chmod a+x ${TESTSFOLDER}/test.sh + ls -lR ${TESTSFOLDER} + ${TESTSFOLDER}/test.sh Linux Unit Test + displayName: Run Tests + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '**/TestResult.xml' + testRunTitle: 'LinuxCore Postgres Unit Tests' + failTaskOnFailedTests: true - stage: Integration displayName: Integration @@ -597,6 +647,65 @@ stages: failTaskOnFailedTests: true displayName: Publish Test Results + - job: Integration_LinuxCore_Postgres + displayName: Integration Native LinuxCore with Postgres Database + variables: + pattern: 'Readarr.*.linux-core-x64.tar.gz' + Readarr__Postgres__Host: 'localhost' + Readarr__Postgres__Port: '5432' + Readarr__Postgres__User: 'readarr' + Readarr__Postgres__Password: 'readarr' + + pool: + vmImage: ${{ variables.linuxImage }} + + steps: + - task: UseDotNet@2 + displayName: 'Install .net core' + inputs: + version: $(dotnetVersion) + - checkout: none + - task: DownloadPipelineArtifact@2 + displayName: Download Test Artifact + inputs: + buildType: 'current' + artifactName: 'linux-x64-tests' + targetPath: $(testsFolder) + - task: DownloadPipelineArtifact@2 + displayName: Download Build Artifact + inputs: + buildType: 'current' + artifactName: Packages + itemPattern: '**/$(pattern)' + targetPath: $(Build.ArtifactStagingDirectory) + - task: ExtractFiles@1 + inputs: + archiveFilePatterns: '$(Build.ArtifactStagingDirectory)/**/$(pattern)' + destinationFolder: '$(Build.ArtifactStagingDirectory)/bin' + displayName: Extract Package + - bash: | + mkdir -p ./bin/ + cp -r -v ${BUILD_ARTIFACTSTAGINGDIRECTORY}/bin/Readarr/. ./bin/ + displayName: Move Package Contents + - bash: | + docker run -d --name=postgres14 \ + -e POSTGRES_PASSWORD=readarr \ + -e POSTGRES_USER=readarr \ + -p 5432:5432/tcp \ + postgres:14 + displayName: Start postgres + - bash: | + chmod a+x ${TESTSFOLDER}/test.sh + ${TESTSFOLDER}/test.sh Linux Integration Test + displayName: Run Integration Tests + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '**/TestResult.xml' + testRunTitle: 'Integration LinuxCore Postgres Database Integration Tests' + failTaskOnFailedTests: true + displayName: Publish Test Results + - job: Integration_FreeBSD displayName: Integration Native FreeBSD workspace: diff --git a/frontend/src/System/Status/About/About.js b/frontend/src/System/Status/About/About.js index 62f54cf97..b8dcacafa 100644 --- a/frontend/src/System/Status/About/About.js +++ b/frontend/src/System/Status/About/About.js @@ -24,6 +24,8 @@ class About extends Component { isDocker, runtimeVersion, migrationVersion, + databaseVersion, + databaseType, appData, startupPath, mode, @@ -77,6 +79,11 @@ class About extends Component { data={migrationVersion} /> + + + @@ -31,6 +32,7 @@ + diff --git a/src/NzbDrone.Automation.Test/AutomationTest.cs b/src/NzbDrone.Automation.Test/AutomationTest.cs index 792222daf..5cbee98a2 100644 --- a/src/NzbDrone.Automation.Test/AutomationTest.cs +++ b/src/NzbDrone.Automation.Test/AutomationTest.cs @@ -44,7 +44,7 @@ namespace NzbDrone.Automation.Test driver.Manage().Window.Size = new System.Drawing.Size(1920, 1080); - _runner = new NzbDroneRunner(LogManager.GetCurrentClassLogger()); + _runner = new NzbDroneRunner(LogManager.GetCurrentClassLogger(), null); _runner.KillAll(); _runner.Start(); diff --git a/src/NzbDrone.Common.Test/InstrumentationTests/CleanseLogMessageFixture.cs b/src/NzbDrone.Common.Test/InstrumentationTests/CleanseLogMessageFixture.cs index 3e1c649fa..b13a58010 100644 --- a/src/NzbDrone.Common.Test/InstrumentationTests/CleanseLogMessageFixture.cs +++ b/src/NzbDrone.Common.Test/InstrumentationTests/CleanseLogMessageFixture.cs @@ -60,6 +60,7 @@ namespace NzbDrone.Common.Test.InstrumentationTests [TestCase("Hardlink '/home/mySecret/Downloads/abs.mkv' to '/media/abc.mkv' failed.")] [TestCase("https://notifiarr.com/notifier.php: api=1234530f-422f-4aac-b6b3-01233210aaaa&radarr_health_issue_message=Download")] [TestCase("/readarr/signalr/messages/negotiate?access_token=1234530f422f4aacb6b301233210aaaa&negotiateVersion=1")] + [TestCase(@"[Info] MigrationController: *** Migrating Database=readarr-main;Host=postgres14;Username=mySecret;Password=mySecret;Port=5432;Enlist=False ***")] // Announce URLs (passkeys) Magnet & Tracker [TestCase(@"magnet_uri"":""magnet:?xt=urn:btih:9pr04sgkillroyimaveql2tyu8xyui&dn=&tr=https%3a%2f%2fxxx.yyy%2f9pr04sg601233210imaveql2tyu8xyui%2fannounce""}")] diff --git a/src/NzbDrone.Common.Test/ServiceFactoryFixture.cs b/src/NzbDrone.Common.Test/ServiceFactoryFixture.cs index 1a3482582..f628452d9 100644 --- a/src/NzbDrone.Common.Test/ServiceFactoryFixture.cs +++ b/src/NzbDrone.Common.Test/ServiceFactoryFixture.cs @@ -4,11 +4,13 @@ using DryIoc.Microsoft.DependencyInjection; using FluentAssertions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; using Moq; using NUnit.Framework; using NzbDrone.Common.Composition.Extensions; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Instrumentation.Extensions; +using NzbDrone.Core.Datastore; using NzbDrone.Core.Datastore.Extensions; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; @@ -29,7 +31,8 @@ namespace NzbDrone.Common.Test .AddDummyDatabase() .AddStartupContext(new StartupContext("first", "second")); - container.RegisterInstance(new Mock().Object); + container.RegisterInstance(new Mock().Object); + container.RegisterInstance(new Mock>().Object); var serviceProvider = container.GetServiceProvider(); serviceProvider.GetRequiredService().Register(); diff --git a/src/NzbDrone.Common/Instrumentation/CleanseLogMessage.cs b/src/NzbDrone.Common/Instrumentation/CleanseLogMessage.cs index 03521958f..9e616e8a0 100644 --- a/src/NzbDrone.Common/Instrumentation/CleanseLogMessage.cs +++ b/src/NzbDrone.Common/Instrumentation/CleanseLogMessage.cs @@ -18,6 +18,7 @@ namespace NzbDrone.Common.Instrumentation new Regex(@"iptorrents\.com/[/a-z0-9?&;]*?(?:[?&;](u|tp)=(?[^&=;]+?))+(?= |;|&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase), new Regex(@"/fetch/[a-z0-9]{32}/(?[a-z0-9]{32})", RegexOptions.Compiled), new Regex(@"getnzb.*?(?<=\?|&)(r)=(?[^&=]+?)(?= |&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase), + new Regex(@"(?<=[?& ;])[^=]*?(_?(?[^&=]+?)(?= |&|$|;)", RegexOptions.Compiled | RegexOptions.IgnoreCase), // Trackers Announce Keys; Designed for Qbit Json; should work for all in theory new Regex(@"announce(\.php)?(/|%2f|%3fpasskey%3d)(?[a-z0-9]{16,})|(?[a-z0-9]{16,})(/|%2f)announce"), diff --git a/src/NzbDrone.Common/Processes/ProcessProvider.cs b/src/NzbDrone.Common/Processes/ProcessProvider.cs index b44e43472..fdb65535d 100644 --- a/src/NzbDrone.Common/Processes/ProcessProvider.cs +++ b/src/NzbDrone.Common/Processes/ProcessProvider.cs @@ -127,7 +127,18 @@ namespace NzbDrone.Common.Processes try { _logger.Trace("Setting environment variable '{0}' to '{1}'", environmentVariable.Key, environmentVariable.Value); - startInfo.EnvironmentVariables.Add(environmentVariable.Key.ToString(), environmentVariable.Value.ToString()); + + var key = environmentVariable.Key.ToString(); + var value = environmentVariable.Value?.ToString(); + + if (startInfo.EnvironmentVariables.ContainsKey(key)) + { + startInfo.EnvironmentVariables[key] = value; + } + else + { + startInfo.EnvironmentVariables.Add(key, value); + } } catch (Exception e) { diff --git a/src/NzbDrone.Core.Test/Datastore/DatabaseFixture.cs b/src/NzbDrone.Core.Test/Datastore/DatabaseFixture.cs index 52d6a5040..d37789f5b 100644 --- a/src/NzbDrone.Core.Test/Datastore/DatabaseFixture.cs +++ b/src/NzbDrone.Core.Test/Datastore/DatabaseFixture.cs @@ -15,7 +15,7 @@ namespace NzbDrone.Core.Test.Datastore public void SingleOrDefault_should_return_null_on_empty_db() { Mocker.Resolve() - .OpenConnection().Query("SELECT * FROM Authors") + .OpenConnection().Query("SELECT * FROM \"Authors\"") .SingleOrDefault(c => c.CleanName == "SomeTitle") .Should() .BeNull(); diff --git a/src/NzbDrone.Core.Test/Datastore/LazyLoadingFixture.cs b/src/NzbDrone.Core.Test/Datastore/LazyLoadingFixture.cs index d24e32609..e7a89a18e 100644 --- a/src/NzbDrone.Core.Test/Datastore/LazyLoadingFixture.cs +++ b/src/NzbDrone.Core.Test/Datastore/LazyLoadingFixture.cs @@ -78,7 +78,7 @@ namespace NzbDrone.Core.Test.Datastore public void should_lazy_load_author_for_trackfile() { var db = Mocker.Resolve(); - var tracks = db.Query(new SqlBuilder()).ToList(); + var tracks = db.Query(new SqlBuilder(db.DatabaseType)).ToList(); Assert.IsNotEmpty(tracks); foreach (var track in tracks) @@ -94,7 +94,7 @@ namespace NzbDrone.Core.Test.Datastore public void should_lazy_load_trackfile_if_not_joined() { var db = Mocker.Resolve(); - var tracks = db.Query(new SqlBuilder()).ToList(); + var tracks = db.Query(new SqlBuilder(db.DatabaseType)).ToList(); foreach (var track in tracks) { @@ -109,7 +109,7 @@ namespace NzbDrone.Core.Test.Datastore { var db = Mocker.Resolve(); var files = MediaFileRepository.Query(db, - new SqlBuilder() + new SqlBuilder(db.DatabaseType) .Join((t, a) => t.EditionId == a.Id) .Join((e, b) => e.BookId == b.Id) .Join((book, author) => book.AuthorMetadataId == author.AuthorMetadataId) diff --git a/src/NzbDrone.Core.Test/Datastore/WhereBuilderPostgresFixture.cs b/src/NzbDrone.Core.Test/Datastore/WhereBuilderPostgresFixture.cs new file mode 100644 index 000000000..ad0aa7881 --- /dev/null +++ b/src/NzbDrone.Core.Test/Datastore/WhereBuilderPostgresFixture.cs @@ -0,0 +1,211 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Books; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Datastore +{ + [TestFixture] + public class WhereBuilderPostgresFixture : CoreTest + { + private WhereBuilderPostgres _subject; + + [OneTimeSetUp] + public void MapTables() + { + // Generate table mapping + Mocker.Resolve(); + } + + private WhereBuilderPostgres Where(Expression> filter) + { + return new WhereBuilderPostgres(filter, true, 0); + } + + private WhereBuilderPostgres WhereMetadata(Expression> filter) + { + return new WhereBuilderPostgres(filter, true, 0); + } + + [Test] + public void postgres_where_equal_const() + { + _subject = Where(x => x.Id == 10); + + _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(10); + } + + [Test] + public void postgres_where_equal_variable() + { + var id = 10; + _subject = Where(x => x.Id == id); + + _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(id); + } + + [Test] + public void postgres_where_equal_property() + { + var author = new Author { Id = 10 }; + _subject = Where(x => x.Id == author.Id); + + _subject.Parameters.ParameterNames.Should().HaveCount(1); + _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(author.Id); + } + + [Test] + public void postgres_where_equal_joined_property() + { + _subject = Where(x => x.QualityProfile.Value.Id == 1); + + _subject.Parameters.ParameterNames.Should().HaveCount(1); + _subject.ToString().Should().Be($"(\"QualityProfiles\".\"Id\" = @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(1); + } + + [Test] + public void postgres_where_throws_without_concrete_condition_if_requiresConcreteCondition() + { + Expression> filter = (x, y) => x.Id == y.Id; + _subject = new WhereBuilderPostgres(filter, true, 0); + Assert.Throws(() => _subject.ToString()); + } + + [Test] + public void postgres_where_allows_abstract_condition_if_not_requiresConcreteCondition() + { + Expression> filter = (x, y) => x.Id == y.Id; + _subject = new WhereBuilderPostgres(filter, false, 0); + _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = \"Authors\".\"Id\")"); + } + + [Test] + public void postgres_where_string_is_null() + { + _subject = Where(x => x.CleanName == null); + + _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" IS NULL)"); + } + + [Test] + public void postgres_where_string_is_null_value() + { + string cleanName = null; + _subject = Where(x => x.CleanName == cleanName); + + _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" IS NULL)"); + } + + [Test] + public void postgres_where_equal_null_property() + { + var author = new Author { CleanName = null }; + _subject = Where(x => x.CleanName == author.CleanName); + + _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" IS NULL)"); + } + + [Test] + public void postgres_where_column_contains_string() + { + var test = "small"; + _subject = Where(x => x.CleanName.Contains(test)); + + _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" ILIKE '%' || @Clause1_P1 || '%')"); + _subject.Parameters.Get("Clause1_P1").Should().Be(test); + } + + [Test] + public void postgres_where_string_contains_column() + { + var test = "small"; + _subject = Where(x => test.Contains(x.CleanName)); + + _subject.ToString().Should().Be($"(@Clause1_P1 ILIKE '%' || \"Authors\".\"CleanName\" || '%')"); + _subject.Parameters.Get("Clause1_P1").Should().Be(test); + } + + [Test] + public void postgres_where_column_starts_with_string() + { + var test = "small"; + _subject = Where(x => x.CleanName.StartsWith(test)); + + _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" ILIKE @Clause1_P1 || '%')"); + _subject.Parameters.Get("Clause1_P1").Should().Be(test); + } + + [Test] + public void postgres_where_column_ends_with_string() + { + var test = "small"; + _subject = Where(x => x.CleanName.EndsWith(test)); + + _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" ILIKE '%' || @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(test); + } + + [Test] + public void postgres_where_in_list() + { + var list = new List { 1, 2, 3 }; + _subject = Where(x => list.Contains(x.Id)); + + _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = ANY (('{{1, 2, 3}}')))"); + } + + [Test] + public void postgres_where_in_list_2() + { + var list = new List { 1, 2, 3 }; + _subject = Where(x => x.CleanName == "test" && list.Contains(x.Id)); + + _subject.ToString().Should().Be($"((\"Authors\".\"CleanName\" = @Clause1_P1) AND (\"Authors\".\"Id\" = ANY (('{{1, 2, 3}}'))))"); + } + + [Test] + public void postgres_where_in_string_list() + { + var list = new List { "first", "second", "third" }; + + _subject = Where(x => list.Contains(x.CleanName)); + + _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" = ANY (@Clause1_P1))"); + } + + [Test] + public void enum_as_int() + { + _subject = WhereMetadata(x => x.Status == AuthorStatusType.Continuing); + + _subject.ToString().Should().Be($"(\"AuthorMetadata\".\"Status\" = @Clause1_P1)"); + } + + [Test] + public void enum_in_list() + { + var allowed = new List { AuthorStatusType.Continuing, AuthorStatusType.Ended }; + _subject = WhereMetadata(x => allowed.Contains(x.Status)); + + _subject.ToString().Should().Be($"(\"AuthorMetadata\".\"Status\" = ANY (@Clause1_P1))"); + } + + [Test] + public void enum_in_array() + { + var allowed = new AuthorStatusType[] { AuthorStatusType.Continuing, AuthorStatusType.Ended }; + _subject = WhereMetadata(x => allowed.Contains(x.Status)); + + _subject.ToString().Should().Be($"(\"AuthorMetadata\".\"Status\" = ANY (@Clause1_P1))"); + } + } +} diff --git a/src/NzbDrone.Core.Test/Datastore/WhereBuilderFixture.cs b/src/NzbDrone.Core.Test/Datastore/WhereBuilderSqliteFixture.cs similarity index 92% rename from src/NzbDrone.Core.Test/Datastore/WhereBuilderFixture.cs rename to src/NzbDrone.Core.Test/Datastore/WhereBuilderSqliteFixture.cs index 75ffb777a..b0163f597 100644 --- a/src/NzbDrone.Core.Test/Datastore/WhereBuilderFixture.cs +++ b/src/NzbDrone.Core.Test/Datastore/WhereBuilderSqliteFixture.cs @@ -11,9 +11,9 @@ using NzbDrone.Core.Test.Framework; namespace NzbDrone.Core.Test.Datastore { [TestFixture] - public class WhereBuilderFixture : CoreTest + public class WhereBuilderSqliteFixture : CoreTest { - private WhereBuilder _subject; + private WhereBuilderSqlite _subject; [OneTimeSetUp] public void MapTables() @@ -22,14 +22,14 @@ namespace NzbDrone.Core.Test.Datastore Mocker.Resolve(); } - private WhereBuilder Where(Expression> filter) + private WhereBuilderSqlite Where(Expression> filter) { - return new WhereBuilder(filter, true, 0); + return new WhereBuilderSqlite(filter, true, 0); } - private WhereBuilder WhereMetadata(Expression> filter) + private WhereBuilderSqlite WhereMetadata(Expression> filter) { - return new WhereBuilder(filter, true, 0); + return new WhereBuilderSqlite(filter, true, 0); } [Test] @@ -76,7 +76,7 @@ namespace NzbDrone.Core.Test.Datastore public void where_throws_without_concrete_condition_if_requiresConcreteCondition() { Expression> filter = (x, y) => x.Id == y.Id; - _subject = new WhereBuilder(filter, true, 0); + _subject = new WhereBuilderSqlite(filter, true, 0); Assert.Throws(() => _subject.ToString()); } @@ -84,7 +84,7 @@ namespace NzbDrone.Core.Test.Datastore public void where_allows_abstract_condition_if_not_requiresConcreteCondition() { Expression> filter = (x, y) => x.Id == y.Id; - _subject = new WhereBuilder(filter, false, 0); + _subject = new WhereBuilderSqlite(filter, false, 0); _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = \"Authors\".\"Id\")"); } diff --git a/src/NzbDrone.Core.Test/Framework/DbTest.cs b/src/NzbDrone.Core.Test/Framework/DbTest.cs index afb6dbdb6..fba791b15 100644 --- a/src/NzbDrone.Core.Test/Framework/DbTest.cs +++ b/src/NzbDrone.Core.Test/Framework/DbTest.cs @@ -1,12 +1,18 @@ -using System; +using System; using System.Collections.Generic; using System.Data.SQLite; +using System.IO; using System.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Npgsql; using NUnit.Framework; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration; using NzbDrone.Core.Datastore; using NzbDrone.Core.Datastore.Migration.Framework; +using NzbDrone.Test.Common.Datastore; namespace NzbDrone.Core.Test.Framework { @@ -47,6 +53,7 @@ namespace NzbDrone.Core.Test.Framework public abstract class DbTest : CoreTest { private ITestDatabase _db; + private DatabaseType _databaseType; protected virtual MigrationType MigrationType => MigrationType.Main; @@ -65,8 +72,7 @@ namespace NzbDrone.Core.Test.Framework protected virtual ITestDatabase WithTestDb(MigrationContext migrationContext) { - var factory = Mocker.Resolve(); - var database = factory.Create(migrationContext); + var database = CreateDatabase(migrationContext); Mocker.SetConstant(database); switch (MigrationType) @@ -98,6 +104,65 @@ namespace NzbDrone.Core.Test.Framework return testDb; } + private IDatabase CreateDatabase(MigrationContext migrationContext) + { + if (_databaseType == DatabaseType.PostgreSQL) + { + CreatePostgresDb(); + } + + var factory = Mocker.Resolve(); + + // If a special migration test or log migration then create new + if (migrationContext.BeforeMigration != null || _databaseType == DatabaseType.PostgreSQL) + { + return factory.Create(migrationContext); + } + + return CreateSqliteDatabase(factory, migrationContext); + } + + private void CreatePostgresDb() + { + var options = Mocker.Resolve>().Value; + PostgresDatabase.Create(options, MigrationType); + } + + private void DropPostgresDb() + { + var options = Mocker.Resolve>().Value; + PostgresDatabase.Drop(options, MigrationType); + } + + private IDatabase CreateSqliteDatabase(IDbFactory factory, MigrationContext migrationContext) + { + // Otherwise try to use a cached migrated db + var cachedDb = SqliteDatabase.GetCachedDb(migrationContext.MigrationType); + var testDb = GetTestSqliteDb(migrationContext.MigrationType); + if (File.Exists(cachedDb)) + { + TestLogger.Info($"Using cached initial database {cachedDb}"); + File.Copy(cachedDb, testDb); + return factory.Create(migrationContext); + } + else + { + var db = factory.Create(migrationContext); + GC.Collect(); + GC.WaitForPendingFinalizers(); + SQLiteConnection.ClearAllPools(); + + TestLogger.Info("Caching database"); + File.Copy(testDb, cachedDb); + return db; + } + } + + private string GetTestSqliteDb(MigrationType type) + { + return type == MigrationType.Main ? TestFolderInfo.GetDatabase() : TestFolderInfo.GetLogDatabase(); + } + protected virtual void SetupLogging() { Mocker.SetConstant(NullLoggerProvider.Instance); @@ -108,6 +173,13 @@ namespace NzbDrone.Core.Test.Framework WithTempAsAppPath(); SetupLogging(); + // populate the possible postgres options + var postgresOptions = PostgresDatabase.GetTestOptions(); + _databaseType = postgresOptions.Host.IsNotNullOrWhiteSpace() ? DatabaseType.PostgreSQL : DatabaseType.SQLite; + + // Set up remaining container services + Mocker.SetConstant(Options.Create(postgresOptions)); + Mocker.SetConstant(Mocker.Resolve()); Mocker.SetConstant(Mocker.Resolve()); Mocker.SetConstant(Mocker.Resolve()); @@ -127,12 +199,19 @@ namespace NzbDrone.Core.Test.Framework // Make sure there are no lingering connections. (When this happens it means we haven't disposed something properly) GC.Collect(); GC.WaitForPendingFinalizers(); + SQLiteConnection.ClearAllPools(); + NpgsqlConnection.ClearAllPools(); if (TestFolderInfo != null) { DeleteTempFolder(TestFolderInfo.AppDataFolder); } + + if (_databaseType == DatabaseType.PostgreSQL) + { + DropPostgresDb(); + } } } } diff --git a/src/NzbDrone.Core.Test/Framework/TestDatabase.cs b/src/NzbDrone.Core.Test/Framework/TestDatabase.cs index 3fbfdf028..5391cfb1f 100644 --- a/src/NzbDrone.Core.Test/Framework/TestDatabase.cs +++ b/src/NzbDrone.Core.Test/Framework/TestDatabase.cs @@ -23,6 +23,7 @@ namespace NzbDrone.Core.Test.Framework where T : ModelBase, new(); IDirectDataMapper GetDirectDataMapper(); IDbConnection OpenConnection(); + DatabaseType DatabaseType { get; } } public class TestDatabase : ITestDatabase @@ -30,6 +31,8 @@ namespace NzbDrone.Core.Test.Framework private readonly IDatabase _dbConnection; private readonly IEventAggregator _eventAggregator; + public DatabaseType DatabaseType => _dbConnection.DatabaseType; + public TestDatabase(IDatabase dbConnection) { _eventAggregator = new Mock().Object; diff --git a/src/NzbDrone.Core.Test/MusicTests/AlbumRepositoryTests/AlbumRepositoryFixture.cs b/src/NzbDrone.Core.Test/MusicTests/AlbumRepositoryTests/AlbumRepositoryFixture.cs index cffd00633..bd6c734fc 100644 --- a/src/NzbDrone.Core.Test/MusicTests/AlbumRepositoryTests/AlbumRepositoryFixture.cs +++ b/src/NzbDrone.Core.Test/MusicTests/AlbumRepositoryTests/AlbumRepositoryFixture.cs @@ -3,8 +3,10 @@ using System.Collections.Generic; using System.Linq; using FizzWare.NBuilder; using FluentAssertions; +using FluentAssertions.Equivalency; using NUnit.Framework; using NzbDrone.Core.Books; +using NzbDrone.Core.Datastore; using NzbDrone.Core.Test.Framework; namespace NzbDrone.Core.Test.MusicTests.BookRepositoryTests @@ -21,6 +23,13 @@ namespace NzbDrone.Core.Test.MusicTests.BookRepositoryTests [SetUp] public void Setup() { + AssertionOptions.AssertEquivalencyUsing(options => + { + options.Using(ctx => ctx.Subject.Should().BeCloseTo(ctx.Expectation.ToUniversalTime())).WhenTypeIs(); + options.Using(ctx => ctx.Subject.Should().BeCloseTo(ctx.Expectation.Value.ToUniversalTime())).WhenTypeIs(); + return options; + }); + _author = new Author { Name = "Alien Ant Farm", @@ -143,7 +152,7 @@ namespace NzbDrone.Core.Test.MusicTests.BookRepositoryTests GivenMultipleBooks(); var result = _bookRepo.GetNextBooks(new[] { _author.AuthorMetadataId }); - result.Should().BeEquivalentTo(_books.Take(1)); + result.Should().BeEquivalentTo(_books.Take(1), BookComparerOptions); } [Test] @@ -152,7 +161,11 @@ namespace NzbDrone.Core.Test.MusicTests.BookRepositoryTests GivenMultipleBooks(); var result = _bookRepo.GetLastBooks(new[] { _author.AuthorMetadataId }); - result.Should().BeEquivalentTo(_books.Skip(2).Take(1)); + result.Should().BeEquivalentTo(_books.Skip(2).Take(1), BookComparerOptions); } + + private EquivalencyAssertionOptions BookComparerOptions(EquivalencyAssertionOptions opts) => opts.ComparingByMembers() + .Excluding(ctx => ctx.SelectedMemberInfo.MemberType.IsGenericType && ctx.SelectedMemberInfo.MemberType.GetGenericTypeDefinition() == typeof(LazyLoaded<>)) + .Excluding(x => x.AuthorId); } } diff --git a/src/NzbDrone.Core.Test/MusicTests/ArtistRepositoryTests/ArtistRepositoryFixture.cs b/src/NzbDrone.Core.Test/MusicTests/ArtistRepositoryTests/ArtistRepositoryFixture.cs index 7f24b1524..d1cc675d7 100644 --- a/src/NzbDrone.Core.Test/MusicTests/ArtistRepositoryTests/ArtistRepositoryFixture.cs +++ b/src/NzbDrone.Core.Test/MusicTests/ArtistRepositoryTests/ArtistRepositoryFixture.cs @@ -3,9 +3,11 @@ using System.Collections.Generic; using System.Data.SQLite; using FizzWare.NBuilder; using FluentAssertions; +using Npgsql; using NUnit.Framework; using NzbDrone.Common.Extensions; using NzbDrone.Core.Books; +using NzbDrone.Core.Datastore; using NzbDrone.Core.Profiles.Metadata; using NzbDrone.Core.Profiles.Qualities; using NzbDrone.Core.Qualities; @@ -145,7 +147,14 @@ namespace NzbDrone.Core.Test.MusicTests.AuthorRepositoryTests _authorRepo.Insert(author1); Action insertDupe = () => _authorRepo.Insert(author2); - insertDupe.Should().Throw(); + if (Db.DatabaseType == DatabaseType.PostgreSQL) + { + insertDupe.Should().Throw(); + } + else + { + insertDupe.Should().Throw(); + } } } } diff --git a/src/NzbDrone.Core.Test/MusicTests/RefreshArtistServiceFixture.cs b/src/NzbDrone.Core.Test/MusicTests/RefreshArtistServiceFixture.cs index da477abd2..0a45d8d09 100644 --- a/src/NzbDrone.Core.Test/MusicTests/RefreshArtistServiceFixture.cs +++ b/src/NzbDrone.Core.Test/MusicTests/RefreshArtistServiceFixture.cs @@ -106,7 +106,7 @@ namespace NzbDrone.Core.Test.MusicTests private void GivenBooksForRefresh(List books) { Mocker.GetMock(MockBehavior.Strict) - .Setup(s => s.GetBooksForRefresh(It.IsAny(), It.IsAny>())) + .Setup(s => s.GetBooksForRefresh(It.IsAny(), It.IsAny>())) .Returns(books); } @@ -238,7 +238,7 @@ namespace NzbDrone.Core.Test.MusicTests Mocker.GetMock(MockBehavior.Strict) .InSequence(seq) - .Setup(x => x.GetBooksForRefresh(It.IsAny(), It.IsAny>())) + .Setup(x => x.GetBooksForRefresh(It.IsAny(), It.IsAny>())) .Returns(new List()); // Update called twice for a move/merge @@ -298,7 +298,7 @@ namespace NzbDrone.Core.Test.MusicTests Mocker.GetMock(MockBehavior.Strict) .InSequence(seq) - .Setup(x => x.GetBooksForRefresh(clash.AuthorMetadataId, It.IsAny>())) + .Setup(x => x.GetBooksForRefresh(clash.AuthorMetadataId, It.IsAny>())) .Returns(_books); // Update called twice for a move/merge diff --git a/src/NzbDrone.Core/AuthorStats/AuthorStatisticsRepository.cs b/src/NzbDrone.Core/AuthorStats/AuthorStatisticsRepository.cs index bbc099961..a788a1c03 100644 --- a/src/NzbDrone.Core/AuthorStats/AuthorStatisticsRepository.cs +++ b/src/NzbDrone.Core/AuthorStats/AuthorStatisticsRepository.cs @@ -16,7 +16,7 @@ namespace NzbDrone.Core.AuthorStats public class AuthorStatisticsRepository : IAuthorStatisticsRepository { - private const string _selectTemplate = "SELECT /**select**/ FROM Editions /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/ /**orderby**/"; + private const string _selectTemplate = "SELECT /**select**/ FROM \"Editions\" /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/ /**orderby**/"; private readonly IMainDatabase _database; @@ -45,14 +45,14 @@ namespace NzbDrone.Core.AuthorStats } } - private SqlBuilder Builder() => new SqlBuilder() - .Select(@"Authors.Id AS AuthorId, - Books.Id AS BookId, - SUM(COALESCE(BookFiles.Size, 0)) AS SizeOnDisk, - 1 AS TotalBookCount, - CASE WHEN BookFiles.Id IS NULL THEN 0 ELSE 1 END AS AvailableBookCount, - CASE WHEN (Books.Monitored = 1 AND (Books.ReleaseDate < @currentDate) OR Books.ReleaseDate IS NULL) OR BookFiles.Id IS NOT NULL THEN 1 ELSE 0 END AS BookCount, - CASE WHEN BookFiles.Id IS NULL THEN 0 ELSE COUNT(BookFiles.Id) END AS BookFileCount") + private SqlBuilder Builder() => new SqlBuilder(_database.DatabaseType) + .Select(@"""Authors"".""Id"" AS ""AuthorId"", + ""Books"".""Id"" AS ""BookId"", + SUM(COALESCE(""BookFiles"".""Size"", 0)) AS ""SizeOnDisk"", + 1 AS ""TotalBookCount"", + CASE WHEN MIN(""BookFiles"".""Id"") IS NULL THEN 0 ELSE 1 END AS ""AvailableBookCount"", + CASE WHEN (""Books"".""Monitored"" = true AND (""Books"".""ReleaseDate"" < @currentDate) OR ""Books"".""ReleaseDate"" IS NULL) OR MIN(""BookFiles"".""Id"") IS NOT NULL THEN 1 ELSE 0 END AS ""BookCount"", + CASE WHEN MIN(""BookFiles"".""Id"") IS NULL THEN 0 ELSE COUNT(""BookFiles"".""Id"") END AS ""BookFileCount""") .Join((e, b) => e.BookId == b.Id) .Join((book, author) => book.AuthorMetadataId == author.AuthorMetadataId) .LeftJoin((t, f) => t.Id == f.EditionId) diff --git a/src/NzbDrone.Core/Backup/BackupService.cs b/src/NzbDrone.Core/Backup/BackupService.cs index 8f7b96801..ba1956742 100644 --- a/src/NzbDrone.Core/Backup/BackupService.cs +++ b/src/NzbDrone.Core/Backup/BackupService.cs @@ -183,9 +183,12 @@ namespace NzbDrone.Core.Backup private void BackupDatabase() { - _logger.ProgressDebug("Backing up database"); + if (_maindDb.DatabaseType == DatabaseType.SQLite) + { + _logger.ProgressDebug("Backing up database"); - _makeDatabaseBackup.BackupDatabase(_maindDb, _backupTempFolder); + _makeDatabaseBackup.BackupDatabase(_maindDb, _backupTempFolder); + } } private void BackupConfigFile() diff --git a/src/NzbDrone.Core/Blocklisting/BlocklistRepository.cs b/src/NzbDrone.Core/Blocklisting/BlocklistRepository.cs index 7ecf79c2e..b5d4e11c2 100644 --- a/src/NzbDrone.Core/Blocklisting/BlocklistRepository.cs +++ b/src/NzbDrone.Core/Blocklisting/BlocklistRepository.cs @@ -34,7 +34,7 @@ namespace NzbDrone.Core.Blocklisting return Query(b => b.AuthorId == authorId); } - protected override SqlBuilder PagedBuilder() => new SqlBuilder() + protected override SqlBuilder PagedBuilder() => new SqlBuilder(_database.DatabaseType) .Join((b, m) => b.AuthorId == m.Id) .Join((l, r) => l.AuthorMetadataId == r.Id); protected override IEnumerable PagedQuery(SqlBuilder builder) => _database.QueryJoined(builder, diff --git a/src/NzbDrone.Core/Books/Repositories/AuthorRepository.cs b/src/NzbDrone.Core/Books/Repositories/AuthorRepository.cs index 312a76854..cc741a0c4 100644 --- a/src/NzbDrone.Core/Books/Repositories/AuthorRepository.cs +++ b/src/NzbDrone.Core/Books/Repositories/AuthorRepository.cs @@ -25,7 +25,7 @@ namespace NzbDrone.Core.Books { } - protected override SqlBuilder Builder() => new SqlBuilder() + protected override SqlBuilder Builder() => new SqlBuilder(_database.DatabaseType) .Join((a, m) => a.AuthorMetadataId == m.Id); protected override List Query(SqlBuilder builder) => Query(_database, builder).ToList(); @@ -60,7 +60,7 @@ namespace NzbDrone.Core.Books { using (var conn = _database.OpenConnection()) { - var strSql = "SELECT Id AS [Key], Path AS [Value] FROM Authors"; + var strSql = "SELECT \"Id\" AS \"Key\", \"Path\" AS \"Value\" FROM \"Authors\""; return conn.Query>(strSql).ToDictionary(x => x.Key, x => x.Value); } } diff --git a/src/NzbDrone.Core/Books/Repositories/BookRepository.cs b/src/NzbDrone.Core/Books/Repositories/BookRepository.cs index a7b8f79ce..6dec0f9d2 100644 --- a/src/NzbDrone.Core/Books/Repositories/BookRepository.cs +++ b/src/NzbDrone.Core/Books/Repositories/BookRepository.cs @@ -15,7 +15,7 @@ namespace NzbDrone.Core.Books List GetLastBooks(IEnumerable authorMetadataIds); List GetNextBooks(IEnumerable authorMetadataIds); List GetBooksByAuthorMetadataId(int authorMetadataId); - List GetBooksForRefresh(int authorMetadataId, IEnumerable foreignIds); + List GetBooksForRefresh(int authorMetadataId, List foreignIds); List GetBooksByFileIds(IEnumerable fileIds); Book FindByTitle(int authorMetadataId, string title); Book FindById(string foreignBookId); @@ -44,17 +44,35 @@ namespace NzbDrone.Core.Books public List GetLastBooks(IEnumerable authorMetadataIds) { var now = DateTime.UtcNow; - return Query(Builder().Where(x => authorMetadataIds.Contains(x.AuthorMetadataId) && x.ReleaseDate < now) - .GroupBy(x => x.AuthorMetadataId) - .Having("Books.ReleaseDate = MAX(Books.ReleaseDate)")); + + var inner = Builder() + .Select("MIN(\"Books\".\"Id\") as id, MAX(\"Books\".\"ReleaseDate\") as date") + .Where(x => authorMetadataIds.Contains(x.AuthorMetadataId) && x.ReleaseDate < now) + .GroupBy(x => x.AuthorMetadataId) + .AddSelectTemplate(typeof(Book)); + + var outer = Builder() + .Join($"({inner.RawSql}) ids on ids.id = \"Books\".\"Id\" and ids.date = \"Books\".\"ReleaseDate\"") + .AddParameters(inner.Parameters); + + return Query(outer); } public List GetNextBooks(IEnumerable authorMetadataIds) { var now = DateTime.UtcNow; - return Query(Builder().Where(x => authorMetadataIds.Contains(x.AuthorMetadataId) && x.ReleaseDate > now) - .GroupBy(x => x.AuthorMetadataId) - .Having("Books.ReleaseDate = MIN(Books.ReleaseDate)")); + + var inner = Builder() + .Select("MIN(\"Books\".\"Id\") as id, MIN(\"Books\".\"ReleaseDate\") as date") + .Where(x => authorMetadataIds.Contains(x.AuthorMetadataId) && x.ReleaseDate > now) + .GroupBy(x => x.AuthorMetadataId) + .AddSelectTemplate(typeof(Book)); + + var outer = Builder() + .Join($"({inner.RawSql}) ids on ids.id = \"Books\".\"Id\" and ids.date = \"Books\".\"ReleaseDate\"") + .AddParameters(inner.Parameters); + + return Query(outer); } public List GetBooksByAuthorMetadataId(int authorMetadataId) @@ -62,14 +80,14 @@ namespace NzbDrone.Core.Books return Query(s => s.AuthorMetadataId == authorMetadataId); } - public List GetBooksForRefresh(int authorMetadataId, IEnumerable foreignIds) + public List GetBooksForRefresh(int authorMetadataId, List foreignIds) { return Query(a => a.AuthorMetadataId == authorMetadataId || foreignIds.Contains(a.ForeignBookId)); } public List GetBooksByFileIds(IEnumerable fileIds) { - return Query(new SqlBuilder() + return Query(new SqlBuilder(_database.DatabaseType) .Join((b, e) => b.Id == e.BookId) .Join((l, r) => l.Id == r.EditionId) .Where(f => fileIds.Contains(f.Id))) @@ -125,7 +143,7 @@ namespace NzbDrone.Core.Books { foreach (var belowCutoff in profile.QualityIds) { - clauses.Add(string.Format("(Authors.[QualityProfileId] = {0} AND BookFiles.Quality LIKE '%_quality_: {1},%')", profile.ProfileId, belowCutoff)); + clauses.Add(string.Format("(\"Authors\".\"QualityProfileId\" = {0} AND \"BookFiles\".\"Quality\" LIKE '%_quality_: {1},%')", profile.ProfileId, belowCutoff)); } } @@ -136,7 +154,7 @@ namespace NzbDrone.Core.Books { pagingSpec.Records = GetPagedRecords(BooksWhereCutoffUnmetBuilder(qualitiesBelowCutoff), pagingSpec, PagedQuery); - var countTemplate = $"SELECT COUNT(*) FROM (SELECT /**select**/ FROM {TableMapping.Mapper.TableNameMapping(typeof(Book))} /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/)"; + var countTemplate = $"SELECT COUNT(*) FROM (SELECT /**select**/ FROM \"{TableMapping.Mapper.TableNameMapping(typeof(Book))}\" /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/) AS \"Inner\""; pagingSpec.TotalRecords = GetPagedRecordCount(BooksWhereCutoffUnmetBuilder(qualitiesBelowCutoff).Select(typeof(Book)), pagingSpec, countTemplate); return pagingSpec; diff --git a/src/NzbDrone.Core/Books/Repositories/EditionRepository.cs b/src/NzbDrone.Core/Books/Repositories/EditionRepository.cs index 89b31fc45..04e408ae6 100644 --- a/src/NzbDrone.Core/Books/Repositories/EditionRepository.cs +++ b/src/NzbDrone.Core/Books/Repositories/EditionRepository.cs @@ -15,7 +15,7 @@ namespace NzbDrone.Core.Books List FindByAuthor(int id); List FindByAuthorMetadataId(int id, bool onlyMonitored); Edition FindByTitle(int authorMetadataId, string title); - List GetEditionsForRefresh(int bookId, IEnumerable foreignEditionIds); + List GetEditionsForRefresh(int bookId, List foreignEditionIds); List SetMonitored(Edition edition); } @@ -38,7 +38,7 @@ namespace NzbDrone.Core.Books return edition; } - public List GetEditionsForRefresh(int bookId, IEnumerable foreignEditionIds) + public List GetEditionsForRefresh(int bookId, List foreignEditionIds) { return Query(r => r.BookId == bookId || foreignEditionIds.Contains(r.ForeignEditionId)); } @@ -47,7 +47,7 @@ namespace NzbDrone.Core.Books { // populate the books and author metadata also // this hopefully speeds up the track matching a lot - var builder = new SqlBuilder() + var builder = new SqlBuilder(_database.DatabaseType) .LeftJoin((e, b) => e.BookId == b.Id) .LeftJoin((b, a) => b.AuthorMetadataId == a.Id) .Where(r => r.BookId == id); diff --git a/src/NzbDrone.Core/Books/Repositories/SeriesRepository.cs b/src/NzbDrone.Core/Books/Repositories/SeriesRepository.cs index 0a89d47f0..9ac6e068f 100644 --- a/src/NzbDrone.Core/Books/Repositories/SeriesRepository.cs +++ b/src/NzbDrone.Core/Books/Repositories/SeriesRepository.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Books public interface ISeriesRepository : IBasicRepository { Series FindById(string foreignSeriesId); - List FindById(IEnumerable foreignSeriesId); + List FindById(List foreignSeriesId); List GetByAuthorMetadataId(int authorMetadataId); List GetByAuthorId(int authorId); } @@ -25,7 +25,7 @@ namespace NzbDrone.Core.Books return Query(x => x.ForeignSeriesId == foreignSeriesId).SingleOrDefault(); } - public List FindById(IEnumerable foreignSeriesId) + public List FindById(List foreignSeriesId) { return Query(x => foreignSeriesId.Contains(x.ForeignSeriesId)); } diff --git a/src/NzbDrone.Core/Books/Services/BookService.cs b/src/NzbDrone.Core/Books/Services/BookService.cs index 34a35e92d..7dc3b2c14 100644 --- a/src/NzbDrone.Core/Books/Services/BookService.cs +++ b/src/NzbDrone.Core/Books/Services/BookService.cs @@ -19,7 +19,7 @@ namespace NzbDrone.Core.Books List GetNextBooksByAuthorMetadataId(IEnumerable authorMetadataIds); List GetLastBooksByAuthorMetadataId(IEnumerable authorMetadataIds); List GetBooksByAuthorMetadataId(int authorMetadataId); - List GetBooksForRefresh(int authorMetadataId, IEnumerable foreignIds); + List GetBooksForRefresh(int authorMetadataId, List foreignIds); List GetBooksByFileIds(IEnumerable fileIds); Book AddBook(Book newBook, bool doRefresh = true); Book FindById(string foreignId); @@ -206,7 +206,7 @@ namespace NzbDrone.Core.Books return _bookRepository.GetBooksByAuthorMetadataId(authorMetadataId).ToList(); } - public List GetBooksForRefresh(int authorMetadataId, IEnumerable foreignIds) + public List GetBooksForRefresh(int authorMetadataId, List foreignIds) { return _bookRepository.GetBooksForRefresh(authorMetadataId, foreignIds); } diff --git a/src/NzbDrone.Core/Books/Services/EditionService.cs b/src/NzbDrone.Core/Books/Services/EditionService.cs index 421a537ff..a7424fe0e 100644 --- a/src/NzbDrone.Core/Books/Services/EditionService.cs +++ b/src/NzbDrone.Core/Books/Services/EditionService.cs @@ -16,7 +16,7 @@ namespace NzbDrone.Core.Books void InsertMany(List editions); void UpdateMany(List editions); void DeleteMany(List editions); - List GetEditionsForRefresh(int bookId, IEnumerable foreignEditionIds); + List GetEditionsForRefresh(int bookId, List foreignEditionIds); List GetEditionsByBook(int bookId); List GetEditionsByAuthor(int authorId); Edition FindByTitle(int authorMetadataId, string title); @@ -72,7 +72,7 @@ namespace NzbDrone.Core.Books } } - public List GetEditionsForRefresh(int bookId, IEnumerable foreignEditionIds) + public List GetEditionsForRefresh(int bookId, List foreignEditionIds) { return _editionRepository.GetEditionsForRefresh(bookId, foreignEditionIds); } diff --git a/src/NzbDrone.Core/Books/Services/RefreshAuthorService.cs b/src/NzbDrone.Core/Books/Services/RefreshAuthorService.cs index 1580bfb5b..020903a25 100644 --- a/src/NzbDrone.Core/Books/Services/RefreshAuthorService.cs +++ b/src/NzbDrone.Core/Books/Services/RefreshAuthorService.cs @@ -238,7 +238,7 @@ namespace NzbDrone.Core.Books protected override List GetLocalChildren(Author entity, List remoteChildren) { return _bookService.GetBooksForRefresh(entity.AuthorMetadataId, - remoteChildren.Select(x => x.ForeignBookId)); + remoteChildren.Select(x => x.ForeignBookId).ToList()); } protected override Tuple> GetMatchingExistingChildren(List existingChildren, Book remote) diff --git a/src/NzbDrone.Core/Books/Services/RefreshBookService.cs b/src/NzbDrone.Core/Books/Services/RefreshBookService.cs index c5478c7e6..2b44bc6bb 100644 --- a/src/NzbDrone.Core/Books/Services/RefreshBookService.cs +++ b/src/NzbDrone.Core/Books/Services/RefreshBookService.cs @@ -250,7 +250,7 @@ namespace NzbDrone.Core.Books protected override List GetLocalChildren(Book entity, List remoteChildren) { - return _editionService.GetEditionsForRefresh(entity.Id, remoteChildren.Select(x => x.ForeignEditionId)); + return _editionService.GetEditionsForRefresh(entity.Id, remoteChildren.Select(x => x.ForeignEditionId).ToList()); } protected override Tuple> GetMatchingExistingChildren(List existingChildren, Edition remote) diff --git a/src/NzbDrone.Core/Books/Services/RefreshSeriesService.cs b/src/NzbDrone.Core/Books/Services/RefreshSeriesService.cs index 2fefe9174..eeafe30d0 100644 --- a/src/NzbDrone.Core/Books/Services/RefreshSeriesService.cs +++ b/src/NzbDrone.Core/Books/Services/RefreshSeriesService.cs @@ -132,7 +132,7 @@ namespace NzbDrone.Core.Books var updated = false; var existingByAuthor = _seriesService.GetByAuthorMetadataId(authorMetadataId); - var existingBySeries = _seriesService.FindById(remoteSeries.Select(x => x.ForeignSeriesId)); + var existingBySeries = _seriesService.FindById(remoteSeries.Select(x => x.ForeignSeriesId).ToList()); var existing = existingByAuthor.Concat(existingBySeries).GroupBy(x => x.ForeignSeriesId).Select(x => x.First()).ToList(); var books = _bookService.GetBooksByAuthorMetadataId(authorMetadataId); diff --git a/src/NzbDrone.Core/Books/Services/SeriesService.cs b/src/NzbDrone.Core/Books/Services/SeriesService.cs index 4176bd1c4..9ee7f409c 100644 --- a/src/NzbDrone.Core/Books/Services/SeriesService.cs +++ b/src/NzbDrone.Core/Books/Services/SeriesService.cs @@ -5,7 +5,7 @@ namespace NzbDrone.Core.Books public interface ISeriesService { Series FindById(string foreignSeriesId); - List FindById(IEnumerable foreignSeriesId); + List FindById(List foreignSeriesId); List GetByAuthorMetadataId(int authorMetadataId); List GetByAuthorId(int authorId); void Delete(int seriesId); @@ -27,7 +27,7 @@ namespace NzbDrone.Core.Books return _seriesRepository.FindById(foreignSeriesId); } - public List FindById(IEnumerable foreignSeriesId) + public List FindById(List foreignSeriesId) { return _seriesRepository.FindById(foreignSeriesId); } diff --git a/src/NzbDrone.Core/Configuration/ConfigFileProvider.cs b/src/NzbDrone.Core/Configuration/ConfigFileProvider.cs index b56c4b2fd..0a1c9bcf8 100644 --- a/src/NzbDrone.Core/Configuration/ConfigFileProvider.cs +++ b/src/NzbDrone.Core/Configuration/ConfigFileProvider.cs @@ -4,12 +4,14 @@ using System.IO; using System.Linq; using System.Xml; using System.Xml.Linq; +using Microsoft.Extensions.Options; using NzbDrone.Common.Cache; using NzbDrone.Common.Disk; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; using NzbDrone.Core.Authentication; using NzbDrone.Core.Configuration.Events; +using NzbDrone.Core.Datastore; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; @@ -18,7 +20,7 @@ using NzbDrone.Core.Update; namespace NzbDrone.Core.Configuration { public interface IConfigFileProvider : IHandleAsync, - IExecute + IExecute { Dictionary GetConfigDictionary(); void SaveConfigDictionary(Dictionary configValues); @@ -48,6 +50,13 @@ namespace NzbDrone.Core.Configuration string SyslogServer { get; } int SyslogPort { get; } string SyslogLevel { get; } + string PostgresHost { get; } + int PostgresPort { get; } + string PostgresUser { get; } + string PostgresPassword { get; } + string PostgresMainDb { get; } + string PostgresLogDb { get; } + string PostgresCacheDb { get; } } public class ConfigFileProvider : IConfigFileProvider @@ -57,6 +66,7 @@ namespace NzbDrone.Core.Configuration private readonly IEventAggregator _eventAggregator; private readonly IDiskProvider _diskProvider; private readonly ICached _cache; + private readonly PostgresOptions _postgresOptions; private readonly string _configFile; @@ -65,12 +75,14 @@ namespace NzbDrone.Core.Configuration public ConfigFileProvider(IAppFolderInfo appFolderInfo, ICacheManager cacheManager, IEventAggregator eventAggregator, - IDiskProvider diskProvider) + IDiskProvider diskProvider, + IOptions postgresOptions) { _cache = cacheManager.GetCache(GetType()); _eventAggregator = eventAggregator; _diskProvider = diskProvider; _configFile = appFolderInfo.GetConfigPath(); + _postgresOptions = postgresOptions.Value; } public Dictionary GetConfigDictionary() @@ -184,6 +196,13 @@ namespace NzbDrone.Core.Configuration public string LogLevel => GetValue("LogLevel", "info"); public string ConsoleLogLevel => GetValue("ConsoleLogLevel", string.Empty, persist: false); + public string PostgresHost => _postgresOptions?.Host ?? GetValue("PostgresHost", string.Empty, persist: false); + public string PostgresUser => _postgresOptions?.User ?? GetValue("PostgresUser", string.Empty, persist: false); + public string PostgresPassword => _postgresOptions?.Password ?? GetValue("PostgresPassword", string.Empty, persist: false); + public string PostgresMainDb => _postgresOptions?.MainDb ?? GetValue("PostgresMainDb", "readarr-main", persist: false); + public string PostgresLogDb => _postgresOptions?.LogDb ?? GetValue("PostgresLogDb", "readarr-log", persist: false); + public string PostgresCacheDb => _postgresOptions?.CacheDb ?? GetValue("PostgresCacheDb", "readarr-cache", persist: false); + public int PostgresPort => (_postgresOptions?.Port ?? 0) != 0 ? _postgresOptions.Port : GetValueInt("PostgresPort", 5432, persist: false); public bool LogSql => GetValueBoolean("LogSql", false, persist: false); public int LogRotate => GetValueInt("LogRotate", 50, persist: false); public bool FilterSentryEvents => GetValueBoolean("FilterSentryEvents", true, persist: false); diff --git a/src/NzbDrone.Core/Datastore/BasicRepository.cs b/src/NzbDrone.Core/Datastore/BasicRepository.cs index b9d3ea400..20880e0fd 100644 --- a/src/NzbDrone.Core/Datastore/BasicRepository.cs +++ b/src/NzbDrone.Core/Datastore/BasicRepository.cs @@ -68,7 +68,7 @@ namespace NzbDrone.Core.Datastore _updateSql = GetUpdateSql(_properties); } - protected virtual SqlBuilder Builder() => new SqlBuilder(); + protected virtual SqlBuilder Builder() => new SqlBuilder(_database.DatabaseType); protected virtual List Query(SqlBuilder builder) => _database.Query(builder).ToList(); @@ -80,7 +80,7 @@ namespace NzbDrone.Core.Datastore { using (var conn = _database.OpenConnection()) { - return conn.ExecuteScalar($"SELECT COUNT(*) FROM {_table}"); + return conn.ExecuteScalar($"SELECT COUNT(*) FROM \"{_table}\""); } } @@ -176,6 +176,11 @@ namespace NzbDrone.Core.Datastore } } + if (_database.DatabaseType == DatabaseType.PostgreSQL) + { + return $"INSERT INTO \"{_table}\" ({sbColumnList.ToString()}) VALUES ({sbParameterList.ToString()}) RETURNING \"Id\""; + } + return $"INSERT INTO {_table} ({sbColumnList.ToString()}) VALUES ({sbParameterList.ToString()}); SELECT last_insert_rowid() id"; } @@ -194,7 +199,8 @@ namespace NzbDrone.Core.Datastore throw; } - var id = (int)multi.Read().First().id; + var multiRead = multi.Read(); + var id = (int)(multiRead.First().id ?? multiRead.First().Id); _keyProperty.SetValue(model, id); return model; @@ -305,7 +311,7 @@ namespace NzbDrone.Core.Datastore { using (var conn = _database.OpenConnection()) { - conn.Execute($"DELETE FROM [{_table}]"); + conn.Execute($"DELETE FROM \"{_table}\""); } if (vacuum) @@ -364,7 +370,7 @@ namespace NzbDrone.Core.Datastore private string GetUpdateSql(List propertiesToUpdate) { var sb = new StringBuilder(); - sb.AppendFormat("UPDATE {0} SET ", _table); + sb.AppendFormat("UPDATE \"{0}\" SET ", _table); for (var i = 0; i < propertiesToUpdate.Count; i++) { @@ -446,9 +452,10 @@ namespace NzbDrone.Core.Datastore pagingSpec.SortKey = $"{_table}.{_keyProperty.Name}"; } + var sortKey = TableMapping.Mapper.GetSortKey(pagingSpec.SortKey); var sortDirection = pagingSpec.SortDirection == SortDirection.Descending ? "DESC" : "ASC"; - var pagingOffset = (pagingSpec.Page - 1) * pagingSpec.PageSize; - builder.OrderBy($"{pagingSpec.SortKey} {sortDirection} LIMIT {pagingSpec.PageSize} OFFSET {pagingOffset}"); + var pagingOffset = Math.Max(pagingSpec.Page - 1, 0) * pagingSpec.PageSize; + builder.OrderBy($"\"{sortKey}\" {sortDirection} LIMIT {pagingSpec.PageSize} OFFSET {pagingOffset}"); return queryFunc(builder).ToList(); } diff --git a/src/NzbDrone.Core/Datastore/CacheDatabase.cs b/src/NzbDrone.Core/Datastore/CacheDatabase.cs index 50c1be8c1..827827d07 100644 --- a/src/NzbDrone.Core/Datastore/CacheDatabase.cs +++ b/src/NzbDrone.Core/Datastore/CacheDatabase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Data; namespace NzbDrone.Core.Datastore @@ -10,10 +10,12 @@ namespace NzbDrone.Core.Datastore public class CacheDatabase : ICacheDatabase { private readonly IDatabase _database; + private readonly DatabaseType _databaseType; public CacheDatabase(IDatabase database) { _database = database; + _databaseType = _database == null ? DatabaseType.SQLite : _database.DatabaseType; } public IDbConnection OpenConnection() @@ -25,6 +27,8 @@ namespace NzbDrone.Core.Datastore public int Migration => _database.Migration; + public DatabaseType DatabaseType => _databaseType; + public void Vacuum() { _database.Vacuum(); diff --git a/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs b/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs index 1e6c0cc2d..4f58d32e6 100644 --- a/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs +++ b/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs @@ -1,7 +1,9 @@ using System; using System.Data.SQLite; +using Npgsql; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration; namespace NzbDrone.Core.Datastore { @@ -15,11 +17,20 @@ namespace NzbDrone.Core.Datastore public class ConnectionStringFactory : IConnectionStringFactory { - public ConnectionStringFactory(IAppFolderInfo appFolderInfo) + private readonly IConfigFileProvider _configFileProvider; + + public ConnectionStringFactory(IAppFolderInfo appFolderInfo, IConfigFileProvider configFileProvider) { - MainDbConnectionString = GetConnectionString(appFolderInfo.GetDatabase()); - LogDbConnectionString = GetConnectionString(appFolderInfo.GetLogDatabase()); - CacheDbConnectionString = GetConnectionString(appFolderInfo.GetCacheDatabase()); + _configFileProvider = configFileProvider; + + MainDbConnectionString = _configFileProvider.PostgresHost.IsNotNullOrWhiteSpace() ? GetPostgresConnectionString(_configFileProvider.PostgresMainDb) : + GetConnectionString(appFolderInfo.GetDatabase()); + + LogDbConnectionString = _configFileProvider.PostgresHost.IsNotNullOrWhiteSpace() ? GetPostgresConnectionString(_configFileProvider.PostgresLogDb) : + GetConnectionString(appFolderInfo.GetLogDatabase()); + + CacheDbConnectionString = _configFileProvider.PostgresHost.IsNotNullOrWhiteSpace() ? GetPostgresConnectionString(_configFileProvider.PostgresCacheDb) : + GetConnectionString(appFolderInfo.GetCacheDatabase()); } public string MainDbConnectionString { get; private set; } @@ -51,5 +62,19 @@ namespace NzbDrone.Core.Datastore return connectionBuilder.ConnectionString; } + + private string GetPostgresConnectionString(string dbName) + { + var connectionBuilder = new NpgsqlConnectionStringBuilder(); + + connectionBuilder.Database = dbName; + connectionBuilder.Host = _configFileProvider.PostgresHost; + connectionBuilder.Username = _configFileProvider.PostgresUser; + connectionBuilder.Password = _configFileProvider.PostgresPassword; + connectionBuilder.Port = _configFileProvider.PostgresPort; + connectionBuilder.Enlist = false; + + return connectionBuilder.ConnectionString; + } } } diff --git a/src/NzbDrone.Core/Datastore/Database.cs b/src/NzbDrone.Core/Datastore/Database.cs index a9b6e807f..433fef0b6 100644 --- a/src/NzbDrone.Core/Datastore/Database.cs +++ b/src/NzbDrone.Core/Datastore/Database.cs @@ -1,5 +1,6 @@ using System; using System.Data; +using System.Text.RegularExpressions; using Dapper; using NLog; using NzbDrone.Common.Instrumentation; @@ -11,6 +12,7 @@ namespace NzbDrone.Core.Datastore IDbConnection OpenConnection(); Version Version { get; } int Migration { get; } + DatabaseType DatabaseType { get; } void Vacuum(); } @@ -32,13 +34,44 @@ namespace NzbDrone.Core.Datastore return _datamapperFactory(); } + public DatabaseType DatabaseType + { + get + { + using (var db = _datamapperFactory()) + { + if (db.ConnectionString.Contains(".db")) + { + return DatabaseType.SQLite; + } + else + { + return DatabaseType.PostgreSQL; + } + } + } + } + public Version Version { get { using (var db = _datamapperFactory()) { - var version = db.QueryFirstOrDefault("SELECT sqlite_version()"); + string version; + + try + { + version = db.QueryFirstOrDefault("SHOW server_version"); + + //Postgres can return extra info about operating system on version call, ignore this + version = Regex.Replace(version, @"\(.*?\)", ""); + } + catch + { + version = db.QueryFirstOrDefault("SELECT sqlite_version()"); + } + return new Version(version); } } @@ -50,7 +83,7 @@ namespace NzbDrone.Core.Datastore { using (var db = _datamapperFactory()) { - return db.QueryFirstOrDefault("SELECT version from VersionInfo ORDER BY version DESC LIMIT 1"); + return db.QueryFirstOrDefault("SELECT \"Version\" from \"VersionInfo\" ORDER BY \"Version\" DESC LIMIT 1"); } } } @@ -73,4 +106,10 @@ namespace NzbDrone.Core.Datastore } } } + + public enum DatabaseType + { + SQLite, + PostgreSQL + } } diff --git a/src/NzbDrone.Core/Datastore/DbFactory.cs b/src/NzbDrone.Core/Datastore/DbFactory.cs index 2db023b5f..a1fb67f5a 100644 --- a/src/NzbDrone.Core/Datastore/DbFactory.cs +++ b/src/NzbDrone.Core/Datastore/DbFactory.cs @@ -1,6 +1,8 @@ using System; +using System.Data.Common; using System.Data.SQLite; using NLog; +using Npgsql; using NzbDrone.Common.Disk; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Exceptions; @@ -92,10 +94,19 @@ namespace NzbDrone.Core.Datastore var db = new Database(migrationContext.MigrationType.ToString(), () => { - var conn = SQLiteFactory.Instance.CreateConnection(); - conn.ConnectionString = connectionString; - conn.Open(); + DbConnection conn; + if (connectionString.Contains(".db")) + { + conn = SQLiteFactory.Instance.CreateConnection(); + conn.ConnectionString = connectionString; + } + else + { + conn = new NpgsqlConnection(connectionString); + } + + conn.Open(); return conn; }); diff --git a/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs b/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs index bb13e09d0..3bff36b7a 100644 --- a/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs +++ b/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs @@ -20,12 +20,12 @@ namespace NzbDrone.Core.Datastore public static SqlBuilder Select(this SqlBuilder builder, params Type[] types) { - return builder.Select(types.Select(x => TableMapping.Mapper.TableNameMapping(x) + ".*").Join(", ")); + return builder.Select(types.Select(x => $"\"{TableMapping.Mapper.TableNameMapping(x)}\".*").Join(", ")); } public static SqlBuilder SelectDistinct(this SqlBuilder builder, params Type[] types) { - return builder.Select("DISTINCT " + types.Select(x => TableMapping.Mapper.TableNameMapping(x) + ".*").Join(", ")); + return builder.Select("DISTINCT " + types.Select(x => $"\"{TableMapping.Mapper.TableNameMapping(x)}\".*").Join(", ")); } public static SqlBuilder SelectCount(this SqlBuilder builder) @@ -42,41 +42,48 @@ namespace NzbDrone.Core.Datastore public static SqlBuilder Where(this SqlBuilder builder, Expression> filter) { - var wb = new WhereBuilder(filter, true, builder.Sequence); + var wb = GetWhereBuilder(builder.DatabaseType, filter, true, builder.Sequence); + + return builder.Where(wb.ToString(), wb.Parameters); + } + + public static SqlBuilder WherePostgres(this SqlBuilder builder, Expression> filter) + { + var wb = new WhereBuilderPostgres(filter, true, builder.Sequence); return builder.Where(wb.ToString(), wb.Parameters); } public static SqlBuilder OrWhere(this SqlBuilder builder, Expression> filter) { - var wb = new WhereBuilder(filter, true, builder.Sequence); + var wb = GetWhereBuilder(builder.DatabaseType, filter, true, builder.Sequence); return builder.OrWhere(wb.ToString(), wb.Parameters); } public static SqlBuilder Join(this SqlBuilder builder, Expression> filter) { - var wb = new WhereBuilder(filter, false, builder.Sequence); + var wb = GetWhereBuilder(builder.DatabaseType, filter, false, builder.Sequence); var rightTable = TableMapping.Mapper.TableNameMapping(typeof(TRight)); - return builder.Join($"{rightTable} ON {wb.ToString()}"); + return builder.Join($"\"{rightTable}\" ON {wb.ToString()}"); } public static SqlBuilder LeftJoin(this SqlBuilder builder, Expression> filter) { - var wb = new WhereBuilder(filter, false, builder.Sequence); + var wb = GetWhereBuilder(builder.DatabaseType, filter, false, builder.Sequence); var rightTable = TableMapping.Mapper.TableNameMapping(typeof(TRight)); - return builder.LeftJoin($"{rightTable} ON {wb.ToString()}"); + return builder.LeftJoin($"\"{rightTable}\" ON {wb.ToString()}"); } public static SqlBuilder GroupBy(this SqlBuilder builder, Expression> property) { var table = TableMapping.Mapper.TableNameMapping(typeof(TModel)); var propName = property.GetMemberName().Name; - return builder.GroupBy($"{table}.{propName}"); + return builder.GroupBy($"\"{table}\".\"{propName}\""); } public static SqlBuilder.Template AddSelectTemplate(this SqlBuilder builder, Type type) @@ -138,6 +145,18 @@ namespace NzbDrone.Core.Datastore return sb.ToString(); } + private static WhereBuilder GetWhereBuilder(DatabaseType databaseType, Expression filter, bool requireConcrete, int seq) + { + if (databaseType == DatabaseType.PostgreSQL) + { + return new WhereBuilderPostgres(filter, requireConcrete, seq); + } + else + { + return new WhereBuilderSqlite(filter, requireConcrete, seq); + } + } + private static Dictionary ToDictionary(this DynamicParameters dynamicParams) { var argsDictionary = new Dictionary(); diff --git a/src/NzbDrone.Core/Datastore/LogDatabase.cs b/src/NzbDrone.Core/Datastore/LogDatabase.cs index f992c8bbe..a770c2661 100644 --- a/src/NzbDrone.Core/Datastore/LogDatabase.cs +++ b/src/NzbDrone.Core/Datastore/LogDatabase.cs @@ -10,10 +10,12 @@ namespace NzbDrone.Core.Datastore public class LogDatabase : ILogDatabase { private readonly IDatabase _database; + private readonly DatabaseType _databaseType; public LogDatabase(IDatabase database) { _database = database; + _databaseType = _database == null ? DatabaseType.SQLite : _database.DatabaseType; } public IDbConnection OpenConnection() @@ -25,6 +27,8 @@ namespace NzbDrone.Core.Datastore public int Migration => _database.Migration; + public DatabaseType DatabaseType => _databaseType; + public void Vacuum() { _database.Vacuum(); diff --git a/src/NzbDrone.Core/Datastore/MainDatabase.cs b/src/NzbDrone.Core/Datastore/MainDatabase.cs index 4a9d3298c..521293299 100644 --- a/src/NzbDrone.Core/Datastore/MainDatabase.cs +++ b/src/NzbDrone.Core/Datastore/MainDatabase.cs @@ -10,10 +10,12 @@ namespace NzbDrone.Core.Datastore public class MainDatabase : IMainDatabase { private readonly IDatabase _database; + private readonly DatabaseType _databaseType; public MainDatabase(IDatabase database) { _database = database; + _databaseType = _database == null ? DatabaseType.SQLite : _database.DatabaseType; } public IDbConnection OpenConnection() @@ -25,6 +27,8 @@ namespace NzbDrone.Core.Datastore public int Migration => _database.Migration; + public DatabaseType DatabaseType => _databaseType; + public void Vacuum() { _database.Vacuum(); diff --git a/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs b/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs index a152edc4d..e9a3774d1 100644 --- a/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs +++ b/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs @@ -37,6 +37,22 @@ namespace NzbDrone.Core.Datastore.Migration .WithColumn("MetadataProfileId").AsInt32().WithDefaultValue(1) .WithColumn("AuthorMetadataId").AsInt32().Unique(); + Create.TableForModel("Books") + .WithColumn("AuthorMetadataId").AsInt32().WithDefaultValue(0) + .WithColumn("ForeignBookId").AsString().Indexed() + .WithColumn("TitleSlug").AsString().Unique() + .WithColumn("Title").AsString() + .WithColumn("ReleaseDate").AsDateTime().Nullable() + .WithColumn("Links").AsString().Nullable() + .WithColumn("Genres").AsString().Nullable() + .WithColumn("Ratings").AsString().Nullable() + .WithColumn("CleanTitle").AsString().Indexed() + .WithColumn("Monitored").AsBoolean() + .WithColumn("AnyEditionOk").AsBoolean() + .WithColumn("LastInfoSync").AsDateTime().Nullable() + .WithColumn("Added").AsDateTime().Nullable() + .WithColumn("AddOptions").AsString().Nullable(); + Create.TableForModel("Series") .WithColumn("ForeignSeriesId").AsString().Unique() .WithColumn("Title").AsString() @@ -68,22 +84,6 @@ namespace NzbDrone.Core.Datastore.Migration .WithColumn("Ratings").AsString().Nullable() .WithColumn("Aliases").AsString().WithDefaultValue("[]"); - Create.TableForModel("Books") - .WithColumn("AuthorMetadataId").AsInt32().WithDefaultValue(0) - .WithColumn("ForeignBookId").AsString().Indexed() - .WithColumn("TitleSlug").AsString().Unique() - .WithColumn("Title").AsString() - .WithColumn("ReleaseDate").AsDateTime().Nullable() - .WithColumn("Links").AsString().Nullable() - .WithColumn("Genres").AsString().Nullable() - .WithColumn("Ratings").AsString().Nullable() - .WithColumn("CleanTitle").AsString().Indexed() - .WithColumn("Monitored").AsBoolean() - .WithColumn("AnyEditionOk").AsBoolean() - .WithColumn("LastInfoSync").AsDateTime().Nullable() - .WithColumn("Added").AsDateTime().Nullable() - .WithColumn("AddOptions").AsString().Nullable(); - Create.TableForModel("Editions") .WithColumn("BookId").AsInt32().WithDefaultValue(0) .WithColumn("ForeignEditionId").AsString().Unique() @@ -136,12 +136,12 @@ namespace NzbDrone.Core.Datastore.Migration .WithColumn("OnUpgrade").AsBoolean().Nullable() .WithColumn("Tags").AsString().Nullable() .WithColumn("OnRename").AsBoolean().NotNullable() - .WithColumn("OnReleaseImport").AsBoolean().WithDefaultValue(0) - .WithColumn("OnHealthIssue").AsBoolean().WithDefaultValue(0) - .WithColumn("IncludeHealthWarnings").AsBoolean().WithDefaultValue(0) - .WithColumn("OnDownloadFailure").AsBoolean().WithDefaultValue(0) - .WithColumn("OnImportFailure").AsBoolean().WithDefaultValue(0) - .WithColumn("OnTrackRetag").AsBoolean().WithDefaultValue(0); + .WithColumn("OnReleaseImport").AsBoolean().WithDefaultValue(false) + .WithColumn("OnHealthIssue").AsBoolean().WithDefaultValue(false) + .WithColumn("IncludeHealthWarnings").AsBoolean().WithDefaultValue(false) + .WithColumn("OnDownloadFailure").AsBoolean().WithDefaultValue(false) + .WithColumn("OnImportFailure").AsBoolean().WithDefaultValue(false) + .WithColumn("OnTrackRetag").AsBoolean().WithDefaultValue(false); Create.TableForModel("ScheduledTasks") .WithColumn("TypeName").AsString().Unique() @@ -327,8 +327,8 @@ namespace NzbDrone.Core.Datastore.Migration .WithColumn("Label").AsString().NotNullable() .WithColumn("Filters").AsString().NotNullable(); - Create.Index().OnTable("Books").OnColumn("AuthorId"); - Create.Index().OnTable("Books").OnColumn("AuthorId").Ascending() + IfDatabase("sqlite").Create.Index().OnTable("Books").OnColumn("AuthorId"); + IfDatabase("sqlite").Create.Index().OnTable("Books").OnColumn("AuthorId").Ascending() .OnColumn("ReleaseDate").Ascending(); Delete.Index().OnTable("History").OnColumn("BookId"); @@ -340,12 +340,15 @@ namespace NzbDrone.Core.Datastore.Migration .OnColumn("Date").Descending(); Create.Index().OnTable("Authors").OnColumn("Monitored").Ascending(); + Create.Index().OnTable("Books").OnColumn("AuthorMetadataId").Ascending(); + Create.Index().OnTable("Books").OnColumn("AuthorMetadataId").Ascending() + .OnColumn("ReleaseDate").Ascending(); Insert.IntoTable("DelayProfiles").Row(new { - EnableUsenet = 1, - EnableTorrent = 1, + EnableUsenet = true, + EnableTorrent = true, PreferredProtocol = 1, UsenetDelay = 0, TorrentDelay = 0, diff --git a/src/NzbDrone.Core/Datastore/Migration/002_import_list_search.cs b/src/NzbDrone.Core/Datastore/Migration/002_import_list_search.cs index 876651f03..2316e6586 100644 --- a/src/NzbDrone.Core/Datastore/Migration/002_import_list_search.cs +++ b/src/NzbDrone.Core/Datastore/Migration/002_import_list_search.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Alter.Table("ImportLists").AddColumn("ShouldSearch").AsInt32().WithDefaultValue(1); + Alter.Table("ImportLists").AddColumn("ShouldSearch").AsBoolean().WithDefaultValue(true); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/006_remove_chown_and_folderchmod_config.cs b/src/NzbDrone.Core/Datastore/Migration/006_remove_chown_and_folderchmod_config.cs index 8ae941761..d9912e40e 100644 --- a/src/NzbDrone.Core/Datastore/Migration/006_remove_chown_and_folderchmod_config.cs +++ b/src/NzbDrone.Core/Datastore/Migration/006_remove_chown_and_folderchmod_config.cs @@ -11,7 +11,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Execute.Sql("DELETE FROM config WHERE Key IN ('folderchmod', 'chownuser')"); + Execute.Sql("DELETE FROM \"Config\" WHERE \"Key\" IN ('folderchmod', 'chownuser')"); Execute.WithConnection(ConvertFileChmodToFolderChmod); } @@ -20,7 +20,7 @@ namespace NzbDrone.Core.Datastore.Migration using (IDbCommand getFileChmodCmd = conn.CreateCommand()) { getFileChmodCmd.Transaction = tran; - getFileChmodCmd.CommandText = @"SELECT Value FROM Config WHERE Key = 'filechmod'"; + getFileChmodCmd.CommandText = @"SELECT ""Value"" FROM ""Config"" WHERE ""Key"" = 'filechmod'"; var fileChmod = getFileChmodCmd.ExecuteScalar() as string; if (fileChmod != null) @@ -35,7 +35,7 @@ namespace NzbDrone.Core.Datastore.Migration using (IDbCommand insertCmd = conn.CreateCommand()) { insertCmd.Transaction = tran; - insertCmd.CommandText = "INSERT INTO Config (Key, Value) VALUES ('chmodfolder', ?)"; + insertCmd.CommandText = "INSERT INTO \"Config\" (\"Key\", \"Value\") VALUES ('chmodfolder', ?)"; insertCmd.AddParameter(folderChmod); insertCmd.ExecuteNonQuery(); @@ -45,7 +45,7 @@ namespace NzbDrone.Core.Datastore.Migration using (IDbCommand deleteCmd = conn.CreateCommand()) { deleteCmd.Transaction = tran; - deleteCmd.CommandText = "DELETE FROM Config WHERE Key = 'filechmod'"; + deleteCmd.CommandText = "DELETE FROM \"Config\" WHERE \"Key\" = 'filechmod'"; deleteCmd.ExecuteNonQuery(); } diff --git a/src/NzbDrone.Core/Datastore/Migration/007_update_notifiarr.cs b/src/NzbDrone.Core/Datastore/Migration/007_update_notifiarr.cs index a94067048..a851279c7 100644 --- a/src/NzbDrone.Core/Datastore/Migration/007_update_notifiarr.cs +++ b/src/NzbDrone.Core/Datastore/Migration/007_update_notifiarr.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Execute.Sql("UPDATE Notifications SET Implementation = Replace(Implementation, 'DiscordNotifier', 'Notifiarr'),ConfigContract = Replace(ConfigContract, 'DiscordNotifierSettings', 'NotifiarrSettings') WHERE Implementation = 'DiscordNotifier';"); + Execute.Sql("UPDATE \"Notifications\" SET \"Implementation\" = Replace(\"Implementation\", 'DiscordNotifier', 'Notifiarr'),\"ConfigContract\" = Replace(\"ConfigContract\", 'DiscordNotifierSettings', 'NotifiarrSettings') WHERE \"Implementation\" = 'DiscordNotifier';"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/009_update_author_sort_name.cs b/src/NzbDrone.Core/Datastore/Migration/009_update_author_sort_name.cs index 95948bfc5..fae9cb9d4 100644 --- a/src/NzbDrone.Core/Datastore/Migration/009_update_author_sort_name.cs +++ b/src/NzbDrone.Core/Datastore/Migration/009_update_author_sort_name.cs @@ -21,14 +21,14 @@ namespace NzbDrone.Core.Datastore.Migration private void MigrateAuthorSortName(IDbConnection conn, IDbTransaction tran) { - var rows = conn.Query("SELECT AuthorMetadata.Id, AuthorMetadata.Name FROM AuthorMetadata", transaction: tran); + var rows = conn.Query("SELECT \"AuthorMetadata\".\"Id\", \"AuthorMetadata\".\"Name\" FROM \"AuthorMetadata\"", transaction: tran); foreach (var row in rows) { row.SortName = row.Name.ToLastFirst().ToLower(); } - var sql = "UPDATE AuthorMetadata SET SortName = @SortName WHERE Id = @Id"; + var sql = "UPDATE \"AuthorMetadata\" SET \"SortName\" = @SortName WHERE \"Id\" = @Id"; conn.Execute(sql, rows, transaction: tran); } diff --git a/src/NzbDrone.Core/Datastore/Migration/011_update_audio_qualities.cs b/src/NzbDrone.Core/Datastore/Migration/011_update_audio_qualities.cs index 6092e2598..575e8dd43 100644 --- a/src/NzbDrone.Core/Datastore/Migration/011_update_audio_qualities.cs +++ b/src/NzbDrone.Core/Datastore/Migration/011_update_audio_qualities.cs @@ -54,13 +54,13 @@ namespace NzbDrone.Core.Datastore.Migration _connection = conn; _transaction = tran; - _profiles = _connection.Query(@"SELECT Id, Name, Cutoff, Items FROM QualityProfiles", + _profiles = _connection.Query(@"SELECT ""Id"", ""Name"", ""Cutoff"", ""Items"" FROM ""QualityProfiles""", transaction: _transaction).ToList(); } public void Commit() { - var sql = "UPDATE QualityProfiles SET Name = @Name, Cutoff = @Cutoff, Items = @Items WHERE Id = @Id"; + var sql = "UPDATE \"QualityProfiles\" SET \"Name\" = @Name, \"Cutoff\" = @Cutoff, \"Items\" = @Items WHERE \"Id\" = @Id"; _connection.Execute(sql, _changedProfiles, transaction: _transaction); _changedProfiles.Clear(); diff --git a/src/NzbDrone.Core/Datastore/Migration/012_add_bookfile_part_naming_token.cs b/src/NzbDrone.Core/Datastore/Migration/012_add_bookfile_part_naming_token.cs index fe5e48440..974a36b23 100644 --- a/src/NzbDrone.Core/Datastore/Migration/012_add_bookfile_part_naming_token.cs +++ b/src/NzbDrone.Core/Datastore/Migration/012_add_bookfile_part_naming_token.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Execute.Sql("UPDATE NamingConfig SET StandardBookFormat = StandardBookFormat || '{ (PartNumber)}'"); + Execute.Sql("UPDATE \"NamingConfig\" SET \"StandardBookFormat\" = \"StandardBookFormat\" || '{ (PartNumber)}'"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/013_update_author_sort_name_again.cs b/src/NzbDrone.Core/Datastore/Migration/013_update_author_sort_name_again.cs index eee19eaee..ee7ace701 100644 --- a/src/NzbDrone.Core/Datastore/Migration/013_update_author_sort_name_again.cs +++ b/src/NzbDrone.Core/Datastore/Migration/013_update_author_sort_name_again.cs @@ -21,7 +21,7 @@ namespace NzbDrone.Core.Datastore.Migration private void MigrateAuthorSortName(IDbConnection conn, IDbTransaction tran) { - var rows = conn.Query("SELECT AuthorMetadata.Id, AuthorMetadata.Name FROM AuthorMetadata", transaction: tran); + var rows = conn.Query("SELECT \"AuthorMetadata\".\"Id\", \"AuthorMetadata\".\"Name\" FROM \"AuthorMetadata\"", transaction: tran); foreach (var row in rows) { @@ -30,7 +30,7 @@ namespace NzbDrone.Core.Datastore.Migration row.SortNameLastFirst = row.Name.ToLastFirst().ToLower(); } - var sql = "UPDATE AuthorMetadata SET NameLastFirst = @NameLastFirst, SortName = @SortName, SortNameLastFirst = @SortNameLastFirst WHERE Id = @Id"; + var sql = "UPDATE \"AuthorMetadata\" SET \"NameLastFirst\" = @NameLastFirst, \"SortName\" = @SortName, \"SortNameLastFirst\" = @SortNameLastFirst WHERE \"Id\" = @Id"; conn.Execute(sql, rows, transaction: tran); } diff --git a/src/NzbDrone.Core/Datastore/Migration/015_fix_indexes.cs b/src/NzbDrone.Core/Datastore/Migration/015_fix_indexes.cs index 35b64aef6..a015ce470 100644 --- a/src/NzbDrone.Core/Datastore/Migration/015_fix_indexes.cs +++ b/src/NzbDrone.Core/Datastore/Migration/015_fix_indexes.cs @@ -8,8 +8,8 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Delete.Index().OnTable("Books").OnColumn("AuthorId"); - Delete.Index().OnTable("Books").OnColumns("AuthorId", "ReleaseDate"); + IfDatabase("sqlite").Delete.Index().OnTable("Books").OnColumn("AuthorId"); + IfDatabase("sqlite").Delete.Index().OnTable("Books").OnColumns("AuthorId", "ReleaseDate"); Create.Index().OnTable("Editions").OnColumn("BookId"); } diff --git a/src/NzbDrone.Core/Datastore/Migration/017_import_list_monitor_existing.cs b/src/NzbDrone.Core/Datastore/Migration/017_import_list_monitor_existing.cs index 1fe332d16..d04aae79e 100644 --- a/src/NzbDrone.Core/Datastore/Migration/017_import_list_monitor_existing.cs +++ b/src/NzbDrone.Core/Datastore/Migration/017_import_list_monitor_existing.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Alter.Table("ImportLists").AddColumn("ShouldMonitorExisting").AsInt32().WithDefaultValue(0); + Alter.Table("ImportLists").AddColumn("ShouldMonitorExisting").AsBoolean().WithDefaultValue(false); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/020_add_download_history.cs b/src/NzbDrone.Core/Datastore/Migration/020_add_download_history.cs index aa972e41d..f9864ef69 100644 --- a/src/NzbDrone.Core/Datastore/Migration/020_add_download_history.cs +++ b/src/NzbDrone.Core/Datastore/Migration/020_add_download_history.cs @@ -28,9 +28,7 @@ namespace NzbDrone.Core.Datastore.Migration Create.Index().OnTable("DownloadHistory").OnColumn("AuthorId"); Create.Index().OnTable("DownloadHistory").OnColumn("DownloadId"); - Execute.WithConnection(InitialImportedDownloadHistory); - - Execute.Sql("DELETE From History where EventType = 8;"); + IfDatabase("sqlite").Execute.WithConnection(InitialImportedDownloadHistory); } private static readonly Dictionary EventTypeMap = new Dictionary() @@ -56,7 +54,7 @@ namespace NzbDrone.Core.Datastore.Migration using (var cmd = conn.CreateCommand()) { cmd.Transaction = tran; - cmd.CommandText = "SELECT AuthorId, DownloadId, EventType, SourceTitle, Date, Data FROM History WHERE DownloadId IS NOT NULL AND EventType IN (1, 8, 4, 10, 7) GROUP BY EventType, DownloadId"; + cmd.CommandText = "SELECT \"AuthorId\", \"DownloadId\", \"EventType\", \"SourceTitle\", \"Date\", \"Data\" FROM \"History\" WHERE \"DownloadId\" IS NOT NULL AND \"EventType\" IN (1, 8, 4, 10, 7) GROUP BY \"EventType\", \"DownloadId\""; using (var reader = cmd.ExecuteReader()) { @@ -87,7 +85,15 @@ namespace NzbDrone.Core.Datastore.Migration using (var updateCmd = conn.CreateCommand()) { updateCmd.Transaction = tran; - updateCmd.CommandText = @"INSERT INTO DownloadHistory (EventType, AuthorId, DownloadId, SourceTitle, Date, Protocol, Data) VALUES (?, ?, ?, ?, ?, ?, ?)"; + if (conn.GetType().FullName == "Npgsql.NpgsqlConnection") + { + updateCmd.CommandText = @"INSERT INTO ""DownloadHistory"" (""EventType"", ""AuthorId"", ""DownloadId"", ""SourceTitle"", ""Date"", ""Protocol"", ""Data"") VALUES ($1, $2, $3, $4, $5, $6, $7)"; + } + else + { + updateCmd.CommandText = @"INSERT INTO ""DownloadHistory"" (""EventType"", ""AuthorId"", ""DownloadId"", ""SourceTitle"", ""Date"", ""Protocol"", ""Data"") VALUES (?, ?, ?, ?, ?, ?, ?)"; + } + updateCmd.AddParameter(downloadHistoryEventType); updateCmd.AddParameter(seriesId); updateCmd.AddParameter(downloadId); diff --git a/src/NzbDrone.Core/Datastore/Migration/021_add_on_delete_to_notifications.cs b/src/NzbDrone.Core/Datastore/Migration/021_add_on_delete_to_notifications.cs index f16e018b5..13e3d92e4 100644 --- a/src/NzbDrone.Core/Datastore/Migration/021_add_on_delete_to_notifications.cs +++ b/src/NzbDrone.Core/Datastore/Migration/021_add_on_delete_to_notifications.cs @@ -8,10 +8,10 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Alter.Table("Notifications").AddColumn("OnAuthorDelete").AsBoolean().WithDefaultValue(0); - Alter.Table("Notifications").AddColumn("OnBookDelete").AsBoolean().WithDefaultValue(0); - Alter.Table("Notifications").AddColumn("OnBookFileDelete").AsBoolean().WithDefaultValue(0); - Alter.Table("Notifications").AddColumn("OnBookFileDeleteForUpgrade").AsBoolean().WithDefaultValue(0); + Alter.Table("Notifications").AddColumn("OnAuthorDelete").AsBoolean().WithDefaultValue(false); + Alter.Table("Notifications").AddColumn("OnBookDelete").AsBoolean().WithDefaultValue(false); + Alter.Table("Notifications").AddColumn("OnBookFileDelete").AsBoolean().WithDefaultValue(false); + Alter.Table("Notifications").AddColumn("OnBookFileDeleteForUpgrade").AsBoolean().WithDefaultValue(false); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs index 35c5c6b59..1249dfd8b 100644 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics; using System.Reflection; using FluentMigrator.Runner; +using FluentMigrator.Runner.Generators; using FluentMigrator.Runner.Initialization; using FluentMigrator.Runner.Processors; using Microsoft.Extensions.DependencyInjection; @@ -34,11 +35,16 @@ namespace NzbDrone.Core.Datastore.Migration.Framework _logger.Info("*** Migrating {0} ***", connectionString); - var serviceProvider = new ServiceCollection() + ServiceProvider serviceProvider; + + var db = connectionString.Contains(".db") ? "sqlite" : "postgres"; + + serviceProvider = new ServiceCollection() .AddLogging(b => b.AddNLog()) .AddFluentMigratorCore() .ConfigureRunner( builder => builder + .AddPostgres() .AddNzbDroneSQLite() .WithGlobalConnectionString(connectionString) .WithMigrationsIn(Assembly.GetExecutingAssembly())) @@ -48,6 +54,14 @@ namespace NzbDrone.Core.Datastore.Migration.Framework opt.PreviewOnly = false; opt.Timeout = TimeSpan.FromSeconds(60); }) + .Configure(cfg => + { + cfg.ProcessorId = db; + }) + .Configure(cfg => + { + cfg.GeneratorId = db; + }) .BuildServiceProvider(); using (var scope = serviceProvider.CreateScope()) diff --git a/src/NzbDrone.Core/Datastore/PostgresOptions.cs b/src/NzbDrone.Core/Datastore/PostgresOptions.cs new file mode 100644 index 000000000..e0a96a56c --- /dev/null +++ b/src/NzbDrone.Core/Datastore/PostgresOptions.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.Configuration; + +namespace NzbDrone.Core.Datastore +{ + public class PostgresOptions + { + public string Host { get; set; } + public int Port { get; set; } + public string User { get; set; } + public string Password { get; set; } + public string MainDb { get; set; } + public string LogDb { get; set; } + public string CacheDb { get; set; } + + public static PostgresOptions GetOptions() + { + var config = new ConfigurationBuilder() + .AddEnvironmentVariables() + .Build(); + + var postgresOptions = new PostgresOptions(); + config.GetSection("Readarr:Postgres").Bind(postgresOptions); + + return postgresOptions; + } + } +} diff --git a/src/NzbDrone.Core/Datastore/SqlBuilder.cs b/src/NzbDrone.Core/Datastore/SqlBuilder.cs index e686d4852..8febd5b6a 100644 --- a/src/NzbDrone.Core/Datastore/SqlBuilder.cs +++ b/src/NzbDrone.Core/Datastore/SqlBuilder.cs @@ -8,9 +8,17 @@ namespace NzbDrone.Core.Datastore public class SqlBuilder { private readonly Dictionary _data = new Dictionary(); + private readonly DatabaseType _databaseType; + + public SqlBuilder(DatabaseType databaseType) + { + _databaseType = databaseType; + } public int Sequence { get; private set; } + public DatabaseType DatabaseType => _databaseType; + public Template AddTemplate(string sql, dynamic parameters = null) => new Template(this, sql, parameters); diff --git a/src/NzbDrone.Core/Datastore/TableMapper.cs b/src/NzbDrone.Core/Datastore/TableMapper.cs index 6ea730648..9f8bc52f8 100644 --- a/src/NzbDrone.Core/Datastore/TableMapper.cs +++ b/src/NzbDrone.Core/Datastore/TableMapper.cs @@ -50,17 +50,17 @@ namespace NzbDrone.Core.Datastore public string SelectTemplate(Type x) { - return $"SELECT /**select**/ FROM {TableMap[x]} /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/ /**orderby**/"; + return $"SELECT /**select**/ FROM \"{TableMap[x]}\" /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/ /**orderby**/"; } public string DeleteTemplate(Type x) { - return $"DELETE FROM {TableMap[x]} /**where**/"; + return $"DELETE FROM \"{TableMap[x]}\" /**where**/"; } public string PageCountTemplate(Type x) { - return $"SELECT /**select**/ FROM {TableMap[x]} /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/"; + return $"SELECT /**select**/ FROM \"{TableMap[x]}\" /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/"; } public bool IsValidSortKey(string sortKey) @@ -91,6 +91,35 @@ namespace NzbDrone.Core.Datastore return true; } + + public string GetSortKey(string sortKey) + { + string table = null; + + if (sortKey.Contains('.')) + { + var split = sortKey.Split('.'); + if (split.Length != 2) + { + return sortKey; + } + + table = split[0]; + sortKey = split[1]; + } + + if (table != null && !TableMap.Values.Contains(table, StringComparer.OrdinalIgnoreCase)) + { + return sortKey; + } + + if (!_allowedOrderBy.Contains(sortKey)) + { + return sortKey; + } + + return _allowedOrderBy.First(x => x.Equals(sortKey, StringComparison.OrdinalIgnoreCase)); + } } public class LazyLoadedProperty @@ -155,7 +184,7 @@ namespace NzbDrone.Core.Datastore (db, parent) => { var id = childIdSelector(parent); - return db.Query(new SqlBuilder().Where(x => x.Id == id)).SingleOrDefault(); + return db.Query(new SqlBuilder(db.DatabaseType).Where(x => x.Id == id)).SingleOrDefault(); }, parent => childIdSelector(parent) > 0); } diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 3cbfb2fee..c42ef592c 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -108,15 +108,15 @@ namespace NzbDrone.Core.Datastore .HasOne(a => a.Metadata, a => a.AuthorMetadataId) .HasOne(a => a.QualityProfile, a => a.QualityProfileId) .HasOne(s => s.MetadataProfile, s => s.MetadataProfileId) - .LazyLoad(a => a.Books, (db, a) => db.Query(new SqlBuilder().Where(b => b.AuthorMetadataId == a.AuthorMetadataId)).ToList(), a => a.AuthorMetadataId > 0); + .LazyLoad(a => a.Books, (db, a) => db.Query(new SqlBuilder(db.DatabaseType).Where(b => b.AuthorMetadataId == a.AuthorMetadataId)).ToList(), a => a.AuthorMetadataId > 0); Mapper.Entity("Series").RegisterModel() .Ignore(s => s.ForeignAuthorId) .LazyLoad(s => s.LinkItems, - (db, series) => db.Query(new SqlBuilder().Where(s => s.SeriesId == series.Id)).ToList(), + (db, series) => db.Query(new SqlBuilder(db.DatabaseType).Where(s => s.SeriesId == series.Id)).ToList(), s => s.Id > 0) .LazyLoad(s => s.Books, - (db, series) => db.Query(new SqlBuilder() + (db, series) => db.Query(new SqlBuilder(db.DatabaseType) .Join((l, r) => l.Id == r.BookId) .Join((l, r) => l.SeriesId == r.Id) .Where(s => s.Id == series.Id)).ToList(), @@ -132,27 +132,27 @@ namespace NzbDrone.Core.Datastore .Ignore(x => x.AuthorId) .HasOne(r => r.AuthorMetadata, r => r.AuthorMetadataId) .LazyLoad(x => x.BookFiles, - (db, book) => db.Query(new SqlBuilder() + (db, book) => db.Query(new SqlBuilder(db.DatabaseType) .Join((l, r) => l.EditionId == r.Id) .Where(b => b.BookId == book.Id)).ToList(), b => b.Id > 0) .LazyLoad(x => x.Editions, - (db, book) => db.Query(new SqlBuilder().Where(e => e.BookId == book.Id)).ToList(), + (db, book) => db.Query(new SqlBuilder(db.DatabaseType).Where(e => e.BookId == book.Id)).ToList(), b => b.Id > 0) .LazyLoad(a => a.Author, (db, book) => AuthorRepository.Query(db, - new SqlBuilder() + new SqlBuilder(db.DatabaseType) .Join((a, m) => a.AuthorMetadataId == m.Id) .Where(a => a.AuthorMetadataId == book.AuthorMetadataId)).SingleOrDefault(), a => a.AuthorMetadataId > 0) .LazyLoad(b => b.SeriesLinks, - (db, book) => db.Query(new SqlBuilder().Where(s => s.BookId == book.Id)).ToList(), + (db, book) => db.Query(new SqlBuilder(db.DatabaseType).Where(s => s.BookId == book.Id)).ToList(), b => b.Id > 0); Mapper.Entity("Editions").RegisterModel() .HasOne(r => r.Book, r => r.BookId) .LazyLoad(x => x.BookFiles, - (db, edition) => db.Query(new SqlBuilder().Where(f => f.EditionId == edition.Id)).ToList(), + (db, edition) => db.Query(new SqlBuilder(db.DatabaseType).Where(f => f.EditionId == edition.Id)).ToList(), b => b.Id > 0); Mapper.Entity("BookFiles").RegisterModel() @@ -160,7 +160,7 @@ namespace NzbDrone.Core.Datastore .HasOne(f => f.Edition, f => f.EditionId) .LazyLoad(x => x.Author, (db, f) => AuthorRepository.Query(db, - new SqlBuilder() + new SqlBuilder(db.DatabaseType) .Join((a, m) => a.AuthorMetadataId == m.Id) .Join((l, r) => l.AuthorMetadataId == r.AuthorMetadataId) .Join((l, r) => l.Id == r.BookId) diff --git a/src/NzbDrone.Core/Datastore/WhereBuilder.cs b/src/NzbDrone.Core/Datastore/WhereBuilder.cs index 3c018d0ad..749515a2d 100644 --- a/src/NzbDrone.Core/Datastore/WhereBuilder.cs +++ b/src/NzbDrone.Core/Datastore/WhereBuilder.cs @@ -1,390 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Data; -using System.Linq; -using System.Linq.Expressions; -using System.Reflection; -using System.Text; using Dapper; using NzbDrone.Common.Extensions; namespace NzbDrone.Core.Datastore { - public class WhereBuilder : ExpressionVisitor + public abstract class WhereBuilder : ExpressionVisitor { - protected StringBuilder _sb; - - private const DbType EnumerableMultiParameter = (DbType)(-1); - private readonly string _paramNamePrefix; - private readonly bool _requireConcreteValue = false; - private int _paramCount = 0; - private bool _gotConcreteValue = false; - - public WhereBuilder(Expression filter, bool requireConcreteValue, int seq) - { - _paramNamePrefix = string.Format("Clause{0}", seq + 1); - _requireConcreteValue = requireConcreteValue; - _sb = new StringBuilder(); - - Parameters = new DynamicParameters(); - - if (filter != null) - { - Visit(filter); - } - } - - public DynamicParameters Parameters { get; private set; } - - private string AddParameter(object value, DbType? dbType = null) - { - _gotConcreteValue = true; - _paramCount++; - var name = _paramNamePrefix + "_P" + _paramCount; - Parameters.Add(name, value, dbType); - return '@' + name; - } - - protected override Expression VisitBinary(BinaryExpression expression) - { - _sb.Append("("); - - Visit(expression.Left); - - _sb.AppendFormat(" {0} ", Decode(expression)); - - Visit(expression.Right); - - _sb.Append(")"); - - return expression; - } - - protected override Expression VisitMethodCall(MethodCallExpression expression) - { - var method = expression.Method.Name; - - switch (expression.Method.Name) - { - case "Contains": - ParseContainsExpression(expression); - break; - - case "StartsWith": - ParseStartsWith(expression); - break; - - case "EndsWith": - ParseEndsWith(expression); - break; - - default: - var msg = string.Format("'{0}' expressions are not yet implemented in the where clause expression tree parser.", method); - throw new NotImplementedException(msg); - } - - return expression; - } - - protected override Expression VisitMemberAccess(MemberExpression expression) - { - var tableName = expression?.Expression?.Type != null ? TableMapping.Mapper.TableNameMapping(expression.Expression.Type) : null; - var gotValue = TryGetRightValue(expression, out var value); - - // Only use the SQL condition if the expression didn't resolve to an actual value - if (tableName != null && !gotValue) - { - _sb.Append($"\"{tableName}\".\"{expression.Member.Name}\""); - } - else - { - if (value != null) - { - // string is IEnumerable but we don't want to pick up that case - var type = value.GetType(); - var typeInfo = type.GetTypeInfo(); - var isEnumerable = - type != typeof(string) && ( - typeInfo.ImplementedInterfaces.Any(ti => ti.IsGenericType && ti.GetGenericTypeDefinition() == typeof(IEnumerable<>)) || - (typeInfo.IsGenericType && typeInfo.GetGenericTypeDefinition() == typeof(IEnumerable<>))); - - var paramName = isEnumerable ? AddParameter(value, EnumerableMultiParameter) : AddParameter(value); - _sb.Append(paramName); - } - else - { - _gotConcreteValue = true; - _sb.Append("NULL"); - } - } - - return expression; - } - - protected override Expression VisitConstant(ConstantExpression expression) - { - if (expression.Value != null) - { - var paramName = AddParameter(expression.Value); - _sb.Append(paramName); - } - else - { - _gotConcreteValue = true; - _sb.Append("NULL"); - } - - return expression; - } - - private bool TryGetConstantValue(Expression expression, out object result) - { - result = null; - - if (expression is ConstantExpression constExp) - { - result = constExp.Value; - return true; - } - - return false; - } - - private bool TryGetPropertyValue(MemberExpression expression, out object result) - { - result = null; - - if (expression.Expression is MemberExpression nested) - { - // Value is passed in as a property on a parent entity - var container = (nested.Expression as ConstantExpression)?.Value; - - if (container == null) - { - return false; - } - - var entity = GetFieldValue(container, nested.Member); - result = GetFieldValue(entity, expression.Member); - return true; - } - - return false; - } - - private bool TryGetVariableValue(MemberExpression expression, out object result) - { - result = null; - - // Value is passed in as a variable - if (expression.Expression is ConstantExpression nested) - { - result = GetFieldValue(nested.Value, expression.Member); - return true; - } - - return false; - } - - private bool TryGetRightValue(Expression expression, out object value) - { - value = null; - - if (TryGetConstantValue(expression, out value)) - { - return true; - } - - var memberExp = expression as MemberExpression; - - if (TryGetPropertyValue(memberExp, out value)) - { - return true; - } - - if (TryGetVariableValue(memberExp, out value)) - { - return true; - } - - return false; - } - - private object GetFieldValue(object entity, MemberInfo member) - { - if (member.MemberType == MemberTypes.Field) - { - return (member as FieldInfo).GetValue(entity); - } - - if (member.MemberType == MemberTypes.Property) - { - return (member as PropertyInfo).GetValue(entity); - } - - throw new ArgumentException(string.Format("WhereBuilder could not get the value for {0}.{1}.", entity.GetType().Name, member.Name)); - } - - private bool IsNullVariable(Expression expression) - { - if (expression.NodeType == ExpressionType.Constant && - TryGetConstantValue(expression, out var constResult) && - constResult == null) - { - return true; - } - - if (expression.NodeType == ExpressionType.MemberAccess && - expression is MemberExpression member && - ((TryGetPropertyValue(member, out var result) && result == null) || - (TryGetVariableValue(member, out result) && result == null))) - { - return true; - } - - return false; - } - - private string Decode(BinaryExpression expression) - { - if (IsNullVariable(expression.Right)) - { - switch (expression.NodeType) - { - case ExpressionType.Equal: return "IS"; - case ExpressionType.NotEqual: return "IS NOT"; - } - } - - switch (expression.NodeType) - { - case ExpressionType.AndAlso: return "AND"; - case ExpressionType.And: return "AND"; - case ExpressionType.Equal: return "="; - case ExpressionType.GreaterThan: return ">"; - case ExpressionType.GreaterThanOrEqual: return ">="; - case ExpressionType.LessThan: return "<"; - case ExpressionType.LessThanOrEqual: return "<="; - case ExpressionType.NotEqual: return "<>"; - case ExpressionType.OrElse: return "OR"; - case ExpressionType.Or: return "OR"; - default: throw new NotSupportedException(string.Format("{0} statement is not supported", expression.NodeType.ToString())); - } - } - - private void ParseContainsExpression(MethodCallExpression expression) - { - var list = expression.Object; - - if (list != null && (list.Type == typeof(string) || list.Type == typeof(List))) - { - ParseStringContains(expression); - return; - } - - ParseEnumerableContains(expression); - } - - private void ParseEnumerableContains(MethodCallExpression body) - { - // Fish out the list and the item to compare - // It's in a different form for arrays and Lists - var list = body.Object; - Expression item; - - if (list != null) - { - // Generic collection - item = body.Arguments[0]; - } - else - { - // Static method - // Must be Enumerable.Contains(source, item) - if (body.Method.DeclaringType != typeof(Enumerable) || body.Arguments.Count != 2) - { - throw new NotSupportedException("Unexpected form of Enumerable.Contains"); - } - - list = body.Arguments[0]; - item = body.Arguments[1]; - } - - _sb.Append("("); - - Visit(item); - - _sb.Append(" IN "); - - // hardcode the integer list if it exists to bypass parameter limit - if (item.Type == typeof(int) && TryGetRightValue(list, out var value)) - { - var items = (IEnumerable)value; - _sb.Append("("); - _sb.Append(string.Join(", ", items)); - _sb.Append(")"); - - _gotConcreteValue = true; - } - else - { - Visit(list); - } - - _sb.Append(")"); - } - - private void ParseStringContains(MethodCallExpression body) - { - _sb.Append("("); - - Visit(body.Object); - - _sb.Append(" LIKE '%' || "); - - Visit(body.Arguments[0]); - - _sb.Append(" || '%')"); - } - - private void ParseStartsWith(MethodCallExpression body) - { - _sb.Append("("); - - Visit(body.Object); - - _sb.Append(" LIKE "); - - Visit(body.Arguments[0]); - - _sb.Append(" || '%')"); - } - - private void ParseEndsWith(MethodCallExpression body) - { - _sb.Append("("); - - Visit(body.Object); - - _sb.Append(" LIKE '%' || "); - - Visit(body.Arguments[0]); - - _sb.Append(")"); - } - - public override string ToString() - { - var sql = _sb.ToString(); - - if (_requireConcreteValue && !_gotConcreteValue) - { - var e = new InvalidOperationException("WhereBuilder requires a concrete condition"); - e.Data.Add("sql", sql); - throw e; - } - - return sql; - } + public DynamicParameters Parameters { get; protected set; } } } diff --git a/src/NzbDrone.Core/Datastore/WhereBuilderPostgres.cs b/src/NzbDrone.Core/Datastore/WhereBuilderPostgres.cs new file mode 100644 index 000000000..f3e31e179 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/WhereBuilderPostgres.cs @@ -0,0 +1,387 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using System.Text; +using Dapper; + +namespace NzbDrone.Core.Datastore +{ + public class WhereBuilderPostgres : WhereBuilder + { + protected StringBuilder _sb; + + private const DbType EnumerableMultiParameter = (DbType)(-1); + private readonly string _paramNamePrefix; + private readonly bool _requireConcreteValue = false; + private int _paramCount = 0; + private bool _gotConcreteValue = false; + + public WhereBuilderPostgres(Expression filter, bool requireConcreteValue, int seq) + { + _paramNamePrefix = string.Format("Clause{0}", seq + 1); + _requireConcreteValue = requireConcreteValue; + _sb = new StringBuilder(); + + Parameters = new DynamicParameters(); + + if (filter != null) + { + Visit(filter); + } + } + + private string AddParameter(object value, DbType? dbType = null) + { + _gotConcreteValue = true; + _paramCount++; + var name = _paramNamePrefix + "_P" + _paramCount; + Parameters.Add(name, value, dbType); + return '@' + name; + } + + protected override Expression VisitBinary(BinaryExpression expression) + { + _sb.Append('('); + + Visit(expression.Left); + + _sb.AppendFormat(" {0} ", Decode(expression)); + + Visit(expression.Right); + + _sb.Append(')'); + + return expression; + } + + protected override Expression VisitMethodCall(MethodCallExpression expression) + { + var method = expression.Method.Name; + + switch (expression.Method.Name) + { + case "Contains": + ParseContainsExpression(expression); + break; + + case "StartsWith": + ParseStartsWith(expression); + break; + + case "EndsWith": + ParseEndsWith(expression); + break; + + default: + var msg = string.Format("'{0}' expressions are not yet implemented in the where clause expression tree parser.", method); + throw new NotImplementedException(msg); + } + + return expression; + } + + protected override Expression VisitMemberAccess(MemberExpression expression) + { + var tableName = expression?.Expression?.Type != null ? TableMapping.Mapper.TableNameMapping(expression.Expression.Type) : null; + var gotValue = TryGetRightValue(expression, out var value); + + // Only use the SQL condition if the expression didn't resolve to an actual value + if (tableName != null && !gotValue) + { + _sb.Append($"\"{tableName}\".\"{expression.Member.Name}\""); + } + else + { + if (value != null) + { + // string is IEnumerable but we don't want to pick up that case + var type = value.GetType(); + var typeInfo = type.GetTypeInfo(); + var isEnumerable = + type != typeof(string) && ( + typeInfo.ImplementedInterfaces.Any(ti => ti.IsGenericType && ti.GetGenericTypeDefinition() == typeof(IEnumerable<>)) || + (typeInfo.IsGenericType && typeInfo.GetGenericTypeDefinition() == typeof(IEnumerable<>))); + + var paramName = isEnumerable ? AddParameter(value, EnumerableMultiParameter) : AddParameter(value); + _sb.Append(paramName); + } + else + { + _gotConcreteValue = true; + _sb.Append("NULL"); + } + } + + return expression; + } + + protected override Expression VisitConstant(ConstantExpression expression) + { + if (expression.Value != null) + { + var paramName = AddParameter(expression.Value); + _sb.Append(paramName); + } + else + { + _gotConcreteValue = true; + _sb.Append("NULL"); + } + + return expression; + } + + private bool TryGetConstantValue(Expression expression, out object result) + { + result = null; + + if (expression is ConstantExpression constExp) + { + result = constExp.Value; + return true; + } + + return false; + } + + private bool TryGetPropertyValue(MemberExpression expression, out object result) + { + result = null; + + if (expression.Expression is MemberExpression nested) + { + // Value is passed in as a property on a parent entity + var container = (nested.Expression as ConstantExpression)?.Value; + + if (container == null) + { + return false; + } + + var entity = GetFieldValue(container, nested.Member); + result = GetFieldValue(entity, expression.Member); + return true; + } + + return false; + } + + private bool TryGetVariableValue(MemberExpression expression, out object result) + { + result = null; + + // Value is passed in as a variable + if (expression.Expression is ConstantExpression nested) + { + result = GetFieldValue(nested.Value, expression.Member); + return true; + } + + return false; + } + + private bool TryGetRightValue(Expression expression, out object value) + { + value = null; + + if (TryGetConstantValue(expression, out value)) + { + return true; + } + + var memberExp = expression as MemberExpression; + + if (TryGetPropertyValue(memberExp, out value)) + { + return true; + } + + if (TryGetVariableValue(memberExp, out value)) + { + return true; + } + + return false; + } + + private object GetFieldValue(object entity, MemberInfo member) + { + if (member.MemberType == MemberTypes.Field) + { + return (member as FieldInfo).GetValue(entity); + } + + if (member.MemberType == MemberTypes.Property) + { + return (member as PropertyInfo).GetValue(entity); + } + + throw new ArgumentException(string.Format("WhereBuilder could not get the value for {0}.{1}.", entity.GetType().Name, member.Name)); + } + + private bool IsNullVariable(Expression expression) + { + if (expression.NodeType == ExpressionType.Constant && + TryGetConstantValue(expression, out var constResult) && + constResult == null) + { + return true; + } + + if (expression.NodeType == ExpressionType.MemberAccess && + expression is MemberExpression member && + ((TryGetPropertyValue(member, out var result) && result == null) || + (TryGetVariableValue(member, out result) && result == null))) + { + return true; + } + + return false; + } + + private string Decode(BinaryExpression expression) + { + if (IsNullVariable(expression.Right)) + { + switch (expression.NodeType) + { + case ExpressionType.Equal: return "IS"; + case ExpressionType.NotEqual: return "IS NOT"; + } + } + + switch (expression.NodeType) + { + case ExpressionType.AndAlso: return "AND"; + case ExpressionType.And: return "AND"; + case ExpressionType.Equal: return "="; + case ExpressionType.GreaterThan: return ">"; + case ExpressionType.GreaterThanOrEqual: return ">="; + case ExpressionType.LessThan: return "<"; + case ExpressionType.LessThanOrEqual: return "<="; + case ExpressionType.NotEqual: return "<>"; + case ExpressionType.OrElse: return "OR"; + case ExpressionType.Or: return "OR"; + default: throw new NotSupportedException(string.Format("{0} statement is not supported", expression.NodeType.ToString())); + } + } + + private void ParseContainsExpression(MethodCallExpression expression) + { + var list = expression.Object; + + if (list != null && (list.Type == typeof(string))) + { + ParseStringContains(expression); + return; + } + + ParseEnumerableContains(expression); + } + + private void ParseEnumerableContains(MethodCallExpression body) + { + // Fish out the list and the item to compare + // It's in a different form for arrays and Lists + var list = body.Object; + Expression item; + + if (list != null) + { + // Generic collection + item = body.Arguments[0]; + } + else + { + // Static method + // Must be Enumerable.Contains(source, item) + if (body.Method.DeclaringType != typeof(Enumerable) || body.Arguments.Count != 2) + { + throw new NotSupportedException("Unexpected form of Enumerable.Contains"); + } + + list = body.Arguments[0]; + item = body.Arguments[1]; + } + + _sb.Append('('); + + Visit(item); + + _sb.Append(" = ANY ("); + + // hardcode the integer list if it exists to bypass parameter limit + if (item.Type == typeof(int) && TryGetRightValue(list, out var value)) + { + var items = (IEnumerable)value; + _sb.Append("('{"); + _sb.Append(string.Join(", ", items)); + _sb.Append("}')"); + + _gotConcreteValue = true; + } + else + { + Visit(list); + } + + _sb.Append("))"); + } + + private void ParseStringContains(MethodCallExpression body) + { + _sb.Append('('); + + Visit(body.Object); + + _sb.Append(" ILIKE '%' || "); + + Visit(body.Arguments[0]); + + _sb.Append(" || '%')"); + } + + private void ParseStartsWith(MethodCallExpression body) + { + _sb.Append('('); + + Visit(body.Object); + + _sb.Append(" ILIKE "); + + Visit(body.Arguments[0]); + + _sb.Append(" || '%')"); + } + + private void ParseEndsWith(MethodCallExpression body) + { + _sb.Append('('); + + Visit(body.Object); + + _sb.Append(" ILIKE '%' || "); + + Visit(body.Arguments[0]); + + _sb.Append(')'); + } + + public override string ToString() + { + var sql = _sb.ToString(); + + if (_requireConcreteValue && !_gotConcreteValue) + { + var e = new InvalidOperationException("WhereBuilder requires a concrete condition"); + e.Data.Add("sql", sql); + throw e; + } + + return sql; + } + } +} diff --git a/src/NzbDrone.Core/Datastore/WhereBuilderSqlite.cs b/src/NzbDrone.Core/Datastore/WhereBuilderSqlite.cs new file mode 100644 index 000000000..54b24d7e1 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/WhereBuilderSqlite.cs @@ -0,0 +1,387 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using System.Text; +using Dapper; + +namespace NzbDrone.Core.Datastore +{ + public class WhereBuilderSqlite : WhereBuilder + { + protected StringBuilder _sb; + + private const DbType EnumerableMultiParameter = (DbType)(-1); + private readonly string _paramNamePrefix; + private readonly bool _requireConcreteValue = false; + private int _paramCount = 0; + private bool _gotConcreteValue = false; + + public WhereBuilderSqlite(Expression filter, bool requireConcreteValue, int seq) + { + _paramNamePrefix = string.Format("Clause{0}", seq + 1); + _requireConcreteValue = requireConcreteValue; + _sb = new StringBuilder(); + + Parameters = new DynamicParameters(); + + if (filter != null) + { + Visit(filter); + } + } + + private string AddParameter(object value, DbType? dbType = null) + { + _gotConcreteValue = true; + _paramCount++; + var name = _paramNamePrefix + "_P" + _paramCount; + Parameters.Add(name, value, dbType); + return '@' + name; + } + + protected override Expression VisitBinary(BinaryExpression expression) + { + _sb.Append('('); + + Visit(expression.Left); + + _sb.AppendFormat(" {0} ", Decode(expression)); + + Visit(expression.Right); + + _sb.Append(')'); + + return expression; + } + + protected override Expression VisitMethodCall(MethodCallExpression expression) + { + var method = expression.Method.Name; + + switch (expression.Method.Name) + { + case "Contains": + ParseContainsExpression(expression); + break; + + case "StartsWith": + ParseStartsWith(expression); + break; + + case "EndsWith": + ParseEndsWith(expression); + break; + + default: + var msg = string.Format("'{0}' expressions are not yet implemented in the where clause expression tree parser.", method); + throw new NotImplementedException(msg); + } + + return expression; + } + + protected override Expression VisitMemberAccess(MemberExpression expression) + { + var tableName = expression?.Expression?.Type != null ? TableMapping.Mapper.TableNameMapping(expression.Expression.Type) : null; + var gotValue = TryGetRightValue(expression, out var value); + + // Only use the SQL condition if the expression didn't resolve to an actual value + if (tableName != null && !gotValue) + { + _sb.Append($"\"{tableName}\".\"{expression.Member.Name}\""); + } + else + { + if (value != null) + { + // string is IEnumerable but we don't want to pick up that case + var type = value.GetType(); + var typeInfo = type.GetTypeInfo(); + var isEnumerable = + type != typeof(string) && ( + typeInfo.ImplementedInterfaces.Any(ti => ti.IsGenericType && ti.GetGenericTypeDefinition() == typeof(IEnumerable<>)) || + (typeInfo.IsGenericType && typeInfo.GetGenericTypeDefinition() == typeof(IEnumerable<>))); + + var paramName = isEnumerable ? AddParameter(value, EnumerableMultiParameter) : AddParameter(value); + _sb.Append(paramName); + } + else + { + _gotConcreteValue = true; + _sb.Append("NULL"); + } + } + + return expression; + } + + protected override Expression VisitConstant(ConstantExpression expression) + { + if (expression.Value != null) + { + var paramName = AddParameter(expression.Value); + _sb.Append(paramName); + } + else + { + _gotConcreteValue = true; + _sb.Append("NULL"); + } + + return expression; + } + + private bool TryGetConstantValue(Expression expression, out object result) + { + result = null; + + if (expression is ConstantExpression constExp) + { + result = constExp.Value; + return true; + } + + return false; + } + + private bool TryGetPropertyValue(MemberExpression expression, out object result) + { + result = null; + + if (expression.Expression is MemberExpression nested) + { + // Value is passed in as a property on a parent entity + var container = (nested.Expression as ConstantExpression)?.Value; + + if (container == null) + { + return false; + } + + var entity = GetFieldValue(container, nested.Member); + result = GetFieldValue(entity, expression.Member); + return true; + } + + return false; + } + + private bool TryGetVariableValue(MemberExpression expression, out object result) + { + result = null; + + // Value is passed in as a variable + if (expression.Expression is ConstantExpression nested) + { + result = GetFieldValue(nested.Value, expression.Member); + return true; + } + + return false; + } + + private bool TryGetRightValue(Expression expression, out object value) + { + value = null; + + if (TryGetConstantValue(expression, out value)) + { + return true; + } + + var memberExp = expression as MemberExpression; + + if (TryGetPropertyValue(memberExp, out value)) + { + return true; + } + + if (TryGetVariableValue(memberExp, out value)) + { + return true; + } + + return false; + } + + private object GetFieldValue(object entity, MemberInfo member) + { + if (member.MemberType == MemberTypes.Field) + { + return (member as FieldInfo).GetValue(entity); + } + + if (member.MemberType == MemberTypes.Property) + { + return (member as PropertyInfo).GetValue(entity); + } + + throw new ArgumentException(string.Format("WhereBuilder could not get the value for {0}.{1}.", entity.GetType().Name, member.Name)); + } + + private bool IsNullVariable(Expression expression) + { + if (expression.NodeType == ExpressionType.Constant && + TryGetConstantValue(expression, out var constResult) && + constResult == null) + { + return true; + } + + if (expression.NodeType == ExpressionType.MemberAccess && + expression is MemberExpression member && + ((TryGetPropertyValue(member, out var result) && result == null) || + (TryGetVariableValue(member, out result) && result == null))) + { + return true; + } + + return false; + } + + private string Decode(BinaryExpression expression) + { + if (IsNullVariable(expression.Right)) + { + switch (expression.NodeType) + { + case ExpressionType.Equal: return "IS"; + case ExpressionType.NotEqual: return "IS NOT"; + } + } + + switch (expression.NodeType) + { + case ExpressionType.AndAlso: return "AND"; + case ExpressionType.And: return "AND"; + case ExpressionType.Equal: return "="; + case ExpressionType.GreaterThan: return ">"; + case ExpressionType.GreaterThanOrEqual: return ">="; + case ExpressionType.LessThan: return "<"; + case ExpressionType.LessThanOrEqual: return "<="; + case ExpressionType.NotEqual: return "<>"; + case ExpressionType.OrElse: return "OR"; + case ExpressionType.Or: return "OR"; + default: throw new NotSupportedException(string.Format("{0} statement is not supported", expression.NodeType.ToString())); + } + } + + private void ParseContainsExpression(MethodCallExpression expression) + { + var list = expression.Object; + + if (list != null && (list.Type == typeof(string))) + { + ParseStringContains(expression); + return; + } + + ParseEnumerableContains(expression); + } + + private void ParseEnumerableContains(MethodCallExpression body) + { + // Fish out the list and the item to compare + // It's in a different form for arrays and Lists + var list = body.Object; + Expression item; + + if (list != null) + { + // Generic collection + item = body.Arguments[0]; + } + else + { + // Static method + // Must be Enumerable.Contains(source, item) + if (body.Method.DeclaringType != typeof(Enumerable) || body.Arguments.Count != 2) + { + throw new NotSupportedException("Unexpected form of Enumerable.Contains"); + } + + list = body.Arguments[0]; + item = body.Arguments[1]; + } + + _sb.Append('('); + + Visit(item); + + _sb.Append(" IN "); + + // hardcode the integer list if it exists to bypass parameter limit + if (item.Type == typeof(int) && TryGetRightValue(list, out var value)) + { + var items = (IEnumerable)value; + _sb.Append('('); + _sb.Append(string.Join(", ", items)); + _sb.Append(')'); + + _gotConcreteValue = true; + } + else + { + Visit(list); + } + + _sb.Append(')'); + } + + private void ParseStringContains(MethodCallExpression body) + { + _sb.Append('('); + + Visit(body.Object); + + _sb.Append(" LIKE '%' || "); + + Visit(body.Arguments[0]); + + _sb.Append(" || '%')"); + } + + private void ParseStartsWith(MethodCallExpression body) + { + _sb.Append('('); + + Visit(body.Object); + + _sb.Append(" LIKE "); + + Visit(body.Arguments[0]); + + _sb.Append(" || '%')"); + } + + private void ParseEndsWith(MethodCallExpression body) + { + _sb.Append('('); + + Visit(body.Object); + + _sb.Append(" LIKE '%' || "); + + Visit(body.Arguments[0]); + + _sb.Append(')'); + } + + public override string ToString() + { + var sql = _sb.ToString(); + + if (_requireConcreteValue && !_gotConcreteValue) + { + var e = new InvalidOperationException("WhereBuilder requires a concrete condition"); + e.Data.Add("sql", sql); + throw e; + } + + return sql; + } + } +} diff --git a/src/NzbDrone.Core/History/HistoryRepository.cs b/src/NzbDrone.Core/History/HistoryRepository.cs index 5686e9ba2..4816b3ded 100644 --- a/src/NzbDrone.Core/History/HistoryRepository.cs +++ b/src/NzbDrone.Core/History/HistoryRepository.cs @@ -90,11 +90,11 @@ namespace NzbDrone.Core.History public List FindDownloadHistory(int idAuthorId, QualityModel quality) { - var allowed = new[] { EntityHistoryEventType.Grabbed, EntityHistoryEventType.DownloadFailed, EntityHistoryEventType.BookFileImported }; + var allowed = new[] { (int)EntityHistoryEventType.Grabbed, (int)EntityHistoryEventType.DownloadFailed, (int)EntityHistoryEventType.BookFileImported }; return Query(h => h.AuthorId == idAuthorId && h.Quality == quality && - allowed.Contains(h.EventType)); + allowed.Contains((int)h.EventType)); } public void DeleteForAuthor(int authorId) @@ -102,7 +102,7 @@ namespace NzbDrone.Core.History Delete(c => c.AuthorId == authorId); } - protected override SqlBuilder PagedBuilder() => new SqlBuilder() + protected override SqlBuilder PagedBuilder() => new SqlBuilder(_database.DatabaseType) .Join((h, a) => h.AuthorId == a.Id) .Join((l, r) => l.AuthorMetadataId == r.Id) .Join((h, a) => h.BookId == a.Id); diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAbsolutePathMetadataFiles.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAbsolutePathMetadataFiles.cs index 1293d4f07..8d50f05de 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAbsolutePathMetadataFiles.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAbsolutePathMetadataFiles.cs @@ -1,4 +1,4 @@ -using Dapper; +using Dapper; using NzbDrone.Core.Datastore; namespace NzbDrone.Core.Housekeeping.Housekeepers @@ -16,16 +16,32 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM MetadataFiles - WHERE Id IN ( - SELECT Id FROM MetadataFiles - WHERE RelativePath + if (_database.DatabaseType == DatabaseType.PostgreSQL) + { + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" = ANY ( + SELECT ""Id"" FROM ""MetadataFiles"" + WHERE ""RelativePath"" + LIKE '_:\\%' + OR ""RelativePath"" + LIKE '\\%' + OR ""RelativePath"" + LIKE '/%' + )"); + } + else + { + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" IN ( + SELECT ""Id"" FROM ""MetadataFiles"" + WHERE ""RelativePath"" LIKE '_:\%' - OR RelativePath + OR ""RelativePath"" LIKE '\%' - OR RelativePath + OR ""RelativePath"" LIKE '/%' )"); + } } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalNamingSpecs.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalNamingSpecs.cs index a1d9c56b7..d9c61e790 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalNamingSpecs.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalNamingSpecs.cs @@ -16,9 +16,9 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM NamingConfig - WHERE ID NOT IN ( - SELECT ID FROM NamingConfig + mapper.Execute(@"DELETE FROM ""NamingConfig"" + WHERE ""Id"" NOT IN ( + SELECT ""Id"" FROM ""NamingConfig"" LIMIT 1)"); } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalUsers.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalUsers.cs index 4460ac83c..3845110f4 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalUsers.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupAdditionalUsers.cs @@ -16,9 +16,9 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM Users - WHERE ID NOT IN ( - SELECT ID FROM Users + mapper.Execute(@"DELETE FROM ""Users"" + WHERE ""Id"" NOT IN ( + SELECT ""Id"" FROM ""Users"" LIMIT 1)"); } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs index 1e331c98b..ea3ffeea9 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs @@ -16,16 +16,29 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers public void Clean() { - using (var mapper = _database.OpenConnection()) + var mapper = _database.OpenConnection(); + + if (_database.DatabaseType == DatabaseType.PostgreSQL) + { + mapper.Execute(@"DELETE FROM ""PendingReleases"" + WHERE ""Added"" < @TwoWeeksAgo + AND ""Reason"" = ANY (@Reasons)", + new + { + TwoWeeksAgo = DateTime.UtcNow.AddDays(-14), + Reasons = new[] { (int)PendingReleaseReason.DownloadClientUnavailable, (int)PendingReleaseReason.Fallback } + }); + } + else { - mapper.Execute(@"DELETE FROM PendingReleases - WHERE Added < @TwoWeeksAgo - AND REASON IN @Reasons", - new - { - TwoWeeksAgo = DateTime.UtcNow.AddDays(-14), - Reasons = new[] { (int)PendingReleaseReason.DownloadClientUnavailable, (int)PendingReleaseReason.Fallback } - }); + mapper.Execute(@"DELETE FROM ""PendingReleases"" + WHERE ""Added"" < @TwoWeeksAgo + AND ""REASON"" IN @Reasons", + new + { + TwoWeeksAgo = DateTime.UtcNow.AddDays(-14), + Reasons = new[] { (int)PendingReleaseReason.DownloadClientUnavailable, (int)PendingReleaseReason.Fallback } + }); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDuplicateMetadataFiles.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDuplicateMetadataFiles.cs index 814976d46..ca13e3c4a 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDuplicateMetadataFiles.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDuplicateMetadataFiles.cs @@ -23,12 +23,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM MetadataFiles - WHERE Id IN ( - SELECT Id FROM MetadataFiles - WHERE Type = 1 - GROUP BY AuthorId, Consumer - HAVING COUNT(AuthorId) > 1 + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" IN ( + SELECT MIN(""Id"") FROM ""MetadataFiles"" + WHERE ""Type"" = 1 + GROUP BY ""AuthorId"", ""Consumer"" + HAVING COUNT(""AuthorId"") > 1 )"); } } @@ -37,12 +37,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM MetadataFiles - WHERE Id IN ( - SELECT Id FROM MetadataFiles - WHERE Type IN (2, 4) - GROUP BY BookId, Consumer - HAVING COUNT(BookId) > 1 + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" IN ( + SELECT MIN(""Id"") FROM ""MetadataFiles"" + WHERE ""Type"" IN (2, 4) + GROUP BY ""BookId"", ""Consumer"" + HAVING COUNT(""BookId"") > 1 )"); } } @@ -51,12 +51,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM MetadataFiles - WHERE Id IN ( - SELECT Id FROM MetadataFiles - WHERE Type IN (2, 4) - GROUP BY BookFileId, Consumer - HAVING COUNT(BookFileId) > 1 + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" IN ( + SELECT MIN(""Id"") FROM ""MetadataFiles"" + WHERE ""Type"" IN (2, 4) + GROUP BY ""BookFileId"", ""Consumer"" + HAVING COUNT(""BookFileId"") > 1 )"); } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedAuthorMetadata.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedAuthorMetadata.cs index 27d8f1188..0b64f930e 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedAuthorMetadata.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedAuthorMetadata.cs @@ -16,12 +16,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM AuthorMetadata - WHERE Id IN ( - SELECT AuthorMetadata.Id FROM AuthorMetadata - LEFT OUTER JOIN Books ON Books.AuthorMetadataId = AuthorMetadata.Id - LEFT OUTER JOIN Authors ON Authors.AuthorMetadataId = AuthorMetadata.Id - WHERE Books.Id IS NULL AND Authors.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""AuthorMetadata"" + WHERE ""Id"" IN ( + SELECT ""AuthorMetadata"".""Id"" FROM ""AuthorMetadata"" + LEFT OUTER JOIN ""Books"" ON ""Books"".""AuthorMetadataId"" = ""AuthorMetadata"".""Id"" + LEFT OUTER JOIN ""Authors"" ON ""Authors"".""AuthorMetadataId"" = ""AuthorMetadata"".""Id"" + WHERE ""Books"".""Id"" IS NULL AND ""Authors"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBlocklist.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBlocklist.cs index f99dcae7f..a4f670c22 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBlocklist.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBlocklist.cs @@ -16,12 +16,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM Blocklist - WHERE Id IN ( - SELECT Blocklist.Id FROM Blocklist - LEFT OUTER JOIN Authors - ON Blocklist.AuthorId = Authors.Id - WHERE Authors.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""Blocklist"" + WHERE ""Id"" IN ( + SELECT ""Blocklist"".""Id"" FROM ""Blocklist"" + LEFT OUTER JOIN ""Authors"" + ON ""Blocklist"".""AuthorId"" = ""Authors"".""Id"" + WHERE ""Authors"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBookFiles.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBookFiles.cs index 581fc841f..e7a396be4 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBookFiles.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBookFiles.cs @@ -17,13 +17,13 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers using (var mapper = _database.OpenConnection()) { // Unlink where track no longer exists - mapper.Execute(@"UPDATE BookFiles - SET EditionId = 0 - WHERE Id IN ( - SELECT BookFiles.Id FROM BookFiles - LEFT OUTER JOIN Editions - ON BookFiles.EditionId = Editions.Id - WHERE Editions.Id IS NULL)"); + mapper.Execute(@"UPDATE ""BookFiles"" + SET ""EditionId"" = 0 + WHERE ""Id"" IN ( + SELECT ""BookFiles"".""Id"" FROM ""BookFiles"" + LEFT OUTER JOIN ""Editions"" + ON ""BookFiles"".""EditionId"" = ""Editions"".""Id"" + WHERE ""Editions"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBooks.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBooks.cs index d67595fa6..16ddb2696 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBooks.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedBooks.cs @@ -16,12 +16,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM Books - WHERE Id IN ( - SELECT Books.Id FROM Books - LEFT OUTER JOIN Authors - ON Books.AuthorMetadataId = Authors.AuthorMetadataId - WHERE Authors.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""Books"" + WHERE ""Id"" IN ( + SELECT ""Books"".""Id"" FROM ""Books"" + LEFT OUTER JOIN ""Authors"" + ON ""Books"".""AuthorMetadataId"" = ""Authors"".""AuthorMetadataId"" + WHERE ""Authors"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs index 91598ace5..846ddae2f 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs @@ -14,15 +14,14 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers public void Clean() { - using (var mapper = _database.OpenConnection()) - { - mapper.Execute(@"DELETE FROM DownloadClientStatus - WHERE Id IN ( - SELECT DownloadClientStatus.Id FROM DownloadClientStatus - LEFT OUTER JOIN DownloadClients - ON DownloadClientStatus.ProviderId = DownloadClients.Id - WHERE DownloadClients.Id IS NULL)"); - } + var mapper = _database.OpenConnection(); + + mapper.Execute(@"DELETE FROM ""DownloadClientStatus"" + WHERE ""Id"" IN ( + SELECT ""DownloadClientStatus"".""Id"" FROM ""DownloadClientStatus"" + LEFT OUTER JOIN ""DownloadClients"" + ON ""DownloadClientStatus"".""ProviderId"" = ""DownloadClients"".""Id"" + WHERE ""DownloadClients"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedEditions.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedEditions.cs index 25159bc5d..af2ff7108 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedEditions.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedEditions.cs @@ -16,12 +16,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM Editions - WHERE Id IN ( - SELECT Editions.Id FROM Editions - LEFT OUTER JOIN Books - ON Editions.BookId = Books.Id - WHERE Books.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""Editions"" + WHERE ""Id"" IN ( + SELECT ""Editions"".""Id"" FROM ""Editions"" + LEFT OUTER JOIN ""Books"" + ON ""Editions"".""BookId"" = ""Books"".""Id"" + WHERE ""Books"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedHistoryItems.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedHistoryItems.cs index afbeca208..5b5d37b78 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedHistoryItems.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedHistoryItems.cs @@ -1,4 +1,4 @@ -using Dapper; +using Dapper; using NzbDrone.Core.Datastore; namespace NzbDrone.Core.Housekeeping.Housekeepers @@ -22,12 +22,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM History - WHERE Id IN ( - SELECT History.Id FROM History - LEFT OUTER JOIN Authors - ON History.AuthorId = Authors.Id - WHERE Authors.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""History"" + WHERE ""Id"" IN ( + SELECT ""History"".""Id"" FROM ""History"" + LEFT OUTER JOIN ""Authors"" + ON ""History"".""AuthorId"" = ""Authors"".""Id"" + WHERE ""Authors"".""Id"" IS NULL)"); } } @@ -35,12 +35,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM History - WHERE Id IN ( - SELECT History.Id FROM History - LEFT OUTER JOIN Books - ON History.BookId = Books.Id - WHERE Books.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""History"" + WHERE ""Id"" IN ( + SELECT ""History"".""Id"" FROM ""History"" + LEFT OUTER JOIN ""Books"" + ON ""History"".""BookId"" = ""Books"".""Id"" + WHERE ""Books"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedImportListStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedImportListStatus.cs index f40ec672f..1ceb0f396 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedImportListStatus.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedImportListStatus.cs @@ -16,12 +16,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM ImportListStatus - WHERE Id IN ( - SELECT ImportListStatus.Id FROM ImportListStatus - LEFT OUTER JOIN ImportLists - ON ImportListStatus.ProviderId = ImportLists.Id - WHERE ImportLists.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""ImportListStatus"" + WHERE ""Id"" IN ( + SELECT ""ImportListStatus"".""Id"" FROM ""ImportListStatus"" + LEFT OUTER JOIN ""ImportLists"" + ON ""ImportListStatus"".""ProviderId"" = ""ImportLists"".""Id"" + WHERE ""ImportLists"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs index 039db9b52..059f059e4 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedIndexerStatus.cs @@ -16,12 +16,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM IndexerStatus - WHERE Id IN ( - SELECT IndexerStatus.Id FROM IndexerStatus - LEFT OUTER JOIN Indexers - ON IndexerStatus.ProviderId = Indexers.Id - WHERE Indexers.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""IndexerStatus"" + WHERE ""Id"" IN ( + SELECT ""IndexerStatus"".""Id"" FROM ""IndexerStatus"" + LEFT OUTER JOIN ""Indexers"" + ON ""IndexerStatus"".""ProviderId"" = ""Indexers"".""Id"" + WHERE ""Indexers"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedMetadataFiles.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedMetadataFiles.cs index 0c0f838ca..f04533489 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedMetadataFiles.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedMetadataFiles.cs @@ -25,12 +25,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM MetadataFiles - WHERE Id IN ( - SELECT MetadataFiles.Id FROM MetadataFiles - LEFT OUTER JOIN Authors - ON MetadataFiles.AuthorId = Authors.Id - WHERE Authors.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" IN ( + SELECT ""MetadataFiles"".""Id"" FROM ""MetadataFiles"" + LEFT OUTER JOIN ""Authors"" + ON ""MetadataFiles"".""AuthorId"" = ""Authors"".""Id"" + WHERE ""Authors"".""Id"" IS NULL)"); } } @@ -38,13 +38,13 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM MetadataFiles - WHERE Id IN ( - SELECT MetadataFiles.Id FROM MetadataFiles - LEFT OUTER JOIN Books - ON MetadataFiles.BookId = Books.Id - WHERE MetadataFiles.BookId > 0 - AND Books.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" IN ( + SELECT ""MetadataFiles"".""Id"" FROM ""MetadataFiles"" + LEFT OUTER JOIN ""Books"" + ON ""MetadataFiles"".""BookId"" = ""Books"".""Id"" + WHERE ""MetadataFiles"".""BookId"" > 0 + AND ""Books"".""Id"" IS NULL)"); } } @@ -52,13 +52,13 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM MetadataFiles - WHERE Id IN ( - SELECT MetadataFiles.Id FROM MetadataFiles - LEFT OUTER JOIN BookFiles - ON MetadataFiles.BookFileId = BookFiles.Id - WHERE MetadataFiles.BookFileId > 0 - AND BookFiles.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" IN ( + SELECT ""MetadataFiles"".""Id"" FROM ""MetadataFiles"" + LEFT OUTER JOIN ""BookFiles"" + ON ""MetadataFiles"".""BookFileId"" = ""BookFiles"".""Id"" + WHERE ""MetadataFiles"".""BookFileId"" > 0 + AND ""BookFiles"".""Id"" IS NULL)"); } } @@ -66,11 +66,11 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM MetadataFiles - WHERE Id IN ( - SELECT Id FROM MetadataFiles - WHERE Type IN (2, 4) - AND BookId = 0)"); + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" IN ( + SELECT ""Id"" FROM ""MetadataFiles"" + WHERE ""Type"" IN (2, 4) + AND ""BookId"" = 0)"); } } @@ -78,11 +78,11 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM MetadataFiles - WHERE Id IN ( - SELECT Id FROM MetadataFiles - WHERE Type IN (2, 4) - AND BookFileId = 0)"); + mapper.Execute(@"DELETE FROM ""MetadataFiles"" + WHERE ""Id"" IN ( + SELECT ""Id"" FROM ""MetadataFiles"" + WHERE ""Type"" IN (2, 4) + AND ""BookFileId"" = 0)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedPendingReleases.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedPendingReleases.cs index e613481a4..2ae415117 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedPendingReleases.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedPendingReleases.cs @@ -16,12 +16,12 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM PendingReleases - WHERE Id IN ( - SELECT PendingReleases.Id FROM PendingReleases - LEFT OUTER JOIN Authors - ON PendingReleases.AuthorId = Authors.Id - WHERE Authors.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""PendingReleases"" + WHERE ""Id"" IN ( + SELECT ""PendingReleases"".""Id"" FROM ""PendingReleases"" + LEFT OUTER JOIN ""Authors"" + ON ""PendingReleases"".""AuthorId"" = ""Authors"".""Id"" + WHERE ""Authors"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedSeriesBookLinks.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedSeriesBookLinks.cs index 3da4157af..54edfb570 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedSeriesBookLinks.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedSeriesBookLinks.cs @@ -16,19 +16,19 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM SeriesBookLink - WHERE Id IN ( - SELECT SeriesBookLink.Id FROM SeriesBookLink - LEFT OUTER JOIN Books - ON SeriesBookLink.BookId = Books.Id - WHERE Books.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""SeriesBookLink"" + WHERE ""Id"" IN ( + SELECT ""SeriesBookLink"".""Id"" FROM ""SeriesBookLink"" + LEFT OUTER JOIN ""Books"" + ON ""SeriesBookLink"".""BookId"" = ""Books"".""Id"" + WHERE ""Books"".""Id"" IS NULL)"); - mapper.Execute(@"DELETE FROM SeriesBookLink - WHERE Id IN ( - SELECT SeriesBookLink.Id FROM SeriesBookLink - LEFT OUTER JOIN Series - ON SeriesBookLink.SeriesId = Series.Id - WHERE Series.Id IS NULL)"); + mapper.Execute(@"DELETE FROM ""SeriesBookLink"" + WHERE ""Id"" IN ( + SELECT ""SeriesBookLink"".""Id"" FROM ""SeriesBookLink"" + LEFT OUTER JOIN ""Series"" + ON ""SeriesBookLink"".""SeriesId"" = ""Series"".""Id"" + WHERE ""Series"".""Id"" IS NULL)"); } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupUnusedTags.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupUnusedTags.cs index 11462cc4b..7968e2e3a 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupUnusedTags.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupUnusedTags.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Data; using System.Linq; using Dapper; +using NzbDrone.Common.Extensions; using NzbDrone.Core.Datastore; namespace NzbDrone.Core.Housekeeping.Housekeepers @@ -24,15 +25,29 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers .Distinct() .ToArray(); - var usedTagsList = string.Join(",", usedTags.Select(d => d.ToString()).ToArray()); + if (usedTags.Any()) + { + var usedTagsList = usedTags.Select(d => d.ToString()).Join(","); - mapper.Execute($"DELETE FROM Tags WHERE NOT Id IN ({usedTagsList})"); + if (_database.DatabaseType == DatabaseType.PostgreSQL) + { + mapper.Execute($"DELETE FROM \"Tags\" WHERE NOT \"Id\" = ANY (\'{{{usedTagsList}}}\'::int[])"); + } + else + { + mapper.Execute($"DELETE FROM \"Tags\" WHERE NOT \"Id\" IN ({usedTagsList})"); + } + } + else + { + mapper.Execute("DELETE FROM \"Tags\""); + } } } private int[] GetUsedTags(string table, IDbConnection mapper) { - return mapper.Query>($"SELECT DISTINCT Tags FROM {table} WHERE NOT Tags = '[]'") + return mapper.Query>($"SELECT DISTINCT \"Tags\" FROM \"{table}\" WHERE NOT \"Tags\" = '[]' AND NOT \"Tags\" IS NULL") .SelectMany(x => x) .Distinct() .ToArray(); diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureRunScheduledTasks.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureRunScheduledTasks.cs index 82a7af7fa..47333e596 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureRunScheduledTasks.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureRunScheduledTasks.cs @@ -26,9 +26,9 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"UPDATE ScheduledTasks - SET LastExecution = @time - WHERE LastExecution > @time", + mapper.Execute(@"UPDATE ""ScheduledTasks"" + SET ""LastExecution"" = @time + WHERE ""LastExecution"" > @time", new { time = DateTime.UtcNow }); } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/TrimHttpCache.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/TrimHttpCache.cs index d90699dae..ac6037074 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/TrimHttpCache.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/TrimHttpCache.cs @@ -16,7 +16,7 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers { using (var mapper = _database.OpenConnection()) { - mapper.Execute(@"DELETE FROM HttpResponse WHERE Expiry < date('now')"); + mapper.Execute(@"DELETE FROM ""HttpResponse"" WHERE ""Expiry"" < date('now')"); } _database.Vacuum(); diff --git a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs index ccca19073..bcc566239 100644 --- a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs +++ b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs @@ -1,9 +1,11 @@ -using System.Data; +using System; +using System.Data; using System.Data.SQLite; using NLog; using NLog.Common; using NLog.Config; using NLog.Targets; +using Npgsql; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Datastore; using NzbDrone.Core.Lifecycle; @@ -13,7 +15,7 @@ namespace NzbDrone.Core.Instrumentation { public class DatabaseTarget : TargetWithLayout, IHandle { - private const string INSERT_COMMAND = "INSERT INTO [Logs]([Message],[Time],[Logger],[Exception],[ExceptionType],[Level]) " + + private const string INSERT_COMMAND = "INSERT INTO \"Logs\" (\"Message\",\"Time\",\"Logger\",\"Exception\",\"ExceptionType\",\"Level\") " + "VALUES(@Message,@Time,@Logger,@Exception,@ExceptionType,@Level)"; private readonly IConnectionStringFactory _connectionStringFactory; @@ -55,7 +57,6 @@ namespace NzbDrone.Core.Instrumentation { try { - using var connection = new SQLiteConnection(_connectionStringFactory.LogDbConnectionString).OpenAndReturn(); var log = new Log(); log.Time = logEvent.TimeStamp; log.Message = CleanseLogMessage.Cleanse(logEvent.FormattedMessage); @@ -84,16 +85,17 @@ namespace NzbDrone.Core.Instrumentation log.Level = logEvent.Level.Name; - var sqlCommand = new SQLiteCommand(INSERT_COMMAND, connection); + var connectionString = _connectionStringFactory.LogDbConnectionString; - sqlCommand.Parameters.Add(new SQLiteParameter("Message", DbType.String) { Value = log.Message }); - sqlCommand.Parameters.Add(new SQLiteParameter("Time", DbType.DateTime) { Value = log.Time.ToUniversalTime() }); - sqlCommand.Parameters.Add(new SQLiteParameter("Logger", DbType.String) { Value = log.Logger }); - sqlCommand.Parameters.Add(new SQLiteParameter("Exception", DbType.String) { Value = log.Exception }); - sqlCommand.Parameters.Add(new SQLiteParameter("ExceptionType", DbType.String) { Value = log.ExceptionType }); - sqlCommand.Parameters.Add(new SQLiteParameter("Level", DbType.String) { Value = log.Level }); - - sqlCommand.ExecuteNonQuery(); + //TODO: Probably need more robust way to differentiate what's being used + if (connectionString.Contains(".db")) + { + WriteSqliteLog(log, connectionString); + } + else + { + WritePostgresLog(log, connectionString); + } } catch (SQLiteException ex) { @@ -102,6 +104,48 @@ namespace NzbDrone.Core.Instrumentation } } + private void WritePostgresLog(Log log, string connectionString) + { + using (var connection = + new NpgsqlConnection(connectionString)) + { + connection.Open(); + using (var sqlCommand = connection.CreateCommand()) + { + sqlCommand.CommandText = INSERT_COMMAND; + sqlCommand.Parameters.Add(new NpgsqlParameter("Message", DbType.String) { Value = log.Message }); + sqlCommand.Parameters.Add(new NpgsqlParameter("Time", DbType.DateTime) { Value = log.Time.ToUniversalTime() }); + sqlCommand.Parameters.Add(new NpgsqlParameter("Logger", DbType.String) { Value = log.Logger }); + sqlCommand.Parameters.Add(new NpgsqlParameter("Exception", DbType.String) { Value = log.Exception == null ? DBNull.Value : log.Exception }); + sqlCommand.Parameters.Add(new NpgsqlParameter("ExceptionType", DbType.String) { Value = log.ExceptionType == null ? DBNull.Value : log.ExceptionType }); + sqlCommand.Parameters.Add(new NpgsqlParameter("Level", DbType.String) { Value = log.Level }); + + sqlCommand.ExecuteNonQuery(); + } + } + } + + private void WriteSqliteLog(Log log, string connectionString) + { + using (var connection = + SQLiteFactory.Instance.CreateConnection()) + { + connection.ConnectionString = connectionString; + connection.Open(); + using (var sqlCommand = connection.CreateCommand()) + { + sqlCommand.CommandText = INSERT_COMMAND; + sqlCommand.Parameters.Add(new SQLiteParameter("Message", DbType.String) { Value = log.Message }); + sqlCommand.Parameters.Add(new SQLiteParameter("Time", DbType.DateTime) { Value = log.Time.ToUniversalTime() }); + sqlCommand.Parameters.Add(new SQLiteParameter("Logger", DbType.String) { Value = log.Logger }); + sqlCommand.Parameters.Add(new SQLiteParameter("Exception", DbType.String) { Value = log.Exception }); + sqlCommand.Parameters.Add(new SQLiteParameter("ExceptionType", DbType.String) { Value = log.ExceptionType }); + sqlCommand.Parameters.Add(new SQLiteParameter("Level", DbType.String) { Value = log.Level }); + sqlCommand.ExecuteNonQuery(); + } + } + } + public void Handle(ApplicationShutdownRequested message) { if (LogManager.Configuration != null && LogManager.Configuration.LoggingRules.Contains(Rule)) diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json index 29668cac9..713dbb20a 100644 --- a/src/NzbDrone.Core/Localization/Core/en.json +++ b/src/NzbDrone.Core/Localization/Core/en.json @@ -142,6 +142,7 @@ "CutoffHelpText": "Once this quality is reached Readarr will no longer download books", "CutoffUnmet": "Cutoff Unmet", "DataAllBooks": "Monitor all books", + "Database": "Database", "DataExistingBooks": "Monitor books that have files or have not released yet", "DataFirstBook": "Monitor the first book. All other books will be ignored", "DataFuturebooks": "Monitor books that have not released yet", diff --git a/src/NzbDrone.Core/MediaFiles/MediaFileRepository.cs b/src/NzbDrone.Core/MediaFiles/MediaFileRepository.cs index c878f840f..c21c09734 100644 --- a/src/NzbDrone.Core/MediaFiles/MediaFileRepository.cs +++ b/src/NzbDrone.Core/MediaFiles/MediaFileRepository.cs @@ -32,7 +32,7 @@ namespace NzbDrone.Core.MediaFiles // always join with all the other good stuff // needed more often than not so better to load it all now - protected override SqlBuilder Builder() => new SqlBuilder() + protected override SqlBuilder Builder() => new SqlBuilder(_database.DatabaseType) .LeftJoin((b, e) => b.EditionId == e.Id) .LeftJoin((e, b) => e.BookId == b.Id) .LeftJoin((book, author) => book.AuthorMetadataId == author.AuthorMetadataId) @@ -86,7 +86,7 @@ namespace NzbDrone.Core.MediaFiles public List GetUnmappedFiles() { - return _database.Query(new SqlBuilder().Select(typeof(BookFile)) + return _database.Query(new SqlBuilder(_database.DatabaseType).Select(typeof(BookFile)) .Where(t => t.EditionId == 0)).ToList(); } @@ -107,7 +107,7 @@ namespace NzbDrone.Core.MediaFiles { // ensure path ends with a single trailing path separator to avoid matching partial paths var safePath = path.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar; - return _database.Query(new SqlBuilder().Where(x => x.Path.StartsWith(safePath))).ToList(); + return _database.Query(new SqlBuilder(_database.DatabaseType).Where(x => x.Path.StartsWith(safePath))).ToList(); } public BookFile GetFileWithPath(string path) @@ -118,7 +118,7 @@ namespace NzbDrone.Core.MediaFiles public List GetFileWithPath(List paths) { // use more limited join for speed - var builder = new SqlBuilder() + var builder = new SqlBuilder(_database.DatabaseType) .LeftJoin((f, t) => f.EditionId == t.Id); var all = _database.QueryJoined(builder, (file, book) => MapTrack(file, book)).ToList(); diff --git a/src/NzbDrone.Core/Messaging/Commands/CommandRepository.cs b/src/NzbDrone.Core/Messaging/Commands/CommandRepository.cs index 7bb1416cb..00b24ad59 100644 --- a/src/NzbDrone.Core/Messaging/Commands/CommandRepository.cs +++ b/src/NzbDrone.Core/Messaging/Commands/CommandRepository.cs @@ -31,7 +31,7 @@ namespace NzbDrone.Core.Messaging.Commands public void OrphanStarted() { - var sql = @"UPDATE Commands SET Status = @Orphaned, EndedAt = @Ended WHERE Status = @Started"; + var sql = @"UPDATE ""Commands"" SET ""Status"" = @Orphaned, ""EndedAt"" = @Ended WHERE ""Status"" = @Started"; var args = new { Orphaned = (int)CommandStatus.Orphaned, diff --git a/src/NzbDrone.Core/Readarr.Core.csproj b/src/NzbDrone.Core/Readarr.Core.csproj index 000ba70a7..57bbaaaf2 100644 --- a/src/NzbDrone.Core/Readarr.Core.csproj +++ b/src/NzbDrone.Core/Readarr.Core.csproj @@ -14,11 +14,13 @@ + + diff --git a/src/NzbDrone.Core/Update/UpdatePackageProvider.cs b/src/NzbDrone.Core/Update/UpdatePackageProvider.cs index 73dfb4d96..9c72adccf 100644 --- a/src/NzbDrone.Core/Update/UpdatePackageProvider.cs +++ b/src/NzbDrone.Core/Update/UpdatePackageProvider.cs @@ -5,6 +5,7 @@ using NzbDrone.Common.Cloud; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Http; using NzbDrone.Core.Analytics; +using NzbDrone.Core.Datastore; namespace NzbDrone.Core.Update { @@ -20,13 +21,15 @@ namespace NzbDrone.Core.Update private readonly IHttpRequestBuilderFactory _requestBuilder; private readonly IPlatformInfo _platformInfo; private readonly IAnalyticsService _analyticsService; + private readonly IMainDatabase _mainDatabase; - public UpdatePackageProvider(IHttpClient httpClient, IReadarrCloudRequestBuilder requestBuilder, IAnalyticsService analyticsService, IPlatformInfo platformInfo) + public UpdatePackageProvider(IHttpClient httpClient, IReadarrCloudRequestBuilder requestBuilder, IAnalyticsService analyticsService, IPlatformInfo platformInfo, IMainDatabase mainDatabase) { _platformInfo = platformInfo; _analyticsService = analyticsService; _requestBuilder = requestBuilder.Services; _httpClient = httpClient; + _mainDatabase = mainDatabase; } public UpdatePackage GetLatestUpdate(string branch, Version currentVersion) @@ -38,6 +41,7 @@ namespace NzbDrone.Core.Update .AddQueryParam("arch", RuntimeInformation.OSArchitecture) .AddQueryParam("runtime", PlatformInfo.Platform.ToString().ToLowerInvariant()) .AddQueryParam("runtimeVer", _platformInfo.Version) + .AddQueryParam("dbType", _mainDatabase.DatabaseType) .SetSegment("branch", branch); if (_analyticsService.IsEnabled) diff --git a/src/NzbDrone.Host.Test/ContainerFixture.cs b/src/NzbDrone.Host.Test/ContainerFixture.cs index 5a658b2c1..3a0f93d43 100644 --- a/src/NzbDrone.Host.Test/ContainerFixture.cs +++ b/src/NzbDrone.Host.Test/ContainerFixture.cs @@ -5,12 +5,14 @@ using DryIoc.Microsoft.DependencyInjection; using FluentAssertions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; using Moq; using NUnit.Framework; using NzbDrone.Common; using NzbDrone.Common.Composition.Extensions; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Instrumentation.Extensions; +using NzbDrone.Core.Datastore; using NzbDrone.Core.Datastore.Extensions; using NzbDrone.Core.Download; using NzbDrone.Core.Download.TrackedDownloads; @@ -43,6 +45,7 @@ namespace NzbDrone.App.Test // set up a dummy broadcaster and lifetime to allow tests to resolve container.RegisterInstance(new Mock().Object); container.RegisterInstance(new Mock().Object); + container.RegisterInstance>(new Mock>().Object); _container = container.GetServiceProvider(); } diff --git a/src/NzbDrone.Host/Bootstrap.cs b/src/NzbDrone.Host/Bootstrap.cs index 90aac1157..9257764aa 100644 --- a/src/NzbDrone.Host/Bootstrap.cs +++ b/src/NzbDrone.Host/Bootstrap.cs @@ -8,7 +8,6 @@ using System.Security.Cryptography.X509Certificates; using System.Text; using DryIoc; using DryIoc.Microsoft.DependencyInjection; -using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -26,6 +25,8 @@ using NzbDrone.Core.Configuration; using NzbDrone.Core.Datastore.Extensions; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; +using NzbDrone.Host; +using PostgresOptions = NzbDrone.Core.Datastore.PostgresOptions; namespace NzbDrone.Host { @@ -141,6 +142,10 @@ namespace NzbDrone.Host .AddStartupContext(context) .Resolve().PublishEvent(new ApplicationStartingEvent()); }) + .ConfigureServices(services => + { + services.Configure(config.GetSection("Readarr:Postgres")); + }) .ConfigureWebHost(builder => { builder.UseConfiguration(config); @@ -213,6 +218,7 @@ namespace NzbDrone.Host return new ConfigurationBuilder() .AddXmlFile(appFolder.GetConfigPath(), optional: true, reloadOnChange: false) .AddInMemoryCollection(new List> { new ("dataProtectionFolder", appFolder.GetDataProtectionPath()) }) + .AddEnvironmentVariables() .Build(); } diff --git a/src/NzbDrone.Integration.Test/IntegrationTest.cs b/src/NzbDrone.Integration.Test/IntegrationTest.cs index 22387c902..19a98963f 100644 --- a/src/NzbDrone.Integration.Test/IntegrationTest.cs +++ b/src/NzbDrone.Integration.Test/IntegrationTest.cs @@ -1,9 +1,15 @@ +using System.Collections.Generic; using System.Threading; +using Microsoft.Extensions.Configuration; using NLog; +using Npgsql; using NUnit.Framework; using NzbDrone.Common.Extensions; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Datastore.Migration.Framework; using NzbDrone.Core.Indexers.Newznab; using NzbDrone.Test.Common; +using NzbDrone.Test.Common.Datastore; using Readarr.Http.ClientSchema; namespace NzbDrone.Integration.Test @@ -19,6 +25,8 @@ namespace NzbDrone.Integration.Test protected int Port { get; private set; } + protected PostgresOptions PostgresOptions { get; set; } = new (); + protected override string RootUrl => $"http://localhost:{Port}/"; protected override string ApiKey => _runner.ApiKey; @@ -27,7 +35,14 @@ namespace NzbDrone.Integration.Test { Port = Interlocked.Increment(ref StaticPort); - _runner = new NzbDroneRunner(LogManager.GetCurrentClassLogger(), Port); + PostgresOptions = PostgresDatabase.GetTestOptions(); + + if (PostgresOptions?.Host != null) + { + CreatePostgresDb(PostgresOptions); + } + + _runner = new NzbDroneRunner(LogManager.GetCurrentClassLogger(), PostgresOptions, Port); _runner.Kill(); _runner.Start(); @@ -59,6 +74,22 @@ namespace NzbDrone.Integration.Test protected override void StopTestTarget() { _runner.Kill(); + if (PostgresOptions?.Host != null) + { + DropPostgresDb(PostgresOptions); + } + } + + private static void CreatePostgresDb(PostgresOptions options) + { + PostgresDatabase.Create(options, MigrationType.Main); + PostgresDatabase.Create(options, MigrationType.Log); + } + + private static void DropPostgresDb(PostgresOptions options) + { + PostgresDatabase.Drop(options, MigrationType.Main); + PostgresDatabase.Drop(options, MigrationType.Log); } } } diff --git a/src/NzbDrone.Test.Common/Datastore/PostgresDatabase.cs b/src/NzbDrone.Test.Common/Datastore/PostgresDatabase.cs new file mode 100644 index 000000000..940334ca2 --- /dev/null +++ b/src/NzbDrone.Test.Common/Datastore/PostgresDatabase.cs @@ -0,0 +1,69 @@ +using System; +using Npgsql; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Test.Common.Datastore +{ + public static class PostgresDatabase + { + public static PostgresOptions GetTestOptions() + { + var options = PostgresOptions.GetOptions(); + + var uid = TestBase.GetUID(); + options.MainDb = uid + "_main"; + options.LogDb = uid + "_log"; + + return options; + } + + public static void Create(PostgresOptions options, MigrationType migrationType) + { + var db = GetDatabaseName(options, migrationType); + var connectionString = GetConnectionString(options); + using var conn = new NpgsqlConnection(connectionString); + conn.Open(); + + using var cmd = conn.CreateCommand(); + cmd.CommandText = $"CREATE DATABASE \"{db}\" WITH OWNER = {options.User} ENCODING = 'UTF8' CONNECTION LIMIT = -1;"; + cmd.ExecuteNonQuery(); + } + + public static void Drop(PostgresOptions options, MigrationType migrationType) + { + var db = GetDatabaseName(options, migrationType); + var connectionString = GetConnectionString(options); + using var conn = new NpgsqlConnection(connectionString); + conn.Open(); + + using var cmd = conn.CreateCommand(); + cmd.CommandText = $"DROP DATABASE \"{db}\" WITH (FORCE);"; + cmd.ExecuteNonQuery(); + } + + private static string GetConnectionString(PostgresOptions options) + { + var builder = new NpgsqlConnectionStringBuilder() + { + Host = options.Host, + Port = options.Port, + Username = options.User, + Password = options.Password, + Enlist = false + }; + + return builder.ConnectionString; + } + + private static string GetDatabaseName(PostgresOptions options, MigrationType migrationType) + { + return migrationType switch + { + MigrationType.Main => options.MainDb, + MigrationType.Log => options.LogDb, + _ => throw new NotImplementedException("Unknown migration type") + }; + } + } +} diff --git a/src/NzbDrone.Test.Common/Datastore/SqliteDatabase.cs b/src/NzbDrone.Test.Common/Datastore/SqliteDatabase.cs new file mode 100644 index 000000000..151e245fc --- /dev/null +++ b/src/NzbDrone.Test.Common/Datastore/SqliteDatabase.cs @@ -0,0 +1,14 @@ +using System.IO; +using NUnit.Framework; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Test.Common.Datastore +{ + public static class SqliteDatabase + { + public static string GetCachedDb(MigrationType type) + { + return Path.Combine(TestContext.CurrentContext.TestDirectory, $"cached_{type}.db"); + } + } +} diff --git a/src/NzbDrone.Test.Common/NzbDroneRunner.cs b/src/NzbDrone.Test.Common/NzbDroneRunner.cs index 6b8ef300c..a10627ea4 100644 --- a/src/NzbDrone.Test.Common/NzbDroneRunner.cs +++ b/src/NzbDrone.Test.Common/NzbDroneRunner.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.Specialized; using System.Diagnostics; using System.IO; using System.Threading; @@ -9,7 +10,9 @@ using NUnit.Framework; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; using NzbDrone.Common.Processes; +using NzbDrone.Common.Serializer; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Datastore; using RestSharp; namespace NzbDrone.Test.Common @@ -23,13 +26,15 @@ namespace NzbDrone.Test.Common public string AppData { get; private set; } public string ApiKey { get; private set; } + public PostgresOptions PostgresOptions { get; private set; } public int Port { get; private set; } - public NzbDroneRunner(Logger logger, int port = 8787) + public NzbDroneRunner(Logger logger, PostgresOptions postgresOptions, int port = 8787) { _processProvider = new ProcessProvider(logger); _restClient = new RestClient($"http://localhost:{port}/api/v1"); + PostgresOptions = postgresOptions; Port = port; } @@ -142,10 +147,24 @@ namespace NzbDrone.Test.Common private void Start(string outputNzbdroneConsoleExe) { + StringDictionary envVars = new (); + if (PostgresOptions?.Host != null) + { + envVars.Add("Readarr__Postgres__Host", PostgresOptions.Host); + envVars.Add("Readarr__Postgres__Port", PostgresOptions.Port.ToString()); + envVars.Add("Readarr__Postgres__User", PostgresOptions.User); + envVars.Add("Readarr__Postgres__Password", PostgresOptions.Password); + envVars.Add("Readarr__Postgres__MainDb", PostgresOptions.MainDb); + envVars.Add("Readarr__Postgres__LogDb", PostgresOptions.LogDb); + envVars.Add("Readarr__Postgres__CacheDb", PostgresOptions.CacheDb); + + TestContext.Progress.WriteLine("Using env vars:\n{0}", envVars.ToJson()); + } + TestContext.Progress.WriteLine("Starting instance from {0} on port {1}", outputNzbdroneConsoleExe, Port); var args = "-nobrowser -nosingleinstancecheck -data=\"" + AppData + "\""; - _nzbDroneProcess = _processProvider.Start(outputNzbdroneConsoleExe, args, null, OnOutputDataReceived, OnOutputDataReceived); + _nzbDroneProcess = _processProvider.Start(outputNzbdroneConsoleExe, args, envVars, OnOutputDataReceived, OnOutputDataReceived); } private void OnOutputDataReceived(string data) diff --git a/src/Readarr.Api.V1/System/SystemController.cs b/src/Readarr.Api.V1/System/SystemController.cs index 52e1b2d22..24f47b09d 100644 --- a/src/Readarr.Api.V1/System/SystemController.cs +++ b/src/Readarr.Api.V1/System/SystemController.cs @@ -79,7 +79,8 @@ namespace Readarr.Api.V1.System Mode = _runtimeInfo.Mode, Branch = _configFileProvider.Branch, Authentication = _configFileProvider.AuthenticationMethod, - SqliteVersion = _database.Version, + DatabaseType = _database.DatabaseType, + DatabaseVersion = _database.Version, MigrationVersion = _database.Migration, UrlBase = _configFileProvider.UrlBase, RuntimeVersion = _platformInfo.Version, diff --git a/src/postgres.runsettings b/src/postgres.runsettings new file mode 100644 index 000000000..38f2e0270 --- /dev/null +++ b/src/postgres.runsettings @@ -0,0 +1,11 @@ + + + + + 192.168.100.5 + 5432 + abc + abc + + +