Fixed: Speed up RSS sync

src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetMovieFixture.cs

@@ -28,7 +28,7 @@ namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests
             Subject.GetMovie(title);
 
             Mocker.GetMock<IMovieService>()
-                  .Verify(s => s.FindByTitle(Parser.Parser.ParseMovieTitle(title, false).MovieTitle), Times.Once());
+                .Verify(s => s.FindByTitle(Parser.Parser.ParseMovieTitle(title, false).MovieTitle, It.IsAny<int>(), null, null, null), Times.Once());
         }
 
         /*[Test]

src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs

@@ -118,7 +118,7 @@ namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests
             Subject.Map(_parsedMovieInfo, "", null);
 
             Mocker.GetMock<IMovieService>()
-                  .Verify(v => v.FindByTitle(It.IsAny<string>(), It.IsAny<int>()), Times.Once());
+                .Verify(v => v.FindByTitle(It.IsAny<string>(), It.IsAny<int>(), null, null, null), Times.Once());
         }
 
         [Test]

src/NzbDrone.Core/Datastore/Migration/185_add_alternative_title_indices.cs

@@ -0,0 +1,15 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(185)]
+    public class add_alternative_title_indices : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Create.Index().OnTable("AlternativeTitles").OnColumn("CleanTitle");
+            Create.Index().OnTable("MovieTranslations").OnColumn("CleanTitle");
+        }
+    }
+}

src/NzbDrone.Core/Movies/MovieRepository.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using Dapper;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Datastore;
 using NzbDrone.Core.MediaFiles;
 using NzbDrone.Core.Messaging.Events;
@@ -109,13 +110,28 @@ namespace NzbDrone.Core.Movies
         public List<Movie> FindByTitles(List<string> titles)
         {
             var distinct = titles.Distinct().ToList();
+
+            var results = new List<Movie>();
+
+            results.AddRange(FindByMovieTitles(distinct));
+            results.AddRange(FindByAltTitles(distinct));
+            results.AddRange(FindByTransTitles(distinct));
+
+            return results.DistinctBy(x => x.Id).ToList();
+        }
+
+        // This is a bit of a hack, but if you try to combine / rationalise these then
+        // SQLite makes a mess of the query plan and ends up doing a table scan
+        private List<Movie> FindByMovieTitles(List<string> titles)
+        {
             var movieDictionary = new Dictionary<int, Movie>();
 
-            var builder = Builder()
+            var builder = new SqlBuilder()
+                .LeftJoin<Movie, AlternativeTitle>((m, t) => m.Id == t.MovieId)
+                .LeftJoin<Movie, MovieFile>((m, f) => m.Id == f.MovieId)
                 .LeftJoin<Movie, MovieTranslation>((m, tr) => m.Id == tr.MovieId)
-                .OrWhere<Movie>(x => distinct.Contains(x.CleanTitle))
+                .Join<Movie, Profile>((m, p) => m.ProfileId == p.Id)
-                .OrWhere<AlternativeTitle>(x => distinct.Contains(x.CleanTitle))
+                .Where<Movie>(x => titles.Contains(x.CleanTitle));
-                .OrWhere<MovieTranslation>(x => distinct.Contains(x.CleanTitle));
 
             _ = _database.QueryJoined<Movie, Profile, AlternativeTitle, MovieFile, MovieTranslation>(
                 builder,
@@ -124,6 +140,50 @@ namespace NzbDrone.Core.Movies
             return movieDictionary.Values.ToList();
         }
 
+        private List<Movie> FindByAltTitles(List<string> titles)
+        {
+            var movieDictionary = new Dictionary<int, Movie>();
+
+            var builder = new SqlBuilder()
+                .LeftJoin<AlternativeTitle, Movie>((t, m) => t.MovieId == m.Id)
+                .LeftJoin<Movie, MovieFile>((m, f) => m.Id == f.MovieId)
+                .LeftJoin<Movie, MovieTranslation>((m, tr) => m.Id == tr.MovieId)
+                .Join<Movie, Profile>((m, p) => m.ProfileId == p.Id)
+                .Where<AlternativeTitle>(x => titles.Contains(x.CleanTitle));
+
+            _ = _database.QueryJoined<AlternativeTitle, Profile, Movie, MovieFile, MovieTranslation>(
+                builder,
+                (altTitle, profile, movie, file, trans) =>
+                {
+                    _ = Map(movieDictionary, movie, profile, altTitle, file, trans);
+                    return null;
+                });
+
+            return movieDictionary.Values.ToList();
+        }
+
+        private List<Movie> FindByTransTitles(List<string> titles)
+        {
+            var movieDictionary = new Dictionary<int, Movie>();
+
+            var builder = new SqlBuilder()
+                .LeftJoin<MovieTranslation, Movie>((tr, m) => tr.MovieId == m.Id)
+                .LeftJoin<Movie, AlternativeTitle>((m, t) => m.Id == t.MovieId)
+                .LeftJoin<Movie, MovieFile>((m, f) => m.Id == f.MovieId)
+                .Join<Movie, Profile>((m, p) => m.ProfileId == p.Id)
+                .Where<MovieTranslation>(x => titles.Contains(x.CleanTitle));
+
+            _ = _database.QueryJoined<MovieTranslation, Profile, Movie, MovieFile, AlternativeTitle>(
+                builder,
+                (trans, profile, movie, file, altTitle) =>
+                {
+                    _ = Map(movieDictionary, movie, profile, altTitle, file, trans);
+                    return null;
+                });
+
+            return movieDictionary.Values.ToList();
+        }
+
         public Movie FindByImdbId(string imdbid)
         {
             var imdbIdWithPrefix = Parser.Parser.NormalizeImdbId(imdbid);

src/NzbDrone.Core/Movies/MovieService.cs

@@ -27,6 +27,8 @@ namespace NzbDrone.Core.Movies
         List<Movie> FindByTmdbId(List<int> tmdbids);
         Movie FindByTitle(string title);
         Movie FindByTitle(string title, int year);
+        Movie FindByTitle(string title, int? year, string arabicTitle, string romanTitle, List<Movie> candidates);
+        List<Movie> FindByTitleCandidates(string title, out string roman, out string arabic);
         Movie FindByTitleSlug(string slug);
         Movie FindByPath(string path);
         List<string> AllMoviePaths();
@@ -103,45 +105,35 @@ namespace NzbDrone.Core.Movies
 
         public Movie FindByTitle(string title)
         {
-            return FindByTitle(title.CleanMovieTitle(), null);
+            var candidates = FindByTitleCandidates(title, out var arabicTitle, out var romanTitle);
+
+            return FindByTitle(title, null, arabicTitle, romanTitle, candidates);
         }
 
         public Movie FindByTitle(string title, int year)
         {
-            return FindByTitle(title.CleanMovieTitle(), year as int?);
+            var candidates = FindByTitleCandidates(title, out var arabicTitle, out var romanTitle);
+
+            return FindByTitle(title, year, arabicTitle, romanTitle, candidates);
         }
 
-        private Movie FindByTitle(string cleanTitle, int? year)
+        public Movie FindByTitle(string cleanTitle, int? year, string arabicTitle, string romanTitle, List<Movie> candidates)
         {
-            cleanTitle = cleanTitle.ToLowerInvariant();
-            var cleanTitleWithRomanNumbers = cleanTitle;
-            var cleanTitleWithArabicNumbers = cleanTitle;
-
-            foreach (var arabicRomanNumeral in RomanNumeralParser.GetArabicRomanNumeralsMapping())
-            {
-                var arabicNumber = arabicRomanNumeral.ArabicNumeralAsString;
-                var romanNumber = arabicRomanNumeral.RomanNumeral;
-                cleanTitleWithRomanNumbers = cleanTitleWithRomanNumbers.Replace(arabicNumber, romanNumber);
-                cleanTitleWithArabicNumbers = cleanTitleWithArabicNumbers.Replace(romanNumber, arabicNumber);
-            }
-
-            var candidates = _movieRepository.FindByTitles(new List<string> { cleanTitle, cleanTitleWithArabicNumbers, cleanTitleWithRomanNumbers });
-
             var result = candidates.Where(x => x.CleanTitle == cleanTitle).FirstWithYear(year);
 
             if (result == null)
             {
                 result =
-                    candidates.Where(movie => movie.CleanTitle == cleanTitleWithArabicNumbers).FirstWithYear(year) ??
+                    candidates.Where(movie => movie.CleanTitle == arabicTitle).FirstWithYear(year) ??
-                    candidates.Where(movie => movie.CleanTitle == cleanTitleWithRomanNumbers).FirstWithYear(year);
+                    candidates.Where(movie => movie.CleanTitle == romanTitle).FirstWithYear(year);
             }
 
             if (result == null)
             {
                 result = candidates
                     .Where(m => m.AlternativeTitles.Any(t => t.CleanTitle == cleanTitle ||
-                                                        t.CleanTitle == cleanTitleWithArabicNumbers ||
+                                                        t.CleanTitle == arabicTitle ||
-                                                        t.CleanTitle == cleanTitleWithRomanNumbers))
+                                                        t.CleanTitle == romanTitle))
                     .FirstWithYear(year);
             }
 
@@ -149,14 +141,34 @@ namespace NzbDrone.Core.Movies
             {
                 result = candidates
                     .Where(m => m.Translations.Any(t => t.CleanTitle == cleanTitle ||
-                                                        t.CleanTitle == cleanTitleWithArabicNumbers ||
+                                                        t.CleanTitle == arabicTitle ||
-                                                        t.CleanTitle == cleanTitleWithRomanNumbers))
+                                                        t.CleanTitle == romanTitle))
                     .FirstWithYear(year);
             }
 
             return result;
         }
 
+        public List<Movie> FindByTitleCandidates(string title, out string arabicTitle, out string romanTitle)
+        {
+            var cleanTitle = title.CleanMovieTitle().ToLowerInvariant();
+            romanTitle = cleanTitle;
+            arabicTitle = cleanTitle;
+
+            foreach (var arabicRomanNumeral in RomanNumeralParser.GetArabicRomanNumeralsMapping())
+            {
+                var arabicNumber = arabicRomanNumeral.ArabicNumeralAsString;
+                var romanNumber = arabicRomanNumeral.RomanNumeral;
+
+                romanTitle = romanTitle.Replace(arabicNumber, romanNumber);
+                arabicTitle = arabicTitle.Replace(romanNumber, arabicNumber);
+            }
+
+            romanTitle = romanTitle.ToLowerInvariant();
+
+            return _movieRepository.FindByTitles(new List<string> { cleanTitle, arabicTitle, romanTitle });
+        }
+
         public Movie FindByImdbId(string imdbid)
         {
             return _movieRepository.FindByImdbId(imdbid);

src/NzbDrone.Core/Parser/ParsingService.cs

@@ -201,20 +201,20 @@ namespace NzbDrone.Core.Parser
 
         private bool TryGetMovieByTitleAndOrYear(ParsedMovieInfo parsedMovieInfo, out MappingResult result)
         {
-            Func<Movie, bool> isNotNull = movie => movie != null;
+            var candidates = _movieService.FindByTitleCandidates(parsedMovieInfo.MovieTitle, out var arabicTitle, out var romanTitle);
-            Movie movieByTitleAndOrYear;
 
+            Movie movieByTitleAndOrYear;
             if (parsedMovieInfo.Year > 1800)
             {
-                movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year);
+                movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year, arabicTitle, romanTitle, candidates);
-                if (isNotNull(movieByTitleAndOrYear))
+                if (movieByTitleAndOrYear != null)
                 {
                     result = new MappingResult { Movie = movieByTitleAndOrYear };
                     return true;
                 }
 
-                movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle);
+                movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, null, arabicTitle, romanTitle, candidates);
-                if (isNotNull(movieByTitleAndOrYear))
+                if (movieByTitleAndOrYear != null)
                 {
                     result = new MappingResult { Movie = movieByTitleAndOrYear, MappingResultType = MappingResultType.WrongYear };
                     return false;
@@ -224,8 +224,8 @@ namespace NzbDrone.Core.Parser
                 return false;
             }
 
-            movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle);
+            movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, null, arabicTitle, romanTitle, candidates);
-            if (isNotNull(movieByTitleAndOrYear))
+            if (movieByTitleAndOrYear != null)
             {
                 result = new MappingResult { Movie = movieByTitleAndOrYear };
                 return true;