From 6d033c57f4fac83e8932bf6faa103c8ad450d1f1 Mon Sep 17 00:00:00 2001 From: Leonardo Galli Date: Sun, 18 Jun 2017 23:12:14 +0200 Subject: [PATCH] Added: More detailed descriptions why a movie was not able to be mapped. (#1696) Added: Option to make mapping more lenient. This should practically allow all movies to be correctly mapped. Though it also opens the path for movies being wrongly mapped! (So it is a toggable option) Added: Improved edition parsing. Now almost all releases should have the correct edition, even ones with no year, etc. --- .../DownloadDecisionMakerFixture.cs | 17 +- .../DownloadClientFixtureBase.cs | 2 +- .../TrackedDownloadServiceFixture.cs | 2 +- .../IndexerIntegrationTests.cs | 1 + .../ParserTests/ParserFixture.cs | 15 +- .../ParsingServiceTests/MapFixture.cs | 28 ++- .../DecisionEngine/DownloadDecisionMaker.cs | 124 ++++++---- .../TrackedDownloadService.cs | 4 +- .../MetadataSource/PreDB/PreDBService.cs | 2 +- src/NzbDrone.Core/Parser/Parser.cs | 77 +++++- src/NzbDrone.Core/Parser/ParsingService.cs | 226 ++++++++++++++---- src/NzbDrone.Core/Tv/MovieService.cs | 33 ++- .../Options/LeniencyTooltipTemplate.hbs | 12 +- 13 files changed, 414 insertions(+), 129 deletions(-) diff --git a/src/NzbDrone.Core.Test/DecisionEngineTests/DownloadDecisionMakerFixture.cs b/src/NzbDrone.Core.Test/DecisionEngineTests/DownloadDecisionMakerFixture.cs index f99c499c4..9673a3978 100644 --- a/src/NzbDrone.Core.Test/DecisionEngineTests/DownloadDecisionMakerFixture.cs +++ b/src/NzbDrone.Core.Test/DecisionEngineTests/DownloadDecisionMakerFixture.cs @@ -19,6 +19,7 @@ namespace NzbDrone.Core.Test.DecisionEngineTests { private List _reports; private RemoteMovie _remoteEpisode; + private MappingResult _mappingResult; private Mock _pass1; private Mock _pass2; @@ -50,11 +51,15 @@ namespace NzbDrone.Core.Test.DecisionEngineTests _reports = new List { new ReleaseInfo { Title = "Trolls.2016.720p.WEB-DL.DD5.1.H264-FGT" } }; _remoteEpisode = new RemoteMovie { Movie = new Movie(), + ParsedMovieInfo = new ParsedMovieInfo() }; + _mappingResult = new MappingResult {Movie = new Movie(), MappingResultType = MappingResultType.Success}; + _mappingResult.RemoteMovie = _remoteEpisode; + + Mocker.GetMock() - .Setup(c => c.Map(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(_remoteEpisode); + .Setup(c => c.Map(It.IsAny(), It.IsAny(), It.IsAny())).Returns(_mappingResult); } private void GivenSpecifications(params Mock[] mocks) @@ -121,6 +126,7 @@ namespace NzbDrone.Core.Test.DecisionEngineTests { GivenSpecifications(_pass1, _pass2, _pass3); _reports[0].Title = "Not parsable"; + _mappingResult.MappingResultType = MappingResultType.NotParsable; var results = Subject.GetRssDecision(_reports).ToList(); @@ -130,7 +136,7 @@ namespace NzbDrone.Core.Test.DecisionEngineTests _pass2.Verify(c => c.IsSatisfiedBy(It.IsAny(), null), Times.Never()); _pass3.Verify(c => c.IsSatisfiedBy(It.IsAny(), null), Times.Never()); - results.Should().BeEmpty(); + results.Should().NotBeEmpty(); } [Test] @@ -138,6 +144,7 @@ namespace NzbDrone.Core.Test.DecisionEngineTests { GivenSpecifications(_pass1, _pass2, _pass3); _reports[0].Title = "1937 - Snow White and the Seven Dwarves"; + _mappingResult.MappingResultType = MappingResultType.NotParsable; var results = Subject.GetRssDecision(_reports).ToList(); @@ -147,7 +154,7 @@ namespace NzbDrone.Core.Test.DecisionEngineTests _pass2.Verify(c => c.IsSatisfiedBy(It.IsAny(), null), Times.Never()); _pass3.Verify(c => c.IsSatisfiedBy(It.IsAny(), null), Times.Never()); - results.Should().BeEmpty(); + results.Should().NotBeEmpty(); } [Test] @@ -156,6 +163,7 @@ namespace NzbDrone.Core.Test.DecisionEngineTests GivenSpecifications(_pass1, _pass2, _pass3); _remoteEpisode.Movie = null; + _mappingResult.MappingResultType = MappingResultType.TitleNotFound; Subject.GetRssDecision(_reports); @@ -249,6 +257,7 @@ namespace NzbDrone.Core.Test.DecisionEngineTests GivenSpecifications(_pass1, _pass2, _pass3); _remoteEpisode.Movie = null; + _mappingResult.MappingResultType = MappingResultType.TitleNotFound; var result = Subject.GetRssDecision(_reports); diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/DownloadClientFixtureBase.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/DownloadClientFixtureBase.cs index 274c979b1..ae34ba53e 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/DownloadClientFixtureBase.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/DownloadClientFixtureBase.cs @@ -31,7 +31,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests Mocker.GetMock() .Setup(s => s.Map(It.IsAny(), It.IsAny(), (SearchCriteriaBase)null)) - .Returns(() => CreateRemoteMovie()); + .Returns(() => new MappingResult{RemoteMovie = CreateRemoteMovie(), MappingResultType = MappingResultType.Success}); Mocker.GetMock() .Setup(s => s.Get(It.IsAny())) diff --git a/src/NzbDrone.Core.Test/Download/TrackedDownloads/TrackedDownloadServiceFixture.cs b/src/NzbDrone.Core.Test/Download/TrackedDownloads/TrackedDownloadServiceFixture.cs index bd5855d1d..a70934be4 100644 --- a/src/NzbDrone.Core.Test/Download/TrackedDownloads/TrackedDownloadServiceFixture.cs +++ b/src/NzbDrone.Core.Test/Download/TrackedDownloads/TrackedDownloadServiceFixture.cs @@ -50,7 +50,7 @@ namespace NzbDrone.Core.Test.Download.TrackedDownloads Mocker.GetMock() .Setup(s => s.Map(It.Is(i => i.MovieTitle == "A Movie"), It.IsAny(), null)) - .Returns(remoteEpisode); + .Returns(new MappingResult{RemoteMovie = remoteEpisode}); var client = new DownloadClientDefinition() { diff --git a/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs b/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs index a72bb8f57..b9ae64a9b 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs @@ -14,6 +14,7 @@ using NzbDrone.Test.Common.Categories; namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests { [IntegrationTest] + [Ignore("Nyaa is down!")] public class IndexerIntegrationTests : CoreTest { private SingleEpisodeSearchCriteria _singleSearchCriteria; diff --git a/src/NzbDrone.Core.Test/ParserTests/ParserFixture.cs b/src/NzbDrone.Core.Test/ParserTests/ParserFixture.cs index 77736d7c5..94cd22956 100644 --- a/src/NzbDrone.Core.Test/ParserTests/ParserFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/ParserFixture.cs @@ -82,6 +82,8 @@ namespace NzbDrone.Core.Test.ParserTests [TestCase("Valana la Legende FRENCH BluRay 720p 2016 kjhlj", "Valana la Legende")] [TestCase("Valana la Legende TRUEFRENCH BluRay 720p 2016 kjhlj", "Valana la Legende")] [TestCase("Mission Impossible: Rogue Nation (2015)�[XviD - Ita Ac3 - SoftSub Ita]azione, spionaggio, thriller *Prima Visione* Team mulnic Tom Cruise", "Mission Impossible Rogue Nation")] + [TestCase("Scary.Movie.2000.FRENCH..BluRay.-AiRLiNE", "Scary Movie")] + [TestCase("My Movie 1999 German Bluray", "My Movie")] public void should_parse_movie_title(string postTitle, string title) { Parser.Parser.ParseMovieTitle(postTitle, true).MovieTitle.Should().Be(title); @@ -135,9 +137,20 @@ namespace NzbDrone.Core.Test.ParserTests [TestCase("Movie IMAX 2012.mkv", "IMAX")] [TestCase("Fake Movie Final Cut 2016", "Final Cut")] [TestCase("Fake Movie 2016 Final Cut ", "Final Cut")] + [TestCase("My Movie GERMAN Extended Cut 2016", "Extended Cut")] + [TestCase("My.Movie.GERMAN.Extended.Cut.2016", "Extended Cut")] + [TestCase("My.Movie.GERMAN.Extended.Cut", "Extended Cut")] + [TestCase("Mission Impossible: Rogue Nation 2012 Bluray", "")] public void should_parse_edition(string postTitle, string edition) { - Parser.Parser.ParseMovieTitle(postTitle, false).Edition.Should().Be(edition); + Parser.Parser.ParseMovieTitle(postTitle, true).Edition.Should().Be(edition); + } + + [TestCase("The Lord of the Rings The Fellowship of the Ring (Extended Edition) 1080p BD25", "The Lord Of The Rings The Fellowship Of The Ring", "Extended Edition")] + [TestCase("The.Lord.of.the.Rings.The.Fellowship.of.the.Ring.(Extended.Edition).1080p.BD25", "The Lord Of The Rings The Fellowship Of The Ring", "Extended Edition")] + public void should_parse_edition_lenient_mapping(string postTitle, string foundTitle, string edition) + { + Parser.Parser.ParseMinimalMovieTitle(postTitle, foundTitle, 1290).Edition.Should().Be(edition); } } } diff --git a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs index 413b2c491..2b36a1bc7 100644 --- a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/MapFixture.cs @@ -21,6 +21,7 @@ namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests private Movie _movie; private ParsedMovieInfo _parsedMovieInfo; private ParsedMovieInfo _wrongYearInfo; + private ParsedMovieInfo _wrongTitleInfo; private ParsedMovieInfo _romanTitleInfo; private ParsedMovieInfo _alternativeTitleInfo; private ParsedMovieInfo _umlautInfo; @@ -71,6 +72,12 @@ namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests Year = 1900, }; + _wrongTitleInfo = new ParsedMovieInfo + { + MovieTitle = "Other Title", + Year = 2015 + }; + _alternativeTitleInfo = new ParsedMovieInfo { MovieTitle = _movie.AlternativeTitles.First(), @@ -139,7 +146,7 @@ namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests Subject.Map(_parsedMovieInfo, "", _movieSearchCriteria); - Mocker.GetMock() + Mocker.GetMock() .Verify(v => v.FindByTitle(It.IsAny()), Times.Never()); } @@ -147,7 +154,24 @@ namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests public void should_not_match_with_wrong_year() { GivenMatchByMovieTitle(); - Subject.Map(_wrongYearInfo, "", _movieSearchCriteria).Movie.Should().BeNull(); + Subject.Map(_wrongYearInfo, "", _movieSearchCriteria).MappingResultType.Should().Be(MappingResultType.WrongYear); + } + + [Test] + public void should_not_match_wrong_title() + { + GivenMatchByMovieTitle(); + Subject.Map(_wrongTitleInfo, "", _movieSearchCriteria).MappingResultType.Should().Be(MappingResultType.WrongTitle); + } + + [Test] + public void should_return_title_not_found_when_all_is_null() + { + Mocker.GetMock() + .Setup(s => s.FindByTitle(It.IsAny())) + .Returns((Movie)null); + Subject.Map(_parsedMovieInfo, "", null).MappingResultType.Should() + .Be(MappingResultType.TitleNotFound); } [Test] diff --git a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs index e205e4e61..ad345c2c9 100644 --- a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs +++ b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs @@ -9,6 +9,7 @@ using NzbDrone.Core.Configuration; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Qualities; namespace NzbDrone.Core.DecisionEngine { @@ -71,50 +72,85 @@ namespace NzbDrone.Core.DecisionEngine { var parsedMovieInfo = Parser.Parser.ParseMovieTitle(report.Title, _configService.ParsingLeniency > 0); - if (parsedMovieInfo != null && !parsedMovieInfo.MovieTitle.IsNullOrWhiteSpace()) - { - RemoteMovie remoteMovie = _parsingService.Map(parsedMovieInfo, report.ImdbId.ToString(), searchCriteria); - remoteMovie.Release = report; - - if (remoteMovie.Movie == null) - { - decision = new DownloadDecision(remoteMovie, new Rejection("Unknown movie. Movie found does not match wanted movie.")); - } - else - { - if (parsedMovieInfo.Quality.HardcodedSubs.IsNotNullOrWhiteSpace()) - { - remoteMovie.DownloadAllowed = true; - if (_configService.AllowHardcodedSubs) - { - decision = GetDecisionForReport(remoteMovie, searchCriteria); - } - else - { - var whitelisted = _configService.WhitelistedHardcodedSubs.Split(','); - _logger.Debug("Testing: {0}", whitelisted); - if (whitelisted != null && whitelisted.Any(t => (parsedMovieInfo.Quality.HardcodedSubs.ToLower().Contains(t.ToLower()) && t.IsNotNullOrWhiteSpace()))) - { - decision = GetDecisionForReport(remoteMovie, searchCriteria); - } - else - { - decision = new DownloadDecision(remoteMovie, new Rejection("Hardcoded subs found: " + parsedMovieInfo.Quality.HardcodedSubs)); - } - } - } - else - { - remoteMovie.DownloadAllowed = true; - decision = GetDecisionForReport(remoteMovie, searchCriteria); - } - - } - } - else - { - _logger.Trace("{0} could not be parsed :(.", report.Title); - } + MappingResult result = null; + + if (parsedMovieInfo == null || parsedMovieInfo.MovieTitle.IsNullOrWhiteSpace()) + { + _logger.Debug("{0} could not be parsed :(.", report.Title); + parsedMovieInfo = new ParsedMovieInfo + { + MovieTitle = report.Title, + Year = 1290, + Language = Language.Unknown, + Quality = new QualityModel(), + }; + + if (_configService.ParsingLeniency == ParsingLeniencyType.MappingLenient) + { + result = _parsingService.Map(parsedMovieInfo, report.ImdbId.ToString(), searchCriteria); + } + + if (result == null || result.MappingResultType != MappingResultType.SuccessLenientMapping) + { + result = new MappingResult {MappingResultType = MappingResultType.NotParsable}; + result.Movie = null; //To ensure we have a remote movie, else null exception on next line! + result.RemoteMovie.ParsedMovieInfo = parsedMovieInfo; + } + else + { + //Enhance Parsed Movie Info! + result.RemoteMovie.ParsedMovieInfo = Parser.Parser.ParseMinimalMovieTitle(parsedMovieInfo.MovieTitle, + result.RemoteMovie.Movie.Title, parsedMovieInfo.Year); + } + + } + else + { + result = _parsingService.Map(parsedMovieInfo, report.ImdbId.ToString(), searchCriteria); + } + + result.ReleaseName = report.Title; + var remoteMovie = result.RemoteMovie; + + remoteMovie.Release = report; + + if (result.MappingResultType != MappingResultType.Success && result.MappingResultType != MappingResultType.SuccessLenientMapping) + { + var rejection = result.ToRejection(); + remoteMovie.Movie = null; // HACK: For now! + decision = new DownloadDecision(remoteMovie, rejection); + + } + else + { + if (parsedMovieInfo.Quality.HardcodedSubs.IsNotNullOrWhiteSpace()) + { + remoteMovie.DownloadAllowed = true; + if (_configService.AllowHardcodedSubs) + { + decision = GetDecisionForReport(remoteMovie, searchCriteria); + } + else + { + var whitelisted = _configService.WhitelistedHardcodedSubs.Split(','); + _logger.Debug("Testing: {0}", whitelisted); + if (whitelisted != null && whitelisted.Any(t => (parsedMovieInfo.Quality.HardcodedSubs.ToLower().Contains(t.ToLower()) && t.IsNotNullOrWhiteSpace()))) + { + decision = GetDecisionForReport(remoteMovie, searchCriteria); + } + else + { + decision = new DownloadDecision(remoteMovie, new Rejection("Hardcoded subs found: " + parsedMovieInfo.Quality.HardcodedSubs)); + } + } + } + else + { + remoteMovie.DownloadAllowed = true; + decision = GetDecisionForReport(remoteMovie, searchCriteria); + } + + } } catch (Exception e) { diff --git a/src/NzbDrone.Core/Download/TrackedDownloads/TrackedDownloadService.cs b/src/NzbDrone.Core/Download/TrackedDownloads/TrackedDownloadService.cs index ff82b1688..0a737f1d2 100644 --- a/src/NzbDrone.Core/Download/TrackedDownloads/TrackedDownloadService.cs +++ b/src/NzbDrone.Core/Download/TrackedDownloads/TrackedDownloadService.cs @@ -65,7 +65,7 @@ namespace NzbDrone.Core.Download.TrackedDownloads if (parsedMovieInfo != null) { - trackedDownload.RemoteMovie = _parsingService.Map(parsedMovieInfo, "", null); + trackedDownload.RemoteMovie = _parsingService.Map(parsedMovieInfo, "", null).RemoteMovie; } if (historyItems.Any()) @@ -81,7 +81,7 @@ namespace NzbDrone.Core.Download.TrackedDownloads if (parsedMovieInfo != null) { - trackedDownload.RemoteMovie = _parsingService.Map(parsedMovieInfo, "", null); + trackedDownload.RemoteMovie = _parsingService.Map(parsedMovieInfo, "", null).RemoteMovie; } } } diff --git a/src/NzbDrone.Core/MetadataSource/PreDB/PreDBService.cs b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBService.cs index e32294add..76b2a77b5 100644 --- a/src/NzbDrone.Core/MetadataSource/PreDB/PreDBService.cs +++ b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBService.cs @@ -185,7 +185,7 @@ namespace NzbDrone.Core.MetadataSource.PreDB } var match = _parsingService.Map(parsed, "", new MovieSearchCriteria { Movie = movie }); - if (match != null && match.Movie != null && match.Movie.Id == movie.Id) + if (match != null && match.RemoteMovie.Movie != null && match.RemoteMovie.Movie.Id == movie.Id) { return true; } diff --git a/src/NzbDrone.Core/Parser/Parser.cs b/src/NzbDrone.Core/Parser/Parser.cs index c23b8441b..0688e59dc 100644 --- a/src/NzbDrone.Core/Parser/Parser.cs +++ b/src/NzbDrone.Core/Parser/Parser.cs @@ -25,11 +25,11 @@ namespace NzbDrone.Core.Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), //Special, Despecialized, etc. Edition Movies, e.g: Mission.Impossible.3.2011.Special.Edition //TODO: Seems to slow down parsing heavily! - new Regex(@"^(?(?![(\[]).+?)?(?:(?:[-_\W](?<![)\[!]))*(?<year>(19|20)\d{2}(?!p|i|(19|20)\d{2}|\]|\W(19|20)\d{2})))+(\W+|_|$)(?!\\)\(?(?<edition>(((Extended.|Ultimate.)?(Director.?s|Collector.?s|Theatrical|Ultimate|Final(?=(.(Cut|Edition|Version)))|Extended|Rogue|Special|Despecialized|\d{2,3}(th)?.Anniversary)(.(Cut|Edition|Version))?(.(Extended|Uncensored|Remastered|Unrated|Uncut|IMAX|Fan.?Edit))?|((Uncensored|Remastered|Unrated|Uncut|IMAX|Fan.?Edit|Edition|Restored|((2|3|4)in1))))))\)?", - RegexOptions.IgnoreCase | RegexOptions.Compiled), + /*new Regex(@"^(?<title>(?![(\[]).+?)?(?:(?:[-_\W](?<![)\[!]))*(?<year>(19|20)\d{2}(?!p|i|(19|20)\d{2}|\]|\W(19|20)\d{2})))+(\W+|_|$)(?!\\)\(?(?<edition>(((Extended.|Ultimate.)?(Director.?s|Collector.?s|Theatrical|Ultimate|Final(?=(.(Cut|Edition|Version)))|Extended|Rogue|Special|Despecialized|\d{2,3}(th)?.Anniversary)(.(Cut|Edition|Version))?(.(Extended|Uncensored|Remastered|Unrated|Uncut|IMAX|Fan.?Edit))?|((Uncensored|Remastered|Unrated|Uncut|IMAX|Fan.?Edit|Edition|Restored|((2|3|4)in1))))))\)?", + RegexOptions.IgnoreCase | RegexOptions.Compiled),*/ //Normal movie format, e.g: Mission.Impossible.3.2011 - new Regex(@"^(?<title>(?![(\[]).+?)?(?:(?:[-_\W](?<![)\[!]))*(?<year>(19|20)\d{2}(?!p|i|\d+|\]|\W\d+)))+(\W+|_|$)(?!\\)", RegexOptions.IgnoreCase | RegexOptions.Compiled), + new Regex(@"^(?<title>(?![(\[]).+?)?(?:(?:[-_\W](?<![)\[!]))*(?<year>(19|20)\d{2}(?!p|i|(19|20)\d{2}|\]|\W(19|20)\d{2})))+(\W+|_|$)(?!\\)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //PassThePopcorn Torrent names: Star.Wars[PassThePopcorn] new Regex(@"^(?<title>.+?)?(?:(?:[-_\W](?<![()\[!]))*(?<year>(\[\w *\])))+(\W+|_|$)(?!\\)", RegexOptions.IgnoreCase | RegexOptions.Compiled), @@ -51,7 +51,7 @@ namespace NzbDrone.Core.Parser private static readonly Regex[] ReportMovieTitleLenientRegexBefore = new[] { //Some german or french tracker formats - new Regex(@"^(?<title>(?![(\[]).+?)((\W|_))(?:(German|French|TrueFrench))(.+?)(?=((19|20)\d{2}|$))(?<year>(19|20)\d{2}(?!p|i|\d+|\]|\W\d+))?(\W+|_|$)(?!\\)", RegexOptions.IgnoreCase | RegexOptions.Compiled), + new Regex(@"^(?<title>(?![(\[]).+?)((\W|_))(?:(?<!(19|20)\d{2}.)(German|French|TrueFrench))(.+?)(?=((19|20)\d{2}|$))(?<year>(19|20)\d{2}(?!p|i|\d+|\]|\W\d+))?(\W+|_|$)(?!\\)", RegexOptions.IgnoreCase | RegexOptions.Compiled), }; private static readonly Regex[] ReportMovieTitleLenientRegexAfter = new Regex[] @@ -321,6 +321,10 @@ namespace NzbDrone.Core.Parser private static readonly Regex DuplicateSpacesRegex = new Regex(@"\s{2,}", RegexOptions.Compiled); private static readonly Regex RequestInfoRegex = new Regex(@"\[.+?\]", RegexOptions.Compiled); + + private static readonly Regex ReportYearRegex = new Regex(@"^.*(?<year>(19|20)\d{2}).*$", RegexOptions.Compiled); + + private static readonly Regex ReportEditionRegex = new Regex(@"(?<edition>(((Extended.|Ultimate.)?(Director.?s|Collector.?s|Theatrical|Ultimate|Final(?=(.(Cut|Edition|Version)))|Extended|Rogue|Special|Despecialized|\d{2,3}(th)?.Anniversary)(.(Cut|Edition|Version))?(.(Extended|Uncensored|Remastered|Unrated|Uncut|IMAX|Fan.?Edit))?|((Uncensored|Remastered|Unrated|Uncut|IMAX|Fan.?Edit|Edition|Restored|((2|3|4)in1))))))\)?", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly string[] Numbers = new[] { "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine" }; private static Dictionary<String, String> _umlautMappings = new Dictionary<string, string> @@ -441,6 +445,11 @@ namespace NzbDrone.Core.Parser result.Quality = QualityParser.ParseQuality(title); Logger.Debug("Quality parsed: {0}", result.Quality); + if (result.Edition.IsNullOrWhiteSpace()) + { + result.Edition = ParseEdition(languageTitle); + } + result.ReleaseGroup = ParseReleaseGroup(title); result.ImdbId = ParseImdbId(title); @@ -482,6 +491,53 @@ namespace NzbDrone.Core.Parser return realResult; } + public static ParsedMovieInfo ParseMinimalMovieTitle(string title, string foundTitle, int foundYear) + { + var result = new ParsedMovieInfo {MovieTitle = foundTitle}; + + var languageTitle = Regex.Replace(title.Replace(".", " "), foundTitle, "A Movie", RegexOptions.IgnoreCase); + + result.Language = LanguageParser.ParseLanguage(title); + Logger.Debug("Language parsed: {0}", result.Language); + + result.Quality = QualityParser.ParseQuality(title); + Logger.Debug("Quality parsed: {0}", result.Quality); + + if (result.Edition.IsNullOrWhiteSpace()) + { + result.Edition = ParseEdition(languageTitle); + } + + result.ReleaseGroup = ParseReleaseGroup(title); + + result.ImdbId = ParseImdbId(title); + + Logger.Debug("Release Group parsed: {0}", result.ReleaseGroup); + + if (foundYear > 1800) + { + result.Year = foundYear; + } + else + { + var match = ReportYearRegex.Match(title); + if (match.Success && match.Groups["year"].Value != null) + { + int year = 1290; + if (int.TryParse(match.Groups["year"].Value, out year)) + { + result.Year = year; + } + else + { + result.Year = year; + } + } + } + + return result; + } + public static string ParseImdbId(string title) { var match = ReportImdbId.Match(title); @@ -499,6 +555,19 @@ namespace NzbDrone.Core.Parser return ""; } + public static string ParseEdition(string languageTitle) + { + var editionMatch = ReportEditionRegex.Match(languageTitle); + + if (editionMatch.Success && editionMatch.Groups["edition"].Value != null && + editionMatch.Groups["edition"].Value.IsNotNullOrWhiteSpace()) + { + return editionMatch.Groups["edition"].Value.Replace(".", " "); + } + + return ""; + } + public static ParsedEpisodeInfo ParseTitle(string title) { diff --git a/src/NzbDrone.Core/Parser/ParsingService.cs b/src/NzbDrone.Core/Parser/ParsingService.cs index 1f27d6a0d..72d7e4d57 100644 --- a/src/NzbDrone.Core/Parser/ParsingService.cs +++ b/src/NzbDrone.Core/Parser/ParsingService.cs @@ -6,6 +6,7 @@ using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Core.Configuration; using NzbDrone.Core.DataAugmentation.Scene; +using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Parser.Model; @@ -24,7 +25,7 @@ namespace NzbDrone.Core.Parser Movie GetMovie(string title); RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, int tvdbId, int tvRageId, SearchCriteriaBase searchCriteria = null); RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, int seriesId, IEnumerable<int> episodeIds); - RemoteMovie Map(ParsedMovieInfo parsedMovieInfo, string imdbId, SearchCriteriaBase searchCriteria = null); + MappingResult Map(ParsedMovieInfo parsedMovieInfo, string imdbId, SearchCriteriaBase searchCriteria = null); List<Episode> GetEpisodes(ParsedEpisodeInfo parsedEpisodeInfo, Series series, bool sceneSource, SearchCriteriaBase searchCriteria = null); ParsedEpisodeInfo ParseSpecialEpisodeTitle(string title, int tvdbId, int tvRageId, SearchCriteriaBase searchCriteria = null); } @@ -218,23 +219,18 @@ namespace NzbDrone.Core.Parser return remoteEpisode; } - public RemoteMovie Map(ParsedMovieInfo parsedMovieInfo, string imdbId, SearchCriteriaBase searchCriteria = null) + public MappingResult Map(ParsedMovieInfo parsedMovieInfo, string imdbId, SearchCriteriaBase searchCriteria = null) { - var remoteMovie = new RemoteMovie - { - ParsedMovieInfo = parsedMovieInfo, - }; + var result = GetMovie(parsedMovieInfo, imdbId, searchCriteria); - var movie = GetMovie(parsedMovieInfo, imdbId, searchCriteria); - - if (movie == null) - { - return remoteMovie; + if (result == null) { + result = new MappingResult {MappingResultType = MappingResultType.Unknown}; + result.Movie = null; } - remoteMovie.Movie = movie; + result.RemoteMovie.ParsedMovieInfo = parsedMovieInfo; - return remoteMovie; + return result; } public RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, int seriesId, IEnumerable<int> episodeIds) @@ -351,77 +347,113 @@ namespace NzbDrone.Core.Parser return null; } - private Movie GetMovie(ParsedMovieInfo parsedMovieInfo, string imdbId, SearchCriteriaBase searchCriteria) + private MappingResult GetMovie(ParsedMovieInfo parsedMovieInfo, string imdbId, SearchCriteriaBase searchCriteria) { // TODO: Answer me this: Wouldn't it be smarter to start out looking for a movie if we have an ImDb Id? + MappingResult result = null; if (!String.IsNullOrWhiteSpace(imdbId) && imdbId != "0") { - Movie movieByImDb; - if (TryGetMovieByImDbId(parsedMovieInfo, imdbId, out movieByImDb)) + if (TryGetMovieByImDbId(parsedMovieInfo, imdbId, out result)) { - return movieByImDb; + return result; } } if (searchCriteria != null) { - Movie movieBySearchCriteria; - if (TryGetMovieBySearchCriteria(parsedMovieInfo, searchCriteria, out movieBySearchCriteria)) + if (TryGetMovieBySearchCriteria(parsedMovieInfo, searchCriteria, out result)) { - return movieBySearchCriteria; + return result; } } else { - Movie movieByTitleAndOrYear; - if (TryGetMovieByTitleAndOrYear(parsedMovieInfo, out movieByTitleAndOrYear)) - { - return movieByTitleAndOrYear; - } + TryGetMovieByTitleAndOrYear(parsedMovieInfo, out result); + return result; } // nothing found up to here => logging that and returning null _logger.Debug($"No matching movie {parsedMovieInfo.MovieTitle}"); - return null; + return result; } - private bool TryGetMovieByImDbId(ParsedMovieInfo parsedMovieInfo, string imdbId, out Movie movie) + private bool TryGetMovieByImDbId(ParsedMovieInfo parsedMovieInfo, string imdbId, out MappingResult result) { - movie = _movieService.FindByImdbId(imdbId); + var movie = _movieService.FindByImdbId(imdbId); //Should fix practically all problems, where indexer is shite at adding correct imdbids to movies. if (movie != null && parsedMovieInfo.Year > 1800 && parsedMovieInfo.Year != movie.Year) { - movie = null; + result = new MappingResult { Movie = movie, MappingResultType = MappingResultType.WrongYear}; return false; } + if (movie != null) { + result = new MappingResult { Movie = movie }; + } else { + result = new MappingResult { Movie = movie, MappingResultType = MappingResultType.TitleNotFound}; + } return movie != null; } - private bool TryGetMovieByTitleAndOrYear(ParsedMovieInfo parsedMovieInfo, out Movie movieByTitleAndOrYear) + private bool TryGetMovieByTitleAndOrYear(ParsedMovieInfo parsedMovieInfo, out MappingResult result) { Func<Movie, bool> isNotNull = movie => movie != null; + Movie movieByTitleAndOrYear = null; + if (parsedMovieInfo.Year > 1800) { movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year); if (isNotNull(movieByTitleAndOrYear)) { + result = new MappingResult { Movie = movieByTitleAndOrYear }; return true; } + movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle); + if (isNotNull(movieByTitleAndOrYear)) + { + result = new MappingResult { Movie = movieByTitleAndOrYear, MappingResultType = MappingResultType.WrongYear}; + return false; + } + + if (_config.ParsingLeniency == ParsingLeniencyType.MappingLenient) + { + movieByTitleAndOrYear = _movieService.FindByTitleInexact(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year); + if (isNotNull(movieByTitleAndOrYear)) + { + result = new MappingResult {Movie = movieByTitleAndOrYear, MappingResultType = MappingResultType.SuccessLenientMapping}; + return true; + } + } + + result = new MappingResult { Movie = movieByTitleAndOrYear, MappingResultType = MappingResultType.TitleNotFound}; return false; } + movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle); if (isNotNull(movieByTitleAndOrYear)) { + result = new MappingResult { Movie = movieByTitleAndOrYear }; return true; } - movieByTitleAndOrYear = null; + + if (_config.ParsingLeniency == ParsingLeniencyType.MappingLenient) + { + movieByTitleAndOrYear = _movieService.FindByTitleInexact(parsedMovieInfo.MovieTitle, null); + if (isNotNull(movieByTitleAndOrYear)) + { + result = new MappingResult {Movie = movieByTitleAndOrYear, MappingResultType = MappingResultType.SuccessLenientMapping}; + return true; + } + } + + result = new MappingResult { Movie = movieByTitleAndOrYear, MappingResultType = MappingResultType.TitleNotFound}; return false; } - private bool TryGetMovieBySearchCriteria(ParsedMovieInfo parsedMovieInfo, SearchCriteriaBase searchCriteria, out Movie possibleMovie) + private bool TryGetMovieBySearchCriteria(ParsedMovieInfo parsedMovieInfo, SearchCriteriaBase searchCriteria, out MappingResult result) { - possibleMovie = null; + Movie possibleMovie = null; + List<string> possibleTitles = new List<string>(); possibleTitles.Add(searchCriteria.Movie.CleanTitle); @@ -445,7 +477,7 @@ namespace NzbDrone.Core.Parser string arabicNumeral = numeralMapping.ArabicNumeralAsString; string romanNumeral = numeralMapping.RomanNumeralLowerCase; - _logger.Debug(cleanTitle); + //_logger.Debug(cleanTitle); if (title.Replace(arabicNumeral, romanNumeral) == parsedMovieInfo.MovieTitle.CleanSeriesTitle()) { @@ -460,11 +492,42 @@ namespace NzbDrone.Core.Parser } } - if (possibleMovie != null && (parsedMovieInfo.Year < 1800 || possibleMovie.Year == parsedMovieInfo.Year)) + if (possibleMovie != null) { - return true; + if (parsedMovieInfo.Year < 1800 || possibleMovie.Year == parsedMovieInfo.Year) + { + result = new MappingResult { Movie = possibleMovie, MappingResultType = MappingResultType.Success }; + return true; + } + result = new MappingResult { Movie = possibleMovie, MappingResultType = MappingResultType.WrongYear }; + return false; } - possibleMovie = null; + + if (_config.ParsingLeniency == ParsingLeniencyType.MappingLenient) + { + if (searchCriteria.Movie.CleanTitle.Contains(cleanTitle) || + cleanTitle.Contains(searchCriteria.Movie.CleanTitle)) + { + possibleMovie = searchCriteria.Movie; + if (parsedMovieInfo.Year > 1800 && parsedMovieInfo.Year == possibleMovie.Year) + { + result = new MappingResult {Movie = possibleMovie, MappingResultType = MappingResultType.SuccessLenientMapping}; + return true; + } + + if (parsedMovieInfo.Year < 1800) + { + result = new MappingResult { Movie = possibleMovie, MappingResultType = MappingResultType.SuccessLenientMapping }; + return true; + } + + result = new MappingResult { Movie = possibleMovie, MappingResultType = MappingResultType.WrongYear }; + return false; + } + } + + result = new MappingResult { Movie = searchCriteria.Movie, MappingResultType = MappingResultType.WrongTitle }; + return false; } @@ -693,29 +756,86 @@ namespace NzbDrone.Core.Parser } } - public class MappingException : Exception - { - public virtual string Reason() - { - return "Parsed movie does not match wanted movie"; - } - } - public class YearDoesNotMatchException : MappingException + public class MappingResult { - public int ExpectedYear { get; set; } - public int? ParsedYear { get; set; } - - override public string Reason() + public string Message { - if (ParsedYear.HasValue && ParsedYear > 1800) + get + { + switch (MappingResultType) + { + case MappingResultType.Success: + return $"Successfully mapped release name {ReleaseName} to movie {Movie}"; + break; + case MappingResultType.SuccessLenientMapping: + return $"Successfully mapped parts of the release name {ReleaseName} to movie {Movie}"; + break; + case MappingResultType.NotParsable: + return $"Failed to find movie title in release name {ReleaseName}"; + break; + case MappingResultType.TitleNotFound: + return $"Could not find {RemoteMovie.ParsedMovieInfo.MovieTitle}"; + break; + case MappingResultType.WrongYear: + return $"Failed to map movie, expected year {RemoteMovie.Movie.Year}, but found {RemoteMovie.ParsedMovieInfo.Year}"; + case MappingResultType.WrongTitle: + var comma = RemoteMovie.Movie.AlternativeTitles.Count > 0 ? ", " : ""; + return + $"Failed to map movie, found title {RemoteMovie.ParsedMovieInfo.MovieTitle}, expected one of: {RemoteMovie.Movie.Title}{comma}{string.Join(", ", RemoteMovie.Movie.AlternativeTitles)}"; + default: + return $"Failed to map movie for unkown reasons"; + } + } + } + + public RemoteMovie RemoteMovie; + public MappingResultType MappingResultType { get; set; } + public Movie Movie { + get { + return RemoteMovie.Movie; + } + set { - return $"Expected {ExpectedYear}, but found {ParsedYear} for year"; + ParsedMovieInfo parsedInfo = null; + if (RemoteMovie != null) + { + parsedInfo = RemoteMovie.ParsedMovieInfo; + } + RemoteMovie = new RemoteMovie + { + Movie = value, + ParsedMovieInfo = parsedInfo + }; } - else + } + + public string ReleaseName { get; set; } + + public override string ToString() { + return string.Format(Message, RemoteMovie.Movie); + } + + public Rejection ToRejection() { + switch (MappingResultType) { - return "Did not find a valid year"; + case MappingResultType.Success: + case MappingResultType.SuccessLenientMapping: + return null; + default: + return new Rejection(Message); } } } + + public enum MappingResultType + { + Unknown = -1, + Success = 0, + SuccessLenientMapping = 1, + WrongYear = 2, + WrongTitle = 3, + TitleNotFound = 4, + NotParsable = 5, + } } \ No newline at end of file diff --git a/src/NzbDrone.Core/Tv/MovieService.cs b/src/NzbDrone.Core/Tv/MovieService.cs index 1d005bced..36dcbaf73 100644 --- a/src/NzbDrone.Core/Tv/MovieService.cs +++ b/src/NzbDrone.Core/Tv/MovieService.cs @@ -28,7 +28,7 @@ namespace NzbDrone.Core.Tv Movie FindByImdbId(string imdbid); Movie FindByTitle(string title); Movie FindByTitle(string title, int year); - Movie FindByTitleInexact(string title); + Movie FindByTitleInexact(string title, int? year); Movie FindByTitleSlug(string slug); bool MovieExists(Movie movie); Movie GetMovieByFileId(int fileId); @@ -238,20 +238,16 @@ namespace NzbDrone.Core.Tv return _movieRepository.FindByImdbId(imdbid); } - public Movie FindByTitleInexact(string title) + private List<Movie> FindByTitleInexactAll(string title) { // find any movie clean title within the provided release title string cleanTitle = title.CleanSeriesTitle(); - var list = _movieRepository.All().Where(s => cleanTitle.Contains(s.CleanTitle)).ToList(); + var list = _movieRepository.All().Where(s => cleanTitle.Contains(s.CleanTitle)) + .Union(_movieRepository.All().Where(s => s.CleanTitle.Contains(cleanTitle))).ToList(); if (!list.Any()) { // no movie matched - return null; - } - if (list.Count == 1) - { - // return the first movie if there is only one - return list.Single(); + return list; } // build ordered list of movie by position in the search string var query = @@ -265,21 +261,34 @@ namespace NzbDrone.Core.Tv .ToList() .OrderBy(s => s.position) .ThenByDescending(s => s.length) + .Select(s => s.movie) .ToList(); + + + return query; + } + + public Movie FindByTitleInexact(string title) + { + var query = FindByTitleInexactAll(title); // get the leftmost movie that is the longest // movie are usually the first thing in release title, so we select the leftmost and longest match - var match = query.First().movie; + var match = query.First(); _logger.Debug("Multiple movie matched {0} from title {1}", match.Title, title); - foreach (var entry in list) + foreach (var entry in query) { _logger.Debug("Multiple movie match candidate: {0} cleantitle: {1}", entry.Title, entry.CleanTitle); } - return match; } + public Movie FindByTitleInexact(string title, int? year) + { + return FindByTitleInexactAll(title).FirstWithYear(year); + } + public Movie FindByTitle(string title, int year) { return _movieRepository.FindByTitle(title.CleanSeriesTitle(), year); diff --git a/src/UI/Settings/Indexers/Options/LeniencyTooltipTemplate.hbs b/src/UI/Settings/Indexers/Options/LeniencyTooltipTemplate.hbs index 5c4b7ffcd..abbe1b794 100644 --- a/src/UI/Settings/Indexers/Options/LeniencyTooltipTemplate.hbs +++ b/src/UI/Settings/Indexers/Options/LeniencyTooltipTemplate.hbs @@ -1,7 +1,11 @@ -How strict the Parser should be. (Note: Strict is strongly recommended!) -<br><br> +<h5><b>How strict the Parser should be. (Note: Strict is strongly recommended!)</b></h5> +<br> <b>Strict:</b> Just as before, year must immediately follow title. <br><br> -<b>Lenient Parsing:</b> Either year or language tag must immediately follow after title. (Note: May prevent Movies with language tags in title - e.g. The Danish Girl - from being parsed correctly) +<b>Lenient Parsing:</b> Either year or language tag must immediately follow after title. Enables releases such as 'Scary Movie German BluRay' to be parsed correctly. +<br> +<b>Note</b>: May prevent Movies with language tags in title - e.g. The Danish Girl - from being parsed correctly <br><br> -<b>Lenient Mapping:</b> Includes Lenient Parsing. When title cannot be found Try mapping just parts of the title. (Useful when no year is present / not after title. <b>NOT IMPLEMENTED YET</b>) \ No newline at end of file +<b>Lenient Mapping:</b> Includes Lenient Parsing. When title cannot be found, try mapping just parts of the title. Useful when no year is present / not after title. +<br> +<b>Warning!:</b> May cause unexpected mappings, e.g. Scary Movie 2 mapped to movie Scary Movie 1, etc. Use with caution. \ No newline at end of file