From 83e3d7145f706fc1321e44d66470d5f13f7d446d Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Mon, 15 Dec 2014 10:52:16 -0800 Subject: [PATCH] Better parsing of Chistmas Specials --- .../AcceptableSizeSpecificationFixture.cs | 8 + .../NzbDrone.Core.Test.csproj | 4 +- ...deProviderTest_GetEpisodesByParseResult.cs | 277 ------------------ .../FindEpisodeByTitleFixture.cs | 45 +++ .../HandleEpisodeFileDeletedFixture.cs | 2 +- .../AcceptableSizeSpecification.cs | 6 + src/NzbDrone.Core/Parser/Parser.cs | 170 +++++------ src/NzbDrone.Core/Parser/ParsingService.cs | 4 +- src/NzbDrone.Core/Tv/EpisodeService.cs | 6 +- 9 files changed, 153 insertions(+), 369 deletions(-) delete mode 100644 src/NzbDrone.Core.Test/TvTests/EpisodeProviderTests/EpisodeProviderTest_GetEpisodesByParseResult.cs create mode 100644 src/NzbDrone.Core.Test/TvTests/EpisodeServiceTests/FindEpisodeByTitleFixture.cs rename src/NzbDrone.Core.Test/TvTests/{EpisodeProviderTests => EpisodeServiceTests}/HandleEpisodeFileDeletedFixture.cs (98%) diff --git a/src/NzbDrone.Core.Test/DecisionEngineTests/AcceptableSizeSpecificationFixture.cs b/src/NzbDrone.Core.Test/DecisionEngineTests/AcceptableSizeSpecificationFixture.cs index ff1e9c5e9..493ffa70c 100644 --- a/src/NzbDrone.Core.Test/DecisionEngineTests/AcceptableSizeSpecificationFixture.cs +++ b/src/NzbDrone.Core.Test/DecisionEngineTests/AcceptableSizeSpecificationFixture.cs @@ -187,5 +187,13 @@ namespace NzbDrone.Core.Test.DecisionEngineTests Subject.IsSatisfiedBy(parseResult, null).Accepted.Should().BeTrue(); } + + [Test] + public void should_return_true_for_special() + { + parseResultSingle.ParsedEpisodeInfo.Special = true; + + Subject.IsSatisfiedBy(parseResultSingle, null).Accepted.Should().BeTrue(); + } } } diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index b85256ae3..ad9006b4c 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -296,8 +296,8 @@ - - + + diff --git a/src/NzbDrone.Core.Test/TvTests/EpisodeProviderTests/EpisodeProviderTest_GetEpisodesByParseResult.cs b/src/NzbDrone.Core.Test/TvTests/EpisodeProviderTests/EpisodeProviderTest_GetEpisodesByParseResult.cs deleted file mode 100644 index 6af23ea4e..000000000 --- a/src/NzbDrone.Core.Test/TvTests/EpisodeProviderTests/EpisodeProviderTest_GetEpisodesByParseResult.cs +++ /dev/null @@ -1,277 +0,0 @@ -/* - - -using System; -using System.Collections.Generic; -using System.Linq; -using FizzWare.NBuilder; -using FluentAssertions; -using NUnit.Framework; -using NzbDrone.Core.Tv; -using NzbDrone.Core.Model; -using NzbDrone.Core.Test.Framework; -using NzbDrone.Test.Common; - -namespace NzbDrone.Core.Test.TvTests.EpisodeProviderTests -{ - [TestFixture] - - public class EpisodeProviderTest_GetEpisodesByParseResult : ObjectDbTest - { - private IEpisodeService episodeService; - - private Series fakeSeries; - private Series fakeDailySeries; - - private Episode fakeEpisode; - private Episode fakeDailyEpisode; - private Episode fakeEpisode2; - - [SetUp] - public void Setup() - { - fakeSeries = Builder.CreateNew().Build(); - - fakeDailySeries = Builder.CreateNew() - .With(c => c.SeriesType = SeriesType.Daily) - .Build(); - - fakeEpisode = Builder.CreateNew() - .With(e => e.SeriesId = fakeSeries.Id) - .With(e => e.Title = "Episode (1)") - .Build(); - - fakeEpisode2 = Builder.CreateNew() - .With(e => e.SeriesId = fakeSeries.Id) - .With(e => e.SeasonNumber = fakeEpisode.SeasonNumber) - .With(e => e.EpisodeNumber = fakeEpisode.EpisodeNumber + 1) - .With(e => e.Title = "Episode (2)") - .Build(); - - fakeDailyEpisode = Builder.CreateNew() - .With(e => e.SeriesId = fakeSeries.Id) - .With(e => e.AirDate = DateTime.Now.Date) - .With(e => e.Title = "Daily Episode 1") - .Build(); - - - - episodeService = Mocker.Resolve(); - } - - [Test] - public void existing_single_episode_should_return_single_existing_episode() - { - Db.Insert(fakeEpisode); - Db.Insert(fakeSeries); - - var parseResult = new EpisodeParseResult - { - Series = fakeSeries, - SeasonNumber = fakeEpisode.SeasonNumber, - EpisodeNumbers = new List { fakeEpisode.EpisodeNumber } - }; - - var ep = episodeService.GetEpisodesByParseResult(parseResult); - - ep.Should().HaveCount(1); - parseResult.EpisodeTitle.Should().Be(fakeEpisode.Title); - VerifyEpisode(ep[0], fakeEpisode); - Db().Should().HaveCount(1); - } - - [Test] - public void single_none_existing_episode_should_return_nothing_and_add_nothing() - { - var parseResult = new EpisodeParseResult - { - Series = fakeSeries, - SeasonNumber = fakeEpisode.SeasonNumber, - EpisodeNumbers = new List { 10 } - }; - - var episode = episodeService.GetEpisodesByParseResult(parseResult); - - episode.Should().BeEmpty(); - Db.Fetch().Should().HaveCount(0); - } - - [Test] - public void single_none_existing_series_should_return_nothing_and_add_nothing() - { - var parseResult = new EpisodeParseResult - { - Series = fakeSeries, - SeasonNumber = 10, - EpisodeNumbers = new List { 10 } - }; - - var episode = episodeService.GetEpisodesByParseResult(parseResult); - - episode.Should().BeEmpty(); - Db.Fetch().Should().HaveCount(0); - } - - [Test] - public void existing_multi_episode_should_return_all_episodes() - { - Db.Insert(fakeSeries); - Db.Insert(fakeEpisode); - Db.Insert(fakeEpisode2); - - - var parseResult = new EpisodeParseResult - { - Series = fakeSeries, - SeasonNumber = fakeEpisode.SeasonNumber, - EpisodeNumbers = new List { fakeEpisode.EpisodeNumber, fakeEpisode2.EpisodeNumber } - }; - - var ep = episodeService.GetEpisodesByParseResult(parseResult); - - ep.Should().HaveCount(2); - Db.Fetch().Should().HaveCount(2); - - VerifyEpisode(ep[0], fakeEpisode); - VerifyEpisode(ep[1], fakeEpisode2); - parseResult.EpisodeTitle.Should().Be("Episode"); - } - - - - - [Test] - public void none_existing_multi_episode_should_not_return_or_add_anything() - { - var parseResult = new EpisodeParseResult - { - Series = fakeSeries, - SeasonNumber = fakeEpisode.SeasonNumber, - EpisodeNumbers = new List { fakeEpisode.EpisodeNumber, fakeEpisode2.EpisodeNumber } - }; - - var ep = episodeService.GetEpisodesByParseResult(parseResult); - - ep.Should().BeEmpty(); - Db.Fetch().Should().BeEmpty(); - } - - - [Test] - public void GetEpisodeParseResult_should_return_empty_list_if_episode_list_is_null() - { - - var episodes = episodeService.GetEpisodesByParseResult(new EpisodeParseResult()); - - episodes.Should().NotBeNull(); - episodes.Should().BeEmpty(); - } - - [Test] - public void GetEpisodeParseResult_should_return_empty_list_if_episode_list_is_empty() - { - - var episodes = episodeService.GetEpisodesByParseResult(new EpisodeParseResult { EpisodeNumbers = new List() }); - - episodes.Should().NotBeNull(); - episodes.Should().BeEmpty(); - } - - [Test] - public void should_return_single_episode_when_air_date_is_provided() - { - - Db.Insert(fakeSeries); - Db.Insert(fakeDailyEpisode); - - - var episodes = episodeService.GetEpisodesByParseResult(new EpisodeParseResult { AirDate = DateTime.Today, Series = fakeDailySeries }); - - - episodes.Should().HaveCount(1); - VerifyEpisode(episodes[0], fakeDailyEpisode); - - Db.Fetch().Should().HaveCount(1); - } - - [Test] - public void should_not_add_episode_when_episode_doesnt_exist() - { - var episodes = episodeService.GetEpisodesByParseResult(new EpisodeParseResult { AirDate = DateTime.Today, Series = fakeDailySeries }); - - - episodes.Should().HaveCount(0); - Db.Fetch().Should().HaveCount(0); - } - - - [Test] - public void GetEpisodeParseResult_should_return_single_title_for_multiple_episodes() - { - Db.Insert(fakeSeries); - Db.Insert(fakeEpisode); - Db.Insert(fakeEpisode2); - - var parseResult = new EpisodeParseResult - { - Series = fakeSeries, - SeasonNumber = fakeEpisode.SeasonNumber, - EpisodeNumbers = new List { fakeEpisode.EpisodeNumber, fakeEpisode2.EpisodeNumber } - }; - - var ep = episodeService.GetEpisodesByParseResult(parseResult); - - ep.Should().HaveCount(2); - Db.Fetch().Should().HaveCount(2); - - VerifyEpisode(ep[0], fakeEpisode); - VerifyEpisode(ep[1], fakeEpisode2); - - parseResult.EpisodeTitle.Should().Be("Episode"); - } - - [Test] - public void GetEpisodeParseResult_should_return_single_title_for_single_episode() - { - Db.Insert(fakeEpisode); - Db.Insert(fakeSeries); - - var parseResult = new EpisodeParseResult - { - Series = fakeSeries, - SeasonNumber = fakeEpisode.SeasonNumber, - EpisodeNumbers = new List { fakeEpisode.EpisodeNumber } - }; - - var ep = episodeService.GetEpisodesByParseResult(parseResult); - - ep.Should().HaveCount(1); - Db.Fetch().Should().HaveCount(1); - ep.First().ShouldHave().AllPropertiesBut(e => e.Series); - parseResult.EpisodeTitle.Should().Be(fakeEpisode.Title); - } - - [Test] - public void GetEpisodeParseResult_should_return_nothing_when_series_is_not_daily_but_parsed_daily() - { - Db.Insert(fakeSeries); - - var parseResult = new EpisodeParseResult - { - Series = fakeSeries, - AirDate = DateTime.Today - }; - - var ep = episodeService.GetEpisodesByParseResult(parseResult); - - ep.Should().BeEmpty(); - ExceptionVerification.ExpectedWarns(1); - } - - private void VerifyEpisode(Episode actual, Episode excpected) - { - actual.ShouldHave().AllProperties().But(e => e.Series).But(e => e.EpisodeFile).EqualTo(excpected); - } - } -} -*/ diff --git a/src/NzbDrone.Core.Test/TvTests/EpisodeServiceTests/FindEpisodeByTitleFixture.cs b/src/NzbDrone.Core.Test/TvTests/EpisodeServiceTests/FindEpisodeByTitleFixture.cs new file mode 100644 index 000000000..b27b45e32 --- /dev/null +++ b/src/NzbDrone.Core.Test/TvTests/EpisodeServiceTests/FindEpisodeByTitleFixture.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using FizzWare.NBuilder; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Test.TvTests.EpisodeServiceTests +{ + [TestFixture] + public class FindEpisodeByTitleFixture : CoreTest + { + private Episode _episode; + + [SetUp] + public void Setup() + { + _episode = Builder.CreateNew().Build(); + } + + private void GivenEpisodeTitle(string title) + { + _episode.Title = title; + + Mocker.GetMock() + .Setup(s => s.GetEpisodes(It.IsAny(), It.IsAny())) + .Returns(new List { _episode }); + } + + [Test] + public void should_find_episode_by_title() + { + GivenEpisodeTitle("A Journey to the Highlands"); + + Subject.FindEpisodeByTitle(1, 1, "Downton.Abbey.A.Journey.To.The.Highlands.720p.BluRay.x264-aAF") + .Title + .Should() + .Be(_episode.Title); + } + } +} diff --git a/src/NzbDrone.Core.Test/TvTests/EpisodeProviderTests/HandleEpisodeFileDeletedFixture.cs b/src/NzbDrone.Core.Test/TvTests/EpisodeServiceTests/HandleEpisodeFileDeletedFixture.cs similarity index 98% rename from src/NzbDrone.Core.Test/TvTests/EpisodeProviderTests/HandleEpisodeFileDeletedFixture.cs rename to src/NzbDrone.Core.Test/TvTests/EpisodeServiceTests/HandleEpisodeFileDeletedFixture.cs index 21503277e..96b5002ff 100644 --- a/src/NzbDrone.Core.Test/TvTests/EpisodeProviderTests/HandleEpisodeFileDeletedFixture.cs +++ b/src/NzbDrone.Core.Test/TvTests/EpisodeServiceTests/HandleEpisodeFileDeletedFixture.cs @@ -9,7 +9,7 @@ using NzbDrone.Core.MediaFiles.Events; using NzbDrone.Core.Test.Framework; using NzbDrone.Core.Tv; -namespace NzbDrone.Core.Test.TvTests.EpisodeProviderTests +namespace NzbDrone.Core.Test.TvTests.EpisodeServiceTests { [TestFixture] public class HandleEpisodeFileDeletedFixture : CoreTest diff --git a/src/NzbDrone.Core/DecisionEngine/Specifications/AcceptableSizeSpecification.cs b/src/NzbDrone.Core/DecisionEngine/Specifications/AcceptableSizeSpecification.cs index cba9bdf4d..28f1ca697 100644 --- a/src/NzbDrone.Core/DecisionEngine/Specifications/AcceptableSizeSpecification.cs +++ b/src/NzbDrone.Core/DecisionEngine/Specifications/AcceptableSizeSpecification.cs @@ -36,6 +36,12 @@ namespace NzbDrone.Core.DecisionEngine.Specifications return Decision.Accept(); } + if (subject.ParsedEpisodeInfo.Special) + { + _logger.Debug("Special release found, skipping size check."); + return Decision.Accept(); + } + var qualityDefinition = _qualityDefinitionService.Get(quality); var minSize = qualityDefinition.MinSize.Megabytes(); diff --git a/src/NzbDrone.Core/Parser/Parser.cs b/src/NzbDrone.Core/Parser/Parser.cs index 0f1fc16cd..f599df220 100644 --- a/src/NzbDrone.Core/Parser/Parser.cs +++ b/src/NzbDrone.Core/Parser/Parser.cs @@ -173,7 +173,7 @@ namespace NzbDrone.Core.Parser private static readonly Regex WordDelimiterRegex = new Regex(@"(\s|\.|,|_|-|=|\|)+", RegexOptions.Compiled); private static readonly Regex PunctuationRegex = new Regex(@"[^\w\s]", RegexOptions.Compiled); private static readonly Regex CommonWordRegex = new Regex(@"\b(a|an|the|and|or|of)\b\s?", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex SpecialEpisodeWordRegex = new Regex(@"\b(part|special|edition)\b\s?", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex SpecialEpisodeWordRegex = new Regex(@"\b(part|special|edition|christmas)\b\s?", RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex DuplicateSpacesRegex = new Regex(@"\s{2,}", RegexOptions.Compiled); private static readonly Regex RequestInfoRegex = new Regex(@"\[.+?\]", RegexOptions.Compiled); @@ -393,6 +393,90 @@ namespace NzbDrone.Core.Parser return title; } + public static Language ParseLanguage(string title) + { + var lowerTitle = title.ToLower(); + + if (lowerTitle.Contains("english")) + return Language.English; + + if (lowerTitle.Contains("french")) + return Language.French; + + if (lowerTitle.Contains("spanish")) + return Language.Spanish; + + if (lowerTitle.Contains("danish")) + return Language.Danish; + + if (lowerTitle.Contains("dutch")) + return Language.Dutch; + + if (lowerTitle.Contains("japanese")) + return Language.Japanese; + + if (lowerTitle.Contains("cantonese")) + return Language.Cantonese; + + if (lowerTitle.Contains("mandarin")) + return Language.Mandarin; + + if (lowerTitle.Contains("korean")) + return Language.Korean; + + if (lowerTitle.Contains("russian")) + return Language.Russian; + + if (lowerTitle.Contains("polish")) + return Language.Polish; + + if (lowerTitle.Contains("vietnamese")) + return Language.Vietnamese; + + if (lowerTitle.Contains("swedish")) + return Language.Swedish; + + if (lowerTitle.Contains("norwegian")) + return Language.Norwegian; + + if (lowerTitle.Contains("nordic")) + return Language.Norwegian; + + if (lowerTitle.Contains("finnish")) + return Language.Finnish; + + if (lowerTitle.Contains("turkish")) + return Language.Turkish; + + if (lowerTitle.Contains("portuguese")) + return Language.Portuguese; + + var match = LanguageRegex.Match(title); + + if (match.Groups["italian"].Captures.Cast().Any()) + return Language.Italian; + + if (match.Groups["german"].Captures.Cast().Any()) + return Language.German; + + if (match.Groups["flemish"].Captures.Cast().Any()) + return Language.Flemish; + + if (match.Groups["greek"].Captures.Cast().Any()) + return Language.Greek; + + if (match.Groups["french"].Success) + return Language.French; + + if (match.Groups["russian"].Success) + return Language.Russian; + + if (match.Groups["dutch"].Success) + return Language.Dutch; + + return Language.English; + } + private static SeriesTitleInfo GetSeriesTitleInfo(string title) { var seriesTitleInfo = new SeriesTitleInfo(); @@ -539,90 +623,6 @@ namespace NzbDrone.Core.Parser return result; } - private static Language ParseLanguage(string title) - { - var lowerTitle = title.ToLower(); - - if (lowerTitle.Contains("english")) - return Language.English; - - if (lowerTitle.Contains("french")) - return Language.French; - - if (lowerTitle.Contains("spanish")) - return Language.Spanish; - - if (lowerTitle.Contains("danish")) - return Language.Danish; - - if (lowerTitle.Contains("dutch")) - return Language.Dutch; - - if (lowerTitle.Contains("japanese")) - return Language.Japanese; - - if (lowerTitle.Contains("cantonese")) - return Language.Cantonese; - - if (lowerTitle.Contains("mandarin")) - return Language.Mandarin; - - if (lowerTitle.Contains("korean")) - return Language.Korean; - - if (lowerTitle.Contains("russian")) - return Language.Russian; - - if (lowerTitle.Contains("polish")) - return Language.Polish; - - if (lowerTitle.Contains("vietnamese")) - return Language.Vietnamese; - - if (lowerTitle.Contains("swedish")) - return Language.Swedish; - - if (lowerTitle.Contains("norwegian")) - return Language.Norwegian; - - if (lowerTitle.Contains("nordic")) - return Language.Norwegian; - - if (lowerTitle.Contains("finnish")) - return Language.Finnish; - - if (lowerTitle.Contains("turkish")) - return Language.Turkish; - - if (lowerTitle.Contains("portuguese")) - return Language.Portuguese; - - var match = LanguageRegex.Match(title); - - if (match.Groups["italian"].Captures.Cast().Any()) - return Language.Italian; - - if (match.Groups["german"].Captures.Cast().Any()) - return Language.German; - - if (match.Groups["flemish"].Captures.Cast().Any()) - return Language.Flemish; - - if (match.Groups["greek"].Captures.Cast().Any()) - return Language.Greek; - - if (match.Groups["french"].Success) - return Language.French; - - if (match.Groups["russian"].Success) - return Language.Russian; - - if (match.Groups["dutch"].Success) - return Language.Dutch; - - return Language.English; - } - private static bool ValidateBeforeParsing(string title) { if (title.ToLower().Contains("password") && title.ToLower().Contains("yenc")) diff --git a/src/NzbDrone.Core/Parser/ParsingService.cs b/src/NzbDrone.Core/Parser/ParsingService.cs index cf4ba1941..1726e0395 100644 --- a/src/NzbDrone.Core/Parser/ParsingService.cs +++ b/src/NzbDrone.Core/Parser/ParsingService.cs @@ -312,7 +312,7 @@ namespace NzbDrone.Core.Parser private ParsedEpisodeInfo ParseSpecialEpisodeTitle(string title, Series series) { // find special episode in series season 0 - var episode = _episodeService.FindEpisodeByName(series.Id, 0, title); + var episode = _episodeService.FindEpisodeByTitle(series.Id, 0, title); if (episode != null) { @@ -326,6 +326,8 @@ namespace NzbDrone.Core.Parser info.FullSeason = false; info.Quality = QualityParser.ParseQuality(title); info.ReleaseGroup = Parser.ParseReleaseGroup(title); + info.Language = Parser.ParseLanguage(title); + info.Special = true; _logger.Info("Found special episode {0} for title '{1}'", info, title); return info; diff --git a/src/NzbDrone.Core/Tv/EpisodeService.cs b/src/NzbDrone.Core/Tv/EpisodeService.cs index 81852354d..991fc2f00 100644 --- a/src/NzbDrone.Core/Tv/EpisodeService.cs +++ b/src/NzbDrone.Core/Tv/EpisodeService.cs @@ -17,7 +17,7 @@ namespace NzbDrone.Core.Tv List GetEpisodes(IEnumerable ids); Episode FindEpisode(int seriesId, int seasonNumber, int episodeNumber); Episode FindEpisode(int seriesId, int absoluteEpisodeNumber); - Episode FindEpisodeByName(int seriesId, int seasonNumber, string episodeTitle); + Episode FindEpisodeByTitle(int seriesId, int seasonNumber, string episodeTitle); List FindEpisodesBySceneNumbering(int seriesId, int seasonNumber, int episodeNumber); Episode FindEpisodeBySceneNumbering(int seriesId, int sceneAbsoluteEpisodeNumber); Episode GetEpisode(int seriesId, String date); @@ -103,10 +103,10 @@ namespace NzbDrone.Core.Tv return _episodeRepository.GetEpisodes(seriesId, seasonNumber); } - public Episode FindEpisodeByName(int seriesId, int seasonNumber, string episodeTitle) + public Episode FindEpisodeByTitle(int seriesId, int seasonNumber, string episodeTitle) { // TODO: can replace this search mechanism with something smarter/faster/better - var search = Parser.Parser.NormalizeEpisodeTitle(episodeTitle); + var search = Parser.Parser.NormalizeEpisodeTitle(episodeTitle).Replace(".", " "); return _episodeRepository.GetEpisodes(seriesId, seasonNumber) .FirstOrDefault(e =>