From 5ac55b042134c88d00aa922330948ec201acab42 Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Mon, 13 Jan 2014 19:47:45 -0800 Subject: [PATCH] Fixed: omgwtfnzbs season/series searches stuck in a loop Fixed: MegaSearch season/series searches stuck in a loop --- .../IndexerTests/IndexerServiceFixture.cs | 4 - .../IndexerIntegrationTests.cs | 8 +- .../IndexerTests/SeasonSearchFixture.cs | 92 +++++++++++++++++++ .../NzbDrone.Core.Test.csproj | 1 + src/NzbDrone.Core/Indexers/Eztv/Eztv.cs | 8 ++ src/NzbDrone.Core/Indexers/IIndexer.cs | 1 + src/NzbDrone.Core/Indexers/IParseFeed.cs | 2 +- src/NzbDrone.Core/Indexers/IndexerBase.cs | 2 + .../Indexers/IndexerFetchService.cs | 15 +-- .../Indexers/IndexerParsingService.cs | 18 ++++ src/NzbDrone.Core/Indexers/Newznab/Newznab.cs | 9 +- .../Indexers/Omgwtfnzbs/Omgwtfnzbs.cs | 8 ++ src/NzbDrone.Core/Indexers/Wombles/Wombles.cs | 8 ++ src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + 14 files changed, 157 insertions(+), 20 deletions(-) create mode 100644 src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs create mode 100644 src/NzbDrone.Core/Indexers/IndexerParsingService.cs diff --git a/src/NzbDrone.Core.Test/IndexerTests/IndexerServiceFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/IndexerServiceFixture.cs index bbfb8404a..6a7a65736 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IndexerServiceFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IndexerServiceFixture.cs @@ -27,7 +27,6 @@ namespace NzbDrone.Core.Test.IndexerTests _indexers.Add(new Wombles()); Mocker.SetConstant>(_indexers); - } [Test] @@ -61,7 +60,6 @@ namespace NzbDrone.Core.Test.IndexerTests indexers.Select(c => c.Name).Should().OnlyHaveUniqueItems(); } - [Test] public void should_remove_missing_indexers_on_startup() { @@ -69,13 +67,11 @@ namespace NzbDrone.Core.Test.IndexerTests Mocker.SetConstant(repo); - var existingIndexers = Builder.CreateNew().BuildNew(); existingIndexers.ConfigContract = typeof (NewznabSettings).Name; repo.Insert(existingIndexers); - Subject.Handle(new ApplicationStartedEvent()); AllStoredModels.Should().NotContain(c => c.Id == existingIndexers.Id); diff --git a/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs b/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs index 28dc053e4..1d12f233e 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs @@ -20,7 +20,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests public void SetUp() { UseRealHttp(); - } [Test] @@ -39,7 +38,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests ValidateResult(result, skipSize: true, skipInfo: true); } - [Test] public void extv_rss() { @@ -55,7 +53,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests ValidateTorrentResult(result, skipSize: false, skipInfo: true); } - [Test] public void nzbsorg_rss() { @@ -73,9 +70,7 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests ValidateResult(result); } - - - + private void ValidateResult(IList reports, bool skipSize = false, bool skipInfo = false) { reports.Should().NotBeEmpty(); @@ -97,7 +92,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests private void ValidateTorrentResult(IList reports, bool skipSize = false, bool skipInfo = false) { - reports.Should().OnlyContain(c => c.GetType() == typeof(TorrentInfo)); ValidateResult(reports, skipSize, skipInfo); diff --git a/src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs new file mode 100644 index 000000000..7c8a87485 --- /dev/null +++ b/src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.Remoting.Messaging; +using System.Text; +using FizzWare.NBuilder; +using Moq; +using NUnit.Framework; +using NzbDrone.Common; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.Newznab; +using NzbDrone.Core.Indexers.Omgwtfnzbs; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Tv; +using NzbDrone.Test.Common; + +namespace NzbDrone.Core.Test.IndexerTests +{ + [TestFixture] + public class SeasonSearchFixture : TestBase + { + private Series _series; + private IIndexer _newznab; + private IIndexer _omgwtfnzbs; + + [SetUp] + public void Setup() + { + _series = Builder.CreateNew().Build(); + + _newznab = new Newznab(); + + _newznab.Definition = new IndexerDefinition(); + _newznab.Definition.Name = "nzbs.org"; + _newznab.Definition.Settings = new NewznabSettings + { + ApiKey = "", + Url = "http://nzbs.org" + }; + + _omgwtfnzbs = new Omgwtfnzbs(); + + _omgwtfnzbs.Definition = new IndexerDefinition(); + _omgwtfnzbs.Definition.Name = "omgwtfnzbs"; + _omgwtfnzbs.Definition.Settings = new OmgwtfnzbsSettings + { + ApiKey = "", + Username = "NzbDrone" + }; + } + + private void WithResults(int count) + { + var results = Builder.CreateListOfSize(count) + .Build(); + + Mocker.GetMock() + .Setup(s => s.Parse(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(results); + + Mocker.GetMock().Setup(s => s.DownloadString(It.IsAny())).Returns(""); + } + + [Test] + public void should_not_use_offset_if_result_count_is_less_than_90() + { + WithResults(25); + Subject.Fetch(_newznab, new SeasonSearchCriteria { Series = _series, SceneTitle = _series.Title }); + + Mocker.GetMock().Verify(v => v.DownloadString(It.IsAny()), Times.Once()); + } + + [Test] + public void should_not_use_offset_for_sites_that_do_not_support_it() + { + WithResults(25); + Subject.Fetch(_omgwtfnzbs, new SeasonSearchCriteria { Series = _series, SceneTitle = _series.Title }); + + Mocker.GetMock().Verify(v => v.DownloadString(It.IsAny()), Times.Once()); + } + + [Test] + public void should_not_use_offset_if_its_already_tried_10_times() + { + WithResults(100); + Subject.Fetch(_newznab, new SeasonSearchCriteria { Series = _series, SceneTitle = _series.Title }); + + Mocker.GetMock().Verify(v => v.DownloadString(It.IsAny()), Times.Exactly(11)); + } + } +} diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index 8e4f7cf7c..d1da02111 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -139,6 +139,7 @@ + diff --git a/src/NzbDrone.Core/Indexers/Eztv/Eztv.cs b/src/NzbDrone.Core/Indexers/Eztv/Eztv.cs index 52c55df60..5f141eacb 100644 --- a/src/NzbDrone.Core/Indexers/Eztv/Eztv.cs +++ b/src/NzbDrone.Core/Indexers/Eztv/Eztv.cs @@ -14,6 +14,14 @@ namespace NzbDrone.Core.Indexers.Eztv } } + public override bool SupportsPaging + { + get + { + return false; + } + } + public override IParseFeed Parser { get diff --git a/src/NzbDrone.Core/Indexers/IIndexer.cs b/src/NzbDrone.Core/Indexers/IIndexer.cs index 34daa6a26..2f850d8de 100644 --- a/src/NzbDrone.Core/Indexers/IIndexer.cs +++ b/src/NzbDrone.Core/Indexers/IIndexer.cs @@ -8,6 +8,7 @@ namespace NzbDrone.Core.Indexers { IParseFeed Parser { get; } DownloadProtocol Protocol { get; } + Boolean SupportsPaging { get; } IEnumerable RecentFeed { get; } IEnumerable GetEpisodeSearchUrls(string seriesTitle, int tvRageId, int seasonNumber, int episodeNumber); diff --git a/src/NzbDrone.Core/Indexers/IParseFeed.cs b/src/NzbDrone.Core/Indexers/IParseFeed.cs index 2722669b8..8410d5e04 100644 --- a/src/NzbDrone.Core/Indexers/IParseFeed.cs +++ b/src/NzbDrone.Core/Indexers/IParseFeed.cs @@ -5,6 +5,6 @@ namespace NzbDrone.Core.Indexers { public interface IParseFeed { - IEnumerable Process(string source, string url); + IEnumerable Process(string xml, string url); } } \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 7d42847cd..bca173fcc 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -34,6 +34,8 @@ namespace NzbDrone.Core.Indexers public abstract DownloadProtocol Protocol { get; } + public abstract bool SupportsPaging { get; } + protected TSettings Settings { get diff --git a/src/NzbDrone.Core/Indexers/IndexerFetchService.cs b/src/NzbDrone.Core/Indexers/IndexerFetchService.cs index 2de0c51b0..8dc19cd35 100644 --- a/src/NzbDrone.Core/Indexers/IndexerFetchService.cs +++ b/src/NzbDrone.Core/Indexers/IndexerFetchService.cs @@ -13,7 +13,6 @@ namespace NzbDrone.Core.Indexers public interface IFetchFeedFromIndexers { IList FetchRss(IIndexer indexer); - IList Fetch(IIndexer indexer, SeasonSearchCriteria searchCriteria); IList Fetch(IIndexer indexer, SingleEpisodeSearchCriteria searchCriteria); IList Fetch(IIndexer indexer, DailyEpisodeSearchCriteria searchCriteria); @@ -23,11 +22,12 @@ namespace NzbDrone.Core.Indexers { private readonly Logger _logger; private readonly IHttpProvider _httpProvider; + private readonly IIndexerParsingService _indexerParsingService; - - public FetchFeedService(IHttpProvider httpProvider, Logger logger) + public FetchFeedService(IHttpProvider httpProvider, IIndexerParsingService indexerParsingService, Logger logger) { _httpProvider = httpProvider; + _indexerParsingService = indexerParsingService; _logger = logger; } @@ -60,12 +60,13 @@ namespace NzbDrone.Core.Indexers var searchUrls = indexer.GetSeasonSearchUrls(searchCriteria.QueryTitle, searchCriteria.Series.TvRageId, searchCriteria.SeasonNumber, offset); var result = Fetch(indexer, searchUrls); - _logger.Info("{0} offset {1}. Found {2}", indexer, searchCriteria, result.Count); - if (result.Count > 90) + if (result.Count > 90 && + offset < 1000 && + indexer.SupportsPaging) { - result.AddRange(Fetch(indexer, searchCriteria, offset + 90)); + result.AddRange(Fetch(indexer, searchCriteria, offset + 100)); } return result; @@ -106,7 +107,7 @@ namespace NzbDrone.Core.Indexers var xml = _httpProvider.DownloadString(url); if (!string.IsNullOrWhiteSpace(xml)) { - result.AddRange(indexer.Parser.Process(xml, url)); + result.AddRange(_indexerParsingService.Parse(indexer, xml, url)); } else { diff --git a/src/NzbDrone.Core/Indexers/IndexerParsingService.cs b/src/NzbDrone.Core/Indexers/IndexerParsingService.cs new file mode 100644 index 000000000..09530baae --- /dev/null +++ b/src/NzbDrone.Core/Indexers/IndexerParsingService.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Indexers +{ + public interface IIndexerParsingService + { + IEnumerable Parse(IIndexer indexer, string xml, string url); + } + + public class IndexerParsingService : IIndexerParsingService + { + public IEnumerable Parse(IIndexer indexer, string xml, string url) + { + return indexer.Parser.Process(xml, url); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Newznab/Newznab.cs b/src/NzbDrone.Core/Indexers/Newznab/Newznab.cs index 8a1be6f66..985ee208b 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/Newznab.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/Newznab.cs @@ -55,7 +55,6 @@ namespace NzbDrone.Core.Indexers.Newznab }); return list; - } } @@ -73,6 +72,14 @@ namespace NzbDrone.Core.Indexers.Newznab return settings; } + public override bool SupportsPaging + { + get + { + return true; + } + } + public override IEnumerable RecentFeed { get diff --git a/src/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs b/src/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs index c4daeab66..4869b7c2b 100644 --- a/src/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs +++ b/src/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs @@ -66,5 +66,13 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs return searchUrls; } + + public override bool SupportsPaging + { + get + { + return false; + } + } } } diff --git a/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs b/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs index 5355d853d..4570022f1 100644 --- a/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs +++ b/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs @@ -14,6 +14,14 @@ namespace NzbDrone.Core.Indexers.Wombles } } + public override bool SupportsPaging + { + get + { + return false; + } + } + public override IParseFeed Parser { get diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 7967454d8..cda67438f 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -270,6 +270,7 @@ +