Fixed: omgwtfnzbs season/series searches stuck in a loop

Fixed:  MegaSearch season/series searches stuck in a loop

src/NzbDrone.Core.Test/IndexerTests/IndexerServiceFixture.cs

@@ -27,7 +27,6 @@ namespace NzbDrone.Core.Test.IndexerTests
             _indexers.Add(new Wombles());
 
             Mocker.SetConstant<IEnumerable<IIndexer>>(_indexers);
-
         }
 
         [Test]
@@ -61,7 +60,6 @@ namespace NzbDrone.Core.Test.IndexerTests
             indexers.Select(c => c.Name).Should().OnlyHaveUniqueItems();
         }
 
-
         [Test]
         public void should_remove_missing_indexers_on_startup()
         {
@@ -69,13 +67,11 @@ namespace NzbDrone.Core.Test.IndexerTests
 
             Mocker.SetConstant<IIndexerRepository>(repo);
 
-
             var existingIndexers = Builder<IndexerDefinition>.CreateNew().BuildNew();
             existingIndexers.ConfigContract = typeof (NewznabSettings).Name;
 
             repo.Insert(existingIndexers);
 
-
             Subject.Handle(new ApplicationStartedEvent());
 
             AllStoredModels.Should().NotContain(c => c.Id == existingIndexers.Id);

src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs

@@ -20,7 +20,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests
         public void SetUp()
         {
             UseRealHttp();
-
         }
 
         [Test]
@@ -39,7 +38,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests
             ValidateResult(result, skipSize: true, skipInfo: true);
         }
 
-
         [Test]
         public void extv_rss()
         {
@@ -55,7 +53,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests
             ValidateTorrentResult(result, skipSize: false, skipInfo: true);
         }
 
-
         [Test]
         public void nzbsorg_rss()
         {
@@ -74,8 +71,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests
             ValidateResult(result);
         }
         
-
-
         private void ValidateResult(IList<ReleaseInfo> reports, bool skipSize = false, bool skipInfo = false)
         {
             reports.Should().NotBeEmpty();
@@ -97,7 +92,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests
 
         private void ValidateTorrentResult(IList<ReleaseInfo> reports, bool skipSize = false, bool skipInfo = false)
         {
-
             reports.Should().OnlyContain(c => c.GetType() == typeof(TorrentInfo));
 
             ValidateResult(reports, skipSize, skipInfo);

src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs

@@ -0,0 +1,92 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.Remoting.Messaging;
+using System.Text;
+using FizzWare.NBuilder;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Newznab;
+using NzbDrone.Core.Indexers.Omgwtfnzbs;
+using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Tv;
+using NzbDrone.Test.Common;
+
+namespace NzbDrone.Core.Test.IndexerTests
+{
+    [TestFixture]
+    public class SeasonSearchFixture : TestBase<FetchFeedService>
+    {
+        private Series _series;
+        private IIndexer _newznab;
+        private IIndexer _omgwtfnzbs;
+
+        [SetUp]
+        public void Setup()
+        {
+            _series = Builder<Series>.CreateNew().Build();
+
+            _newznab = new Newznab();
+
+            _newznab.Definition = new IndexerDefinition();
+            _newznab.Definition.Name = "nzbs.org";
+            _newznab.Definition.Settings = new NewznabSettings
+            {
+                ApiKey = "",
+                Url = "http://nzbs.org"
+            };
+
+            _omgwtfnzbs = new Omgwtfnzbs();
+
+            _omgwtfnzbs.Definition = new IndexerDefinition();
+            _omgwtfnzbs.Definition.Name = "omgwtfnzbs";
+            _omgwtfnzbs.Definition.Settings = new OmgwtfnzbsSettings
+                                              {
+                                                  ApiKey = "",
+                                                  Username = "NzbDrone"
+                                              };
+        }
+
+        private void WithResults(int count)
+        {
+            var results = Builder<ReleaseInfo>.CreateListOfSize(count)
+                .Build();
+
+            Mocker.GetMock<IIndexerParsingService>()
+                  .Setup(s => s.Parse(It.IsAny<IIndexer>(), It.IsAny<String>(), It.IsAny<String>()))
+                  .Returns(results);
+
+            Mocker.GetMock<IHttpProvider>().Setup(s => s.DownloadString(It.IsAny<String>())).Returns("<xml></xml>");
+        }
+
+        [Test]
+        public void should_not_use_offset_if_result_count_is_less_than_90()
+        {
+            WithResults(25);
+            Subject.Fetch(_newznab, new SeasonSearchCriteria { Series = _series, SceneTitle =  _series.Title });
+
+            Mocker.GetMock<IHttpProvider>().Verify(v => v.DownloadString(It.IsAny<String>()), Times.Once());
+        }
+
+        [Test]
+        public void should_not_use_offset_for_sites_that_do_not_support_it()
+        {
+            WithResults(25);
+            Subject.Fetch(_omgwtfnzbs, new SeasonSearchCriteria { Series = _series, SceneTitle = _series.Title });
+
+            Mocker.GetMock<IHttpProvider>().Verify(v => v.DownloadString(It.IsAny<String>()), Times.Once());
+        }
+
+        [Test]
+        public void should_not_use_offset_if_its_already_tried_10_times()
+        {
+            WithResults(100);
+            Subject.Fetch(_newznab, new SeasonSearchCriteria { Series = _series, SceneTitle = _series.Title });
+
+            Mocker.GetMock<IHttpProvider>().Verify(v => v.DownloadString(It.IsAny<String>()), Times.Exactly(11));
+        }
+    }
+}

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -139,6 +139,7 @@
     <Compile Include="IndexerTests\IndexerServiceFixture.cs" />
     <Compile Include="IndexerTests\IntegrationTests\IndexerIntegrationTests.cs" />
     <Compile Include="IndexerTests\NewznabTests\NewznabSettingFixture.cs" />
+    <Compile Include="IndexerTests\SeasonSearchFixture.cs" />
     <Compile Include="IndexerTests\XElementExtensionsFixture.cs" />
     <Compile Include="JobTests\JobRepositoryFixture.cs" />
     <Compile Include="DecisionEngineTests\LanguageSpecificationFixture.cs" />

src/NzbDrone.Core/Indexers/Eztv/Eztv.cs

@@ -14,6 +14,14 @@ namespace NzbDrone.Core.Indexers.Eztv
             }
         }
 
+        public override bool SupportsPaging
+        {
+            get
+            {
+                return false;
+            }
+        }
+
         public override IParseFeed Parser
         {
             get

src/NzbDrone.Core/Indexers/IIndexer.cs

@@ -8,6 +8,7 @@ namespace NzbDrone.Core.Indexers
     {
         IParseFeed Parser { get; }
         DownloadProtocol Protocol { get; }
+        Boolean SupportsPaging { get; }
 
         IEnumerable<string> RecentFeed { get; }
         IEnumerable<string> GetEpisodeSearchUrls(string seriesTitle, int tvRageId, int seasonNumber, int episodeNumber);

src/NzbDrone.Core/Indexers/IParseFeed.cs

@@ -5,6 +5,6 @@ namespace NzbDrone.Core.Indexers
 {
     public interface IParseFeed
     {
-        IEnumerable<ReleaseInfo> Process(string source, string url);
+        IEnumerable<ReleaseInfo> Process(string xml, string url);
     }
 }

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -34,6 +34,8 @@ namespace NzbDrone.Core.Indexers
 
         public abstract DownloadProtocol Protocol { get; }
 
+        public abstract bool SupportsPaging { get; }
+
         protected TSettings Settings
         {
             get

src/NzbDrone.Core/Indexers/IndexerFetchService.cs

@@ -13,7 +13,6 @@ namespace NzbDrone.Core.Indexers
     public interface IFetchFeedFromIndexers
     {
         IList<ReleaseInfo> FetchRss(IIndexer indexer);
-
         IList<ReleaseInfo> Fetch(IIndexer indexer, SeasonSearchCriteria searchCriteria);
         IList<ReleaseInfo> Fetch(IIndexer indexer, SingleEpisodeSearchCriteria searchCriteria);
         IList<ReleaseInfo> Fetch(IIndexer indexer, DailyEpisodeSearchCriteria searchCriteria);
@@ -23,11 +22,12 @@ namespace NzbDrone.Core.Indexers
     {
         private readonly Logger _logger;
         private readonly IHttpProvider _httpProvider;
+        private readonly IIndexerParsingService _indexerParsingService;
 
-
+        public FetchFeedService(IHttpProvider httpProvider, IIndexerParsingService indexerParsingService, Logger logger)
-        public FetchFeedService(IHttpProvider httpProvider, Logger logger)
         {
             _httpProvider = httpProvider;
+            _indexerParsingService = indexerParsingService;
             _logger = logger;
         }
 
@@ -60,12 +60,13 @@ namespace NzbDrone.Core.Indexers
             var searchUrls = indexer.GetSeasonSearchUrls(searchCriteria.QueryTitle, searchCriteria.Series.TvRageId, searchCriteria.SeasonNumber, offset);
             var result = Fetch(indexer, searchUrls);
 
-
             _logger.Info("{0} offset {1}. Found {2}", indexer, searchCriteria, result.Count);
 
-            if (result.Count > 90)
+            if (result.Count > 90 && 
+                offset < 1000 &&
+                indexer.SupportsPaging)
             {
-                result.AddRange(Fetch(indexer, searchCriteria, offset + 90));
+                result.AddRange(Fetch(indexer, searchCriteria, offset + 100));
             }
 
             return result;
@@ -106,7 +107,7 @@ namespace NzbDrone.Core.Indexers
                     var xml = _httpProvider.DownloadString(url);
                     if (!string.IsNullOrWhiteSpace(xml))
                     {
-                        result.AddRange(indexer.Parser.Process(xml, url));
+                        result.AddRange(_indexerParsingService.Parse(indexer, xml, url));
                     }
                     else
                     {

src/NzbDrone.Core/Indexers/IndexerParsingService.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Indexers
+{
+    public interface IIndexerParsingService
+    {
+        IEnumerable<ReleaseInfo> Parse(IIndexer indexer, string xml, string url);
+    }
+
+    public class IndexerParsingService : IIndexerParsingService
+    {
+        public IEnumerable<ReleaseInfo> Parse(IIndexer indexer, string xml, string url)
+        {
+            return indexer.Parser.Process(xml, url);
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Newznab/Newznab.cs

@@ -55,7 +55,6 @@ namespace NzbDrone.Core.Indexers.Newznab
                 });
 
                 return list;
-
             }
         }
 
@@ -73,6 +72,14 @@ namespace NzbDrone.Core.Indexers.Newznab
             return settings;
         }
 
+        public override bool SupportsPaging
+        {
+            get
+            {
+                return true;
+            }
+        }
+
         public override IEnumerable<string> RecentFeed
         {
             get

src/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs

@@ -66,5 +66,13 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs
 
             return searchUrls;
         }
+
+        public override bool SupportsPaging
+        {
+            get
+            {
+                return false;
+            }
+        }
     }
 }

src/NzbDrone.Core/Indexers/Wombles/Wombles.cs

@@ -14,6 +14,14 @@ namespace NzbDrone.Core.Indexers.Wombles
             }
         }
 
+        public override bool SupportsPaging
+        {
+            get
+            {
+                return false;
+            }
+        }
+
         public override IParseFeed Parser
         {
             get

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -270,6 +270,7 @@
     <Compile Include="Indexers\Eztv\Eztv.cs" />
     <Compile Include="Indexers\FetchAndParseRssService.cs" />
     <Compile Include="Indexers\IIndexer.cs" />
+    <Compile Include="Indexers\IndexerParsingService.cs" />
     <Compile Include="Indexers\IndexerSettingUpdatedEvent.cs" />
     <Compile Include="Indexers\NewznabTestService.cs" />
     <Compile Include="Indexers\IParseFeed.cs" />