New: Use PageSize reported by newznab/torznab caps instead of hardcoded 100.

ref Jackett/Jackett#27

src/NzbDrone.Core.Test/Files/Indexers/Newznab/newznab_caps.xml

@@ -0,0 +1,31 @@
+<caps>
+  <server appversion="" version="0.1" />
+  <limits max="60" default="25"/>
+  <registration available="yes" open="no"/>
+  <searching>
+    <search available="yes"/>
+    <tv-search available="yes"/>
+    <movie-search available="yes"/>
+    <audio-search available="yes"/>
+  </searching>
+  <categories>
+    <category id="5000" name="TV">
+      <subcat id="5070" name="Anime"/>
+      <subcat id="5080" name="Documentary"/>
+      <subcat id="5020" name="Foreign"/>
+      <subcat id="5040" name="HD"/>
+      <subcat id="5050" name="Other"/>
+      <subcat id="5030" name="SD"/>
+      <subcat id="5060" name="Sport"/>
+      <subcat id="5010" name="WEB-DL"/>
+    </category>
+    <category id="7000" name="Other">
+      <subcat id="7010" name="Misc"/>
+    </category>
+    <category id="8000" name="Books">
+      <subcat id="8020" name="Comics"/>
+    </category>
+  </categories>
+  <groups></groups>
+  <genres></genres>
+</caps>

src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs

@@ -0,0 +1,68 @@
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers.Newznab;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
+{
+    [TestFixture]
+    public class NewznabCapabilitiesProviderFixture : CoreTest<NewznabCapabilitiesProvider>
+    {
+        private NewznabSettings _settings;
+        private string _caps;
+
+        [SetUp]
+        public void SetUp()
+        {
+            _settings = new NewznabSettings()
+            {
+                Url = "http://indxer.local"
+            };
+
+            _caps = ReadAllText("Files", "Indexers", "Newznab", "newznab_caps.xml");
+        }
+
+        private void GivenCapsResponse(string caps)
+        {
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Get(It.IsAny<HttpRequest>()))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), caps));
+        }
+
+        [Test]
+        public void should_not_request_same_caps_twice()
+        {
+            GivenCapsResponse(_caps);
+
+            Subject.GetCapabilities(_settings);
+            Subject.GetCapabilities(_settings);
+
+            Mocker.GetMock<IHttpClient>()
+                .Verify(o => o.Get(It.IsAny<HttpRequest>()), Times.Once());
+        }
+
+        [Test]
+        public void should_report_pagesize()
+        {
+            GivenCapsResponse(_caps);
+
+            var caps = Subject.GetCapabilities(_settings);
+
+            caps.DefaultPageSize.Should().Be(25);
+            caps.MaxPageSize.Should().Be(60);
+        }
+
+        [Test]
+        public void should_use_default_pagesize_if_missing()
+        {
+            GivenCapsResponse(_caps.Replace("<limits", "<abclimits"));
+
+            var caps = Subject.GetCapabilities(_settings);
+
+            caps.DefaultPageSize.Should().Be(100);
+            caps.MaxPageSize.Should().Be(100);
+        }
+    }
+}

src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs

@@ -14,6 +14,8 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
     [TestFixture]
     public class NewznabFixture : CoreTest<Newznab>
     {
+        private NewznabCapabilities _caps;
+
         [SetUp]
         public void Setup()
         {
@@ -28,9 +30,10 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
                         }
                 };
 
+            _caps = new NewznabCapabilities();
             Mocker.GetMock<INewznabCapabilitiesProvider>()
                 .Setup(v => v.GetCapabilities(It.IsAny<NewznabSettings>()))
-                .Returns(new NewznabCapabilities());
+                .Returns(_caps);
         }
 
         [Test]
@@ -58,5 +61,14 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
             releaseInfo.PublishDate.Should().Be(DateTime.Parse("2012/02/27 16:09:39"));
             releaseInfo.Size.Should().Be(1183105773);
         }
+
+        [Test]
+        public void should_use_pagesize_reported_by_caps()
+        {
+            _caps.MaxPageSize = 30;
+            _caps.DefaultPageSize = 25;
+
+            Subject.PageSize.Should().Be(25);
+        }
     }
 }

src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs

@@ -15,6 +15,8 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests
     [TestFixture]
     public class TorznabFixture : CoreTest<Torznab>
     {
+        private NewznabCapabilities _caps;
+
         [SetUp]
         public void Setup()
         {
@@ -28,9 +30,10 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests
                         }
                 };
 
+            _caps = new NewznabCapabilities();
             Mocker.GetMock<INewznabCapabilitiesProvider>()
                 .Setup(v => v.GetCapabilities(It.IsAny<NewznabSettings>()))
-                .Returns(new NewznabCapabilities());
+                .Returns(_caps);
         }
 
         [Test]
@@ -93,5 +96,14 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests
             releaseInfo.Seeders.Should().Be(34128);
             releaseInfo.Peers.Should().Be(36724);
         }
+
+        [Test]
+        public void should_use_pagesize_reported_by_caps()
+        {
+            _caps.MaxPageSize = 30;
+            _caps.DefaultPageSize = 25;
+
+            Subject.PageSize.Should().Be(25);
+        }
     }
 }

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -226,6 +226,7 @@
     <Compile Include="IndexerTests\IndexerServiceFixture.cs" />
     <Compile Include="IndexerTests\IndexerStatusServiceFixture.cs" />
     <Compile Include="IndexerTests\IntegrationTests\IndexerIntegrationTests.cs" />
+    <Compile Include="IndexerTests\NewznabTests\NewznabCapabilitiesProviderFixture.cs" />
     <Compile Include="IndexerTests\RarbgTests\RarbgFixture.cs" />
     <Compile Include="IndexerTests\TorrentRssIndexerTests\TorrentRssParserFactoryFixture.cs" />
     <Compile Include="IndexerTests\TorrentRssIndexerTests\TorrentRssSettingsDetectorFixture.cs" />
@@ -409,6 +410,9 @@
     <Content Include="Files\Indexers\KickassTorrents\KickassTorrents_accents.xml">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
+    <Content Include="Files\Indexers\Newznab\newznab_caps.xml">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
     <Content Include="Files\Indexers\relative_urls.xml">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>

src/NzbDrone.Core/Indexers/Newznab/Newznab.cs

@@ -24,7 +24,10 @@ namespace NzbDrone.Core.Indexers.Newznab
         }
 
         public override DownloadProtocol Protocol { get { return DownloadProtocol.Usenet; } }
-        public override int PageSize { get { return 100; } }
+        public override int PageSize
+        {
+            get { return _capabilitiesProvider.GetCapabilities(Settings).DefaultPageSize; }
+        }
 
         public override IIndexerRequestGenerator GetRequestGenerator()
         {

src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilities.cs

@@ -5,6 +5,8 @@ namespace NzbDrone.Core.Indexers.Newznab
 {
     public class NewznabCapabilities
     {
+        public int DefaultPageSize { get; set; }
+        public int MaxPageSize { get; set; }
         public string[] SupportedSearchParameters { get; set; }
         public string[] SupportedTvSearchParameters { get; set; }
         public bool SupportsAggregateIdSearch { get; set; }
@@ -12,6 +14,8 @@ namespace NzbDrone.Core.Indexers.Newznab
 
         public NewznabCapabilities()
         {
+            DefaultPageSize = 100;
+            MaxPageSize = 100;
             SupportedSearchParameters = new[] { "q" };
             SupportedTvSearchParameters = new[] { "q", "rid", "season", "ep" }; // This should remain 'rid' for older newznab installs.
             SupportsAggregateIdSearch = false;

src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs

@@ -68,6 +68,13 @@ namespace NzbDrone.Core.Indexers.Newznab
 
             var xmlRoot = XDocument.Parse(response.Content).Element("caps");
 
+            var xmlLimits = xmlRoot.Element("limits");
+            if (xmlLimits != null)
+            {
+                capabilities.DefaultPageSize = int.Parse(xmlLimits.Attribute("default").Value);
+                capabilities.MaxPageSize = int.Parse(xmlLimits.Attribute("max").Value);
+            }
+
             var xmlSearching = xmlRoot.Element("searching");
             if (xmlSearching != null)
             {