From 1ad1d73c91d4bdc897431044bd9b28934e44d90a Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Fri, 9 Oct 2015 20:56:40 +0200 Subject: [PATCH] Added tiered indexer requests to support fallback to wildcard queries. --- .../NewznabRequestGeneratorFixture.cs | 76 +++++++--- .../IndexerTests/SeasonSearchFixture.cs | 5 +- .../BitMeTv/BitMeTvRequestGenerator.cs | 26 ++-- .../BroadcastheNetRequestGenerator.cs | 56 +++---- .../Indexers/Fanzub/FanzubRequestGenerator.cs | 26 ++-- .../Indexers/HDBits/HDBitsRequestGenerator.cs | 39 ++--- src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 66 ++++---- .../Indexers/IIndexerRequestGenerator.cs | 14 +- .../IPTorrents/IPTorrentsRequestGenerator.cs | 26 ++-- .../Indexers/IndexerPageableRequest.cs | 25 ++++ .../Indexers/IndexerPageableRequestChain.cs | 51 +++++++ .../KickassTorrentsRequestGenerator.cs | 38 ++--- .../Newznab/NewznabRequestGenerator.cs | 141 +++++++----------- .../Indexers/Nyaa/NyaaRequestGenerator.cs | 32 ++-- .../Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs | 34 ++--- .../Indexers/Rarbg/RarbgRequestGenerator.cs | 32 ++-- .../Indexers/RssIndexerRequestGenerator.cs | 25 ++-- .../TitansOfTv/TitansOfTvRequestGenerator.cs | 40 ++--- .../TorrentRssIndexerRequestGenerator.cs | 26 ++-- .../TorrentRss/TorrentRssSettingsDetector.cs | 2 +- .../TorrentleechRequestGenerator.cs | 26 ++-- src/NzbDrone.Core/NzbDrone.Core.csproj | 2 + 22 files changed, 448 insertions(+), 360 deletions(-) create mode 100644 src/NzbDrone.Core/Indexers/IndexerPageableRequest.cs create mode 100644 src/NzbDrone.Core/Indexers/IndexerPageableRequestChain.cs diff --git a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs index 66cc059b3..20f7f70ea 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs @@ -53,9 +53,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests { var results = Subject.GetRecentRequests(); - results.Should().HaveCount(1); + results.GetAllTiers().Should().HaveCount(1); - var page = results.First().First(); + var page = results.GetAllTiers().First().First(); page.Url.Query.Should().Contain("&cat=1,2,3,4&"); } @@ -67,9 +67,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests var results = Subject.GetRecentRequests(); - results.Should().HaveCount(1); + results.GetAllTiers().Should().HaveCount(1); - var page = results.First().First(); + var page = results.GetAllTiers().First().First(); page.Url.Query.Should().Contain("&cat=1,2,3,4&"); } @@ -79,9 +79,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests { var results = Subject.GetSearchRequests(_animeSearchCriteria); - results.Should().HaveCount(1); + results.GetAllTiers().Should().HaveCount(1); - var page = results.First().First(); + var page = results.GetAllTiers().First().First(); page.Url.Query.Should().Contain("&cat=3,4&"); } @@ -91,9 +91,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests { var results = Subject.GetSearchRequests(_animeSearchCriteria); - results.Should().HaveCount(1); + results.GetAllTiers().Should().HaveCount(1); - var page = results.First().First(); + var page = results.GetAllTiers().First().First(); page.Url.Query.Should().Contain("?t=search&"); } @@ -103,9 +103,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests { var results = Subject.GetSearchRequests(_animeSearchCriteria); - results.Should().HaveCount(1); + results.GetAllTiers().Should().HaveCount(1); - var pages = results.First().Take(3).ToList(); + var pages = results.GetAllTiers().First().Take(3).ToList(); pages[0].Url.Query.Should().Contain("&offset=0&"); pages[1].Url.Query.Should().Contain("&offset=100&"); @@ -117,9 +117,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests { var results = Subject.GetSearchRequests(_animeSearchCriteria); - results.Should().HaveCount(1); + results.GetAllTiers().Should().HaveCount(1); - var pages = results.First().Take(500).ToList(); + var pages = results.GetAllTiers().First().Take(500).ToList(); pages.Count.Should().BeLessThan(500); } @@ -131,9 +131,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests var results = Subject.GetSearchRequests(_singleEpisodeSearchCriteria); - results.Should().HaveCount(1); + results.GetAllTiers().Should().HaveCount(1); - var page = results.First().First(); + var page = results.GetAllTiers().First().First(); page.Url.Query.Should().NotContain("rid=10"); page.Url.Query.Should().Contain("q=Monkey"); @@ -143,9 +143,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests public void should_search_by_rid_if_supported() { var results = Subject.GetSearchRequests(_singleEpisodeSearchCriteria); - results.Should().HaveCount(1); + results.GetTier(0).Should().HaveCount(1); - var page = results.First().First(); + var page = results.GetAllTiers().First().First(); page.Url.Query.Should().Contain("rid=10"); } @@ -156,10 +156,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests _capabilities.SupportedTvSearchParameters = new[] { "q", "season", "ep" }; var results = Subject.GetSearchRequests(_singleEpisodeSearchCriteria); + results.GetTier(0).Should().HaveCount(1); - results.Should().HaveCount(1); - - var page = results.First().First(); + var page = results.GetAllTiers().First().First(); page.Url.Query.Should().NotContain("rid=10"); page.Url.Query.Should().Contain("q=Monkey"); @@ -171,9 +170,9 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests _capabilities.SupportedTvSearchParameters = new[] { "q", "tvdbid", "season", "ep" }; var results = Subject.GetSearchRequests(_singleEpisodeSearchCriteria); - results.Should().HaveCount(1); + results.GetTier(0).Should().HaveCount(1); - var page = results.First().First(); + var page = results.GetAllTiers().First().First(); page.Url.Query.Should().Contain("tvdbid=20"); } @@ -184,12 +183,43 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests _capabilities.SupportedTvSearchParameters = new[] { "q", "tvdbid", "rid", "season", "ep" }; var results = Subject.GetSearchRequests(_singleEpisodeSearchCriteria); - results.Should().HaveCount(1); + results.GetTier(0).Should().HaveCount(1); - var page = results.First().First(); + var page = results.GetAllTiers().First().First(); page.Url.Query.Should().Contain("tvdbid=20"); page.Url.Query.Should().NotContain("rid=10"); } + + [Test] + public void should_use_aggregrated_id_search_if_supported() + { + _capabilities.SupportedTvSearchParameters = new[] { "q", "tvdbid", "rid", "season", "ep" }; + _capabilities.SupportsAggregateIdSearch = true; + + var results = Subject.GetSearchRequests(_singleEpisodeSearchCriteria); + results.GetTier(0).Should().HaveCount(1); + + var page = results.GetTier(0).First().First(); + + page.Url.Query.Should().Contain("tvdbid=20"); + page.Url.Query.Should().Contain("rid=10"); + } + + [Test] + public void should_fallback_to_q() + { + _capabilities.SupportedTvSearchParameters = new[] { "q", "tvdbid", "rid", "season", "ep" }; + _capabilities.SupportsAggregateIdSearch = true; + + var results = Subject.GetSearchRequests(_singleEpisodeSearchCriteria); + results.Tiers.Should().Be(2); + + var pageTier2 = results.GetTier(1).First().First(); + + pageTier2.Url.Query.Should().NotContain("tvdbid=20"); + pageTier2.Url.Query.Should().NotContain("rid=10"); + pageTier2.Url.Query.Should().Contain("q="); + } } } diff --git a/src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs index 1244814d6..075bb73e2 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs @@ -43,8 +43,11 @@ namespace NzbDrone.Core.Test.IndexerTests .With(v => v.HttpRequest.Method = HttpMethod.GET) .Build(); + var pageable = new IndexerPageableRequestChain(); + pageable.Add(requests); + requestGenerator.Setup(s => s.GetSearchRequests(It.IsAny())) - .Returns(new List> { requests }); + .Returns(pageable); var parser = Mocker.GetMock(); Subject._parser = parser.Object; diff --git a/src/NzbDrone.Core/Indexers/BitMeTv/BitMeTvRequestGenerator.cs b/src/NzbDrone.Core/Indexers/BitMeTv/BitMeTvRequestGenerator.cs index 826e4537f..ff9165b6a 100644 --- a/src/NzbDrone.Core/Indexers/BitMeTv/BitMeTvRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/BitMeTv/BitMeTvRequestGenerator.cs @@ -10,38 +10,38 @@ namespace NzbDrone.Core.Indexers.BitMeTv { public BitMeTvSettings Settings { get; set; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetRssRequests()); + pageableRequests.Add(GetRssRequests()); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } private IEnumerable GetRssRequests() diff --git a/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNetRequestGenerator.cs b/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNetRequestGenerator.cs index 24fb674c9..7a8034440 100644 --- a/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNetRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/BroadcastheNet/BroadcastheNetRequestGenerator.cs @@ -19,18 +19,18 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet PageSize = 100; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, null)); + pageableRequests.Add(GetPagedRequests(MaxPages, null)); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - var pageableRequest = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var parameters = new BroadcastheNetTorrentQuery(); if (AddSeriesSearchParameters(parameters, searchCriteria)) @@ -42,7 +42,7 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet parameters.Category = "Episode"; parameters.Name = string.Format("S{0:00}E{1:00}", episode.SeasonNumber, episode.EpisodeNumber); - pageableRequest.AddIfNotNull(GetPagedRequests(MaxPages, parameters)); + pageableRequests.Add(GetPagedRequests(MaxPages, parameters)); } foreach (var seasonNumber in searchCriteria.Episodes.Select(v => v.SeasonNumber).Distinct()) @@ -52,42 +52,42 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet parameters.Category = "Season"; parameters.Name = string.Format("Season {0}", seasonNumber); - pageableRequest.AddIfNotNull(GetPagedRequests(MaxPages, parameters)); + pageableRequests.Add(GetPagedRequests(MaxPages, parameters)); } } - return pageableRequest; + return pageableRequests; } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - var pageableRequest = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var parameters = new BroadcastheNetTorrentQuery(); if (AddSeriesSearchParameters(parameters, searchCriteria)) { foreach (var seasonNumber in searchCriteria.Episodes.Select(v => v.SeasonNumber).Distinct()) { - parameters.Category = "Episode"; - parameters.Name = string.Format("S{0:00}E%", seasonNumber); + parameters.Category = "Season"; + parameters.Name = string.Format("Season {0}", seasonNumber); - pageableRequest.AddIfNotNull(GetPagedRequests(MaxPages, parameters)); + pageableRequests.Add(GetPagedRequests(MaxPages, parameters)); parameters = parameters.Clone(); - parameters.Category = "Season"; - parameters.Name = string.Format("Season {0}", seasonNumber); + parameters.Category = "Episode"; + parameters.Name = string.Format("S{0:00}E%", seasonNumber); - pageableRequest.AddIfNotNull(GetPagedRequests(MaxPages, parameters)); + pageableRequests.Add(GetPagedRequests(MaxPages, parameters)); } } - return pageableRequest; + return pageableRequests; } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - var pageableRequest = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var parameters = new BroadcastheNetTorrentQuery(); if (AddSeriesSearchParameters(parameters, searchCriteria)) @@ -95,15 +95,15 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet parameters.Category = "Episode"; parameters.Name = string.Format("{0:yyyy}.{0:MM}.{0:dd}", searchCriteria.AirDate); - pageableRequest.AddIfNotNull(GetPagedRequests(MaxPages, parameters)); + pageableRequests.Add(GetPagedRequests(MaxPages, parameters)); } - return pageableRequest; + return pageableRequests; } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - var pageableRequest = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var parameters = new BroadcastheNetTorrentQuery(); if (AddSeriesSearchParameters(parameters, searchCriteria)) @@ -115,7 +115,7 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet parameters.Category = "Episode"; parameters.Name = string.Format("S{0:00}E{1:00}", episode.SeasonNumber, episode.EpisodeNumber); - pageableRequest.AddIfNotNull(GetPagedRequests(MaxPages, parameters)); + pageableRequests.Add(GetPagedRequests(MaxPages, parameters)); } foreach (var seasonNumber in searchCriteria.Episodes.Select(v => v.SeasonNumber).Distinct()) @@ -125,16 +125,16 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet parameters.Category = "Season"; parameters.Name = string.Format("Season {0}", seasonNumber); - pageableRequest.AddIfNotNull(GetPagedRequests(MaxPages, parameters)); + pageableRequests.Add(GetPagedRequests(MaxPages, parameters)); } } - return pageableRequest; + return pageableRequests; } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } private bool AddSeriesSearchParameters(BroadcastheNetTorrentQuery parameters, SearchCriteriaBase searchCriteria) diff --git a/src/NzbDrone.Core/Indexers/Fanzub/FanzubRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Fanzub/FanzubRequestGenerator.cs index 89bf9fc0a..b16d8d620 100644 --- a/src/NzbDrone.Core/Indexers/Fanzub/FanzubRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Fanzub/FanzubRequestGenerator.cs @@ -21,33 +21,33 @@ namespace NzbDrone.Core.Indexers.Fanzub PageSize = 100; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests(null)); + pageableRequests.Add(GetPagedRequests(null)); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var searchTitles = searchCriteria.QueryTitles.SelectMany(v => GetTitleSearchStrings(v, searchCriteria.AbsoluteEpisodeNumber)).ToList(); @@ -56,9 +56,9 @@ namespace NzbDrone.Core.Indexers.Fanzub return pageableRequests; } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } private IEnumerable GetPagedRequests(string query) diff --git a/src/NzbDrone.Core/Indexers/HDBits/HDBitsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/HDBits/HDBitsRequestGenerator.cs index bf3979cad..9adcf1c46 100644 --- a/src/NzbDrone.Core/Indexers/HDBits/HDBitsRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/HDBits/HDBitsRequestGenerator.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; using NzbDrone.Core.IndexerSearch.Definitions; @@ -11,18 +12,18 @@ namespace NzbDrone.Core.Indexers.HDBits { public HDBitsSettings Settings { get; set; } - public IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); pageableRequests.Add(GetRequest(new TorrentQuery())); return pageableRequests; } - public IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - var requests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var queryBase = new TorrentQuery(); if (TryAddSearchParameters(queryBase, searchCriteria)) @@ -36,32 +37,32 @@ namespace NzbDrone.Core.Indexers.HDBits } } - return requests; + return pageableRequests; } - public IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - var requests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var query = new TorrentQuery(); if (TryAddSearchParameters(query, searchCriteria)) { query.Search = string.Format("{0:yyyy}-{0:MM}-{0:dd}", searchCriteria.AirDate); - requests.Add(GetRequest(query)); + pageableRequests.Add(GetRequest(query)); } - return requests; + return pageableRequests; } - public IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - var requests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var queryBase = new TorrentQuery(); if (TryAddSearchParameters(queryBase, searchCriteria)) @@ -72,16 +73,16 @@ namespace NzbDrone.Core.Indexers.HDBits query.TvdbInfo.Season = seasonNumber; - requests.Add(GetRequest(query)); + pageableRequests.Add(GetRequest(query)); } } - return requests; + return pageableRequests; } - public IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - var requests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var queryBase = new TorrentQuery(); if (TryAddSearchParameters(queryBase, searchCriteria)) @@ -93,11 +94,11 @@ namespace NzbDrone.Core.Indexers.HDBits query.TvdbInfo.Season = episode.SeasonNumber; query.TvdbInfo.Episode = episode.EpisodeNumber; - requests.Add(GetRequest(query)); + pageableRequests.Add(GetRequest(query)); } } - return requests; + return pageableRequests; } private bool TryAddSearchParameters(TorrentQuery query, SearchCriteriaBase searchCriteria) diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index b4fdee029..5c3eb3e6d 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -111,7 +111,7 @@ namespace NzbDrone.Core.Indexers return FetchReleases(generator.GetSearchRequests(searchCriteria)); } - protected virtual IList FetchReleases(IList> pageableRequests, bool isRecent = false) + protected virtual IList FetchReleases(IndexerPageableRequestChain pageableRequestChain, bool isRecent = false) { var releases = new List(); var url = string.Empty; @@ -127,51 +127,61 @@ namespace NzbDrone.Core.Indexers lastReleaseInfo = _indexerStatusService.GetLastRssSyncReleaseInfo(Definition.Id); } - foreach (var pageableRequest in pageableRequests) + for (int i = 0; i < pageableRequestChain.Tiers; i++) { - var pagedReleases = new List(); + var pageableRequests = pageableRequestChain.GetTier(i); - foreach (var request in pageableRequest) + foreach (var pageableRequest in pageableRequests) { - url = request.Url.ToString(); + var pagedReleases = new List(); - var page = FetchPage(request, parser); + foreach (var request in pageableRequest) + { + url = request.Url.ToString(); - pagedReleases.AddRange(page); + var page = FetchPage(request, parser); - if (isRecent && page.Any()) - { - if (lastReleaseInfo == null) + pagedReleases.AddRange(page); + + if (isRecent && page.Any()) { - fullyUpdated = true; - break; + if (lastReleaseInfo == null) + { + fullyUpdated = true; + break; + } + var oldestReleaseDate = page.Select(v => v.PublishDate).Min(); + if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl)) + { + fullyUpdated = true; + break; + } + + if (pagedReleases.Count >= MaxNumResultsPerQuery && + oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24)) + { + fullyUpdated = false; + break; + } } - var oldestReleaseDate = page.Select(v => v.PublishDate).Min(); - if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl)) + else if (pagedReleases.Count >= MaxNumResultsPerQuery) { - fullyUpdated = true; break; } - if (pagedReleases.Count >= MaxNumResultsPerQuery && - oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24)) + if (!IsFullPage(page)) { - fullyUpdated = false; break; } } - else if (pagedReleases.Count >= MaxNumResultsPerQuery) - { - break; - } - if (!IsFullPage(page)) - { - break; - } + releases.AddRange(pagedReleases); } - releases.AddRange(pagedReleases); + if (releases.Any()) + { + break; + } } if (isRecent && !releases.Empty()) @@ -277,7 +287,7 @@ namespace NzbDrone.Core.Indexers { var parser = GetParser(); var generator = GetRequestGenerator(); - var releases = FetchPage(generator.GetRecentRequests().First().First(), parser); + var releases = FetchPage(generator.GetRecentRequests().GetAllTiers().First().First(), parser); if (releases.Empty()) { diff --git a/src/NzbDrone.Core/Indexers/IIndexerRequestGenerator.cs b/src/NzbDrone.Core/Indexers/IIndexerRequestGenerator.cs index 02ef5ee3e..5926af8da 100644 --- a/src/NzbDrone.Core/Indexers/IIndexerRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/IIndexerRequestGenerator.cs @@ -5,11 +5,11 @@ namespace NzbDrone.Core.Indexers { public interface IIndexerRequestGenerator { - IList> GetRecentRequests(); - IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria); - IList> GetSearchRequests(SeasonSearchCriteria searchCriteria); - IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria); - IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria); - IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria); + IndexerPageableRequestChain GetRecentRequests(); + IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria); + IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria); + IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria); + IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria); + IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria); } -} +} \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrentsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrentsRequestGenerator.cs index d69adc270..c9f238e38 100644 --- a/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrentsRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/IPTorrents/IPTorrentsRequestGenerator.cs @@ -9,38 +9,38 @@ namespace NzbDrone.Core.Indexers.IPTorrents { public IPTorrentsSettings Settings { get; set; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetRssRequests()); + pageableRequests.Add(GetRssRequests()); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } private IEnumerable GetRssRequests() diff --git a/src/NzbDrone.Core/Indexers/IndexerPageableRequest.cs b/src/NzbDrone.Core/Indexers/IndexerPageableRequest.cs new file mode 100644 index 000000000..dff34143e --- /dev/null +++ b/src/NzbDrone.Core/Indexers/IndexerPageableRequest.cs @@ -0,0 +1,25 @@ +using System.Collections; +using System.Collections.Generic; + +namespace NzbDrone.Core.Indexers +{ + public class IndexerPageableRequest : IEnumerable + { + private readonly IEnumerable _enumerable; + + public IndexerPageableRequest(IEnumerable enumerable) + { + _enumerable = enumerable; + } + + public IEnumerator GetEnumerator() + { + return _enumerable.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return _enumerable.GetEnumerator(); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerPageableRequestChain.cs b/src/NzbDrone.Core/Indexers/IndexerPageableRequestChain.cs new file mode 100644 index 000000000..498f21ec1 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/IndexerPageableRequestChain.cs @@ -0,0 +1,51 @@ +using System.Collections.Generic; +using System.Linq; + +namespace NzbDrone.Core.Indexers +{ + public class IndexerPageableRequestChain + { + private List> _chains; + + public IndexerPageableRequestChain() + { + _chains = new List>(); + _chains.Add(new List()); + } + + public int Tiers + { + get { return _chains.Count; } + } + + public IEnumerable GetAllTiers() + { + return _chains.SelectMany(v => v); + } + + public IEnumerable GetTier(int index) + { + return _chains[index]; + } + + public void Add(IEnumerable request) + { + if (request == null) return; + + _chains.Last().Add(new IndexerPageableRequest(request)); + } + + public void AddTier(IEnumerable request) + { + AddTier(); + Add(request); + } + + public void AddTier() + { + if (_chains.Last().Count == 0) return; + + _chains.Add(new List()); + } + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs index b887ec112..b8541e83a 100644 --- a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs @@ -20,28 +20,28 @@ namespace NzbDrone.Core.Indexers.KickassTorrents PageSize = 25; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, "tv")); + pageableRequests.Add(GetPagedRequests(MaxPages, "tv")); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.QueryTitles) { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, "usearch", + pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", PrepareQuery(queryTitle), "category:tv", string.Format("season:{0}", searchCriteria.SeasonNumber), string.Format("episode:{0}", searchCriteria.EpisodeNumber))); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, "usearch", + pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", PrepareQuery(queryTitle), string.Format("S{0:00}E{1:00}", searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber), "category:tv")); @@ -50,18 +50,18 @@ namespace NzbDrone.Core.Indexers.KickassTorrents return pageableRequests; } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.QueryTitles) { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, "usearch", + pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", PrepareQuery(queryTitle), "category:tv", string.Format("season:{0}", searchCriteria.SeasonNumber))); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, "usearch", + pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", PrepareQuery(queryTitle), "category:tv", string.Format("S{0:00}", searchCriteria.SeasonNumber))); @@ -70,13 +70,13 @@ namespace NzbDrone.Core.Indexers.KickassTorrents return pageableRequests; } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.QueryTitles) { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, "usearch", + pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", PrepareQuery(queryTitle), string.Format("{0:yyyy-MM-dd}", searchCriteria.AirDate), "category:tv")); @@ -85,18 +85,18 @@ namespace NzbDrone.Core.Indexers.KickassTorrents return pageableRequests; } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.EpisodeQueryTitles) { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, "usearch", + pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", PrepareQuery(queryTitle), "category:tv")); } diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs index 61ca41b12..c88a1b79f 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs @@ -72,128 +72,63 @@ namespace NzbDrone.Core.Indexers.Newznab } } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); var capabilities = _capabilitiesProvider.GetCapabilities(Settings); if (capabilities.SupportedTvSearchParameters != null) { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", "")); + pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", "")); } return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - if (searchCriteria.Series.TvdbId > 0 && SupportsTvdbSearch) - { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&tvdbid={0}&season={1}&ep={2}", - searchCriteria.Series.TvdbId, - searchCriteria.SeasonNumber, - searchCriteria.EpisodeNumber))); - } - else if (searchCriteria.Series.TvRageId > 0 && SupportsTvRageSearch) - { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&rid={0}&season={1}&ep={2}", - searchCriteria.Series.TvRageId, + AddTvIdPageableRequests(pageableRequests, MaxPages, Settings.Categories, searchCriteria, + string.Format("&season={0}&ep={1}", searchCriteria.SeasonNumber, - searchCriteria.EpisodeNumber))); - } - else if (SupportsTvSearch) - { - foreach (var queryTitle in searchCriteria.QueryTitles) - { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&q={0}&season={1}&ep={2}", - NewsnabifyTitle(queryTitle), - searchCriteria.SeasonNumber, - searchCriteria.EpisodeNumber))); - } - } + searchCriteria.EpisodeNumber)); return pageableRequests; } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - if (searchCriteria.Series.TvdbId > 0 && SupportsTvdbSearch) - { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&tvdbid={0}&season={1}", - searchCriteria.Series.TvdbId, - searchCriteria.SeasonNumber))); - } - else if (searchCriteria.Series.TvRageId > 0 && SupportsTvRageSearch) - { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&rid={0}&season={1}", - searchCriteria.Series.TvRageId, - searchCriteria.SeasonNumber))); - } - else if (SupportsTvSearch) - { - foreach (var queryTitle in searchCriteria.QueryTitles) - { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&q={0}&season={1}", - NewsnabifyTitle(queryTitle), - searchCriteria.SeasonNumber))); - } - } + AddTvIdPageableRequests(pageableRequests, MaxPages, Settings.Categories, searchCriteria, + string.Format("&season={0}", + searchCriteria.SeasonNumber)); return pageableRequests; } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - if (searchCriteria.Series.TvdbId > 0 && SupportsTvdbSearch) - { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&tvdbid={0}&season={1:yyyy}&ep={1:MM}/{1:dd}", - searchCriteria.Series.TvdbId, - searchCriteria.AirDate))); - } - else if (searchCriteria.Series.TvRageId > 0 && SupportsTvRageSearch) - { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&rid={0}&season={1:yyyy}&ep={1:MM}/{1:dd}", - searchCriteria.Series.TvRageId, - searchCriteria.AirDate))); - } - else if (SupportsTvSearch) - { - foreach (var queryTitle in searchCriteria.QueryTitles) - { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&q={0}&season={1:yyyy}&ep={1:MM}/{1:dd}", - NewsnabifyTitle(queryTitle), - searchCriteria.AirDate))); - } - } + AddTvIdPageableRequests(pageableRequests, MaxPages, Settings.Categories, searchCriteria, + string.Format("&season={0:yyyy}&ep={0:MM}/{0:dd}", + searchCriteria.AirDate)); return pageableRequests; } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); if (SupportsSearch) { foreach (var queryTitle in searchCriteria.QueryTitles) { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.AnimeCategories, "search", + pageableRequests.Add(GetPagedRequests(MaxPages, Settings.AnimeCategories, "search", string.Format("&q={0}+{1:00}", NewsnabifyTitle(queryTitle), searchCriteria.AbsoluteEpisodeNumber))); @@ -203,9 +138,9 @@ namespace NzbDrone.Core.Indexers.Newznab return pageableRequests; } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); if (SupportsSearch) { @@ -214,7 +149,7 @@ namespace NzbDrone.Core.Indexers.Newznab var query = queryTitle.Replace('+', ' '); query = System.Web.HttpUtility.UrlEncode(query); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "search", + pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "search", string.Format("&q={0}", query))); } @@ -223,6 +158,34 @@ namespace NzbDrone.Core.Indexers.Newznab return pageableRequests; } + private void AddTvIdPageableRequests(IndexerPageableRequestChain chain, int maxPages, IEnumerable categories, SearchCriteriaBase searchCriteria, string parameters) + { + { + if (searchCriteria.Series.TvdbId > 0 && SupportsTvdbSearch) + { + chain.Add(GetPagedRequests(maxPages, categories, "tvsearch", + string.Format("&tvdbid={0}{1}", searchCriteria.Series.TvdbId, parameters))); + } + else if (searchCriteria.Series.TvRageId > 0 && SupportsTvRageSearch) + { + chain.Add(GetPagedRequests(maxPages, categories, "tvsearch", + string.Format("&rid={0}{1}", searchCriteria.Series.TvRageId, parameters))); + } + } + + if (SupportsTvSearch) + { + chain.AddTier(); + foreach (var queryTitle in searchCriteria.QueryTitles) + { + chain.Add(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", + string.Format("&q={0}{1}", + NewsnabifyTitle(queryTitle), + parameters))); + } + } + } + private IEnumerable GetPagedRequests(int maxPages, IEnumerable categories, string searchType, string parameters) { if (categories.Empty()) diff --git a/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs index 543b26ee9..23bc8ce6d 100644 --- a/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Nyaa/NyaaRequestGenerator.cs @@ -19,46 +19,46 @@ namespace NzbDrone.Core.Indexers.Nyaa PageSize = 100; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, null)); + pageableRequests.Add(GetPagedRequests(MaxPages, null)); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.QueryTitles) { var searchTitle = PrepareQuery(queryTitle); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, + pageableRequests.Add(GetPagedRequests(MaxPages, string.Format("&term={0}+{1:0}", searchTitle, searchCriteria.AbsoluteEpisodeNumber))); if (searchCriteria.AbsoluteEpisodeNumber < 10) { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, + pageableRequests.Add(GetPagedRequests(MaxPages, string.Format("&term={0}+{1:00}", searchTitle, searchCriteria.AbsoluteEpisodeNumber))); @@ -68,13 +68,13 @@ namespace NzbDrone.Core.Indexers.Nyaa return pageableRequests; } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.EpisodeQueryTitles) { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, + pageableRequests.Add(GetPagedRequests(MaxPages, string.Format("&term={0}", PrepareQuery(queryTitle)))); } diff --git a/src/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs index a557d0e32..d9c0ea2f8 100644 --- a/src/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsRequestGenerator.cs @@ -17,22 +17,22 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs BaseUrl = "https://rss.omgwtfnzbs.org/rss-download.php"; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests(null)); + pageableRequests.Add(GetPagedRequests(null)); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.QueryTitles) { - pageableRequests.AddIfNotNull(GetPagedRequests(string.Format("{0}+S{1:00}E{2:00}", + pageableRequests.Add(GetPagedRequests(string.Format("{0}+S{1:00}E{2:00}", queryTitle, searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber))); @@ -41,13 +41,13 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs return pageableRequests; } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.QueryTitles) { - pageableRequests.AddIfNotNull(GetPagedRequests(string.Format("{0}+S{1:00}", + pageableRequests.Add(GetPagedRequests(string.Format("{0}+S{1:00}", queryTitle, searchCriteria.SeasonNumber))); } @@ -55,13 +55,13 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs return pageableRequests; } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.QueryTitles) { - pageableRequests.AddIfNotNull(GetPagedRequests(string.Format("{0}+{1:yyyy MM dd}", + pageableRequests.Add(GetPagedRequests(string.Format("{0}+{1:yyyy MM dd}", queryTitle, searchCriteria.AirDate))); } @@ -69,21 +69,21 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs return pageableRequests; } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); foreach (var queryTitle in searchCriteria.EpisodeQueryTitles) { var query = queryTitle.Replace('+', ' '); query = System.Web.HttpUtility.UrlEncode(query); - pageableRequests.AddIfNotNull(GetPagedRequests(query)); + pageableRequests.Add(GetPagedRequests(query)); } return pageableRequests; diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs index 3c27c8dd6..342e7d767 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs @@ -17,50 +17,50 @@ namespace NzbDrone.Core.Indexers.Rarbg _tokenProvider = tokenProvider; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests("list", null, null)); + pageableRequests.Add(GetPagedRequests("list", null, null)); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}E{1:00}", searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber)); + pageableRequests.Add(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}E{1:00}", searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber)); return pageableRequests; } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}", searchCriteria.SeasonNumber)); + pageableRequests.Add(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}", searchCriteria.SeasonNumber)); return pageableRequests; } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests("search", searchCriteria.Series.TvdbId, "\"{0:yyyy MM dd}\"", searchCriteria.AirDate)); + pageableRequests.Add(GetPagedRequests("search", searchCriteria.Series.TvdbId, "\"{0:yyyy MM dd}\"", searchCriteria.AirDate)); return pageableRequests; } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } private IEnumerable GetPagedRequests(string mode, int? tvdbId, string query, params object[] args) diff --git a/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs b/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs index 0c94604ad..a8f8b6e2f 100644 --- a/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.IndexerSearch.Definitions; @@ -15,38 +16,38 @@ namespace NzbDrone.Core.Indexers } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); pageableRequests.Add(new[] { new IndexerRequest(_baseUrl, HttpAccept.Rss) }); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } } } diff --git a/src/NzbDrone.Core/Indexers/TitansOfTv/TitansOfTvRequestGenerator.cs b/src/NzbDrone.Core/Indexers/TitansOfTv/TitansOfTvRequestGenerator.cs index a7f665087..fc88432db 100644 --- a/src/NzbDrone.Core/Indexers/TitansOfTv/TitansOfTvRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/TitansOfTv/TitansOfTvRequestGenerator.cs @@ -19,62 +19,64 @@ namespace NzbDrone.Core.Indexers.TitansOfTv PageSize = 100; } - public IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages)); + pageableRequests.Add(GetPagedRequests(MaxPages)); return pageableRequests; } - public IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, + pageableRequests.Add(GetPagedRequests(MaxPages, series_id: searchCriteria.Series.TvdbId, episode: string.Format("S{0:00}E{1:00}", searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber))); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, + pageableRequests.Add(GetPagedRequests(MaxPages, series_id: searchCriteria.Series.TvdbId, season: string.Format("Season {0:00}", searchCriteria.SeasonNumber))); return pageableRequests; } - public IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - // TODO: Search for all episodes?!? - - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, + pageableRequests.Add(GetPagedRequests(MaxPages, series_id: searchCriteria.Series.TvdbId, season: string.Format("Season {0:00}", searchCriteria.SeasonNumber))); + pageableRequests.AddTier(); + + // TODO: Search for all episodes?!? + return pageableRequests; } - public IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, + pageableRequests.Add(GetPagedRequests(MaxPages, series_id: searchCriteria.Series.TvdbId, air_date: searchCriteria.AirDate)); return pageableRequests; } - public IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } private IEnumerable GetPagedRequests(int maxPages, int? series_id = null, string episode = null, string season = null, DateTime? air_date = null) diff --git a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexerRequestGenerator.cs b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexerRequestGenerator.cs index b288a5acd..ea5526cf9 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexerRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssIndexerRequestGenerator.cs @@ -10,38 +10,38 @@ namespace NzbDrone.Core.Indexers.TorrentRss { public TorrentRssIndexerSettings Settings { get; set; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetRssRequests(null)); + pageableRequests.Add(GetRssRequests(null)); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } private IEnumerable GetRssRequests(string searchParameters) diff --git a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs index fb98b7b46..1b58b83bf 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs @@ -41,7 +41,7 @@ namespace NzbDrone.Core.Indexers.TorrentRss _logger.Debug("Evaluating TorrentRss feed '{0}'", indexerSettings.BaseUrl); var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = indexerSettings }; - var request = requestGenerator.GetRecentRequests().First().First(); + var request = requestGenerator.GetRecentRequests().GetAllTiers().First().First(); HttpResponse httpResponse = null; try diff --git a/src/NzbDrone.Core/Indexers/Torrentleech/TorrentleechRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Torrentleech/TorrentleechRequestGenerator.cs index dc6e6c72f..699f163a6 100644 --- a/src/NzbDrone.Core/Indexers/Torrentleech/TorrentleechRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Torrentleech/TorrentleechRequestGenerator.cs @@ -10,38 +10,38 @@ namespace NzbDrone.Core.Indexers.Torrentleech { public TorrentleechSettings Settings { get; set; } - public virtual IList> GetRecentRequests() + public virtual IndexerPageableRequestChain GetRecentRequests() { - var pageableRequests = new List>(); + var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.AddIfNotNull(GetRssRequests(null)); + pageableRequests.Add(GetRssRequests(null)); return pageableRequests; } - public virtual IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } - public virtual IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - return new List>(); + return new IndexerPageableRequestChain(); } private IEnumerable GetRssRequests(string searchParameters) diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 620ae5dc8..b5f216998 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -503,6 +503,8 @@ + +